[Reproducible-commits] [blog] 02/02: Initial work on a draft generator.
Chris Lamb
chris at chris-lamb.co.uk
Sun Jul 31 16:40:45 UTC 2016
This is an automated email from the git hooks/post-receive script.
lamby pushed a commit to branch master
in repository blog.
commit 68beac34c4e70cd917fa5a0534590272e1bbefc0
Author: Chris Lamb <lamby at debian.org>
Date: Sun Jul 31 12:40:35 2016 -0400
Initial work on a draft generator.
UDD schema is weird..
---
bin/generate-draft | 166 ++++++++++++++++++++++++++++++++++++++++++++
bin/generate-draft.template | 71 +++++++++++++++++++
2 files changed, 237 insertions(+)
diff --git a/bin/generate-draft b/bin/generate-draft
new file mode 100755
index 0000000..dfcbb72
--- /dev/null
+++ b/bin/generate-draft
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import sys
+import time
+import jinja2
+import pickle
+import calendar
+import datetime
+import requests
+import debianbts
+import subprocess
+import collections
+
+def main(*args):
+ week = int(args[0])
+
+ data = get_data(week)
+
+ env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(os.path.dirname(__file__))
+ )
+ print(env.get_template('generate-draft.template').render(**data))
+
+ return 0
+
+def log(msg, *args, **kwargs):
+ print("I: " + msg.format(*args, **kwargs), file=sys.stderr)
+
+def get_data(week, max_age=3600):
+ filename = '/tmp/generate-draft-{}.pickle'.format(week)
+
+ try:
+ mtime = os.path.getmtime(filename)
+
+ if mtime >= time.time() - max_age:
+ log("Using cache from {}", filename)
+
+ with open(filename, 'rb') as f:
+ return pickle.load(f)
+ except (EOFError, OSError):
+ pass
+
+ log("Getting new data")
+ week_start = datetime.date(2015, 4, 19) + datetime.timedelta(days=week * 7)
+ week_end = week_start + datetime.timedelta(days=6)
+
+ data = {
+ 'week': week,
+ 'week_start': week_start,
+ 'week_end': week_end,
+ }
+
+ for k, v in (
+ ('author', get_author),
+ ('uploads', get_uploads),
+ ('ftbfs_bugs', get_ftbfs_bugs),
+ ):
+ data[k] = v(week_start, week_end)
+
+ log("Saving cache to {}", filename)
+
+ with open(filename, 'wb') as f:
+ pickle.dump(data, f)
+
+ return data
+
+def get_author(week_start, week_end):
+ return os.environ.get('DEBFULLNAME', 'FIXME')
+
+def get_ftbfs_bugs(week_start, week_end):
+ log("Querying UDD for FTBFS bugs")
+
+ data = udd("""
+ SELECT
+ id,
+ source,
+ submitter,
+ title,
+ arrival
+ FROM
+ bugs
+ INNER JOIN
+ bugs_usertags USING (id)
+ WHERE
+ bugs_usertags.email = 'reproducible-builds at lists.alioth.debian.org'
+ AND
+ bugs_usertags.tag = 'ftbfs'
+ AND
+ arrival BETWEEN '{week_start}' AND '{week_end}'
+ """.format(**{
+ 'week_start': week_start.strftime('%F'),
+ 'week_end': week_end.strftime('%F'),
+ }), ('id', 'source', 'submitter', 'subject', 'arrival'))
+
+ result = {}
+ for x in data:
+ result.setdefault(x['submitter'], []).append(x)
+
+ return result
+
+def get_uploads(week_start, week_end):
+ log("Querying UDD for uploads")
+
+ fields = (
+ 'source',
+ 'version',
+ 'distribution',
+ 'signed_by_name',
+ )
+
+ sources = (
+ 'strip-nondeterminism',
+ 'diffoscope',
+ 'disorderfs',
+ 'reprotest',
+ )
+
+ data = udd("""
+ SELECT
+ {fields}
+ FROM
+ upload_history
+ WHERE
+ source IN ({sources})
+ AND
+ date BETWEEN '{week_start}' AND '{week_end}'
+ ORDER BY
+ date
+ """.format(**{
+ 'fields': ', '.join(fields),
+ 'sources': ', '.join("'{}'".format(x) for x in sources),
+ 'week_start': week_start.strftime('%F'),
+ 'week_end': week_end.strftime('%F'),
+ }), fields)
+
+ result = {}
+ for x in data:
+ result.setdefault(x['source'], []).append(x)
+
+ return result
+
+def udd(query, fields):
+ lines = subprocess.check_output("""
+ echo "{}" | ssh alioth.debian.org psql --no-psqlrc service=udd
+ """.format(query), shell=True)
+
+ data = []
+
+ for line in lines.splitlines()[2:]:
+ split = line.decode('utf-8').split('|')
+
+ if len(split) != len(fields):
+ continue
+
+ row = dict(zip(fields, [x.strip() for x in split]))
+
+ data.append(row)
+
+ return data
+
+
+
+if __name__ == '__main__':
+ sys.exit(main(*sys.argv[1:]))
diff --git a/bin/generate-draft.template b/bin/generate-draft.template
new file mode 100644
index 0000000..393cb5c
--- /dev/null
+++ b/bin/generate-draft.template
@@ -0,0 +1,71 @@
+[[!meta title="Reproducible builds: week {{ week }} in Stretch cycle"]]
+
+What happened in the [Reproducible
+Builds](https://wiki.debian.org/ReproducibleBuilds) effort between {{ week_start.strftime('%B') }} {{ week_start.day }} and {{ week_end.strftime('%B') }} {{ week_start.day }} {{ week_end.year }}:
+
+Media coverage
+--------------
+
+GSoC and Outreachy updates
+--------------------------
+
+Documentation update
+--------------------
+
+Toolchain fixes
+---------------
+
+Other upstream fixes
+--------------------
+
+Reproducible work in other projects
+-----------------------------------
+
+FIXME
+
+Packages reviewed and fixed, and bugs filed
+-------------------------------------------
+
+
+Packages fixed
+--------------
+
+The following XXX packages have become reproducible due to changes in their
+build dependencies:
+
+The following packages have become reproducible after being fixed:
+
+Some uploads have fixed some reproducibility issues, but not all of them:
+
+Patches submitted that have not made their way to the archive yet:
+
+Package reviews
+---------------
+
+XXX reviews have been added, XXX have been updated and XXX have been removed in this week,
+adding to our knowledge about [identified issues](https://tests.reproducible-builds.org/debian/index_issues.html).
+
+XXX new issue types have been found:
+
+Weekly QA work
+--------------
+
+FTBFS bugs have been reported by:
+{% for k, v in ftbfs_bugs.items()|sort %}
+ - {{ k }} ({{ v|length }}){% endfor %}
+
+{% for k, v in uploads.items()|sort %}
+{{ k }} development
+----------------------
+{% for x in v %}
+- {{ k }} {{ x['version'] }} was uploaded to {{ x['distribution'] }} by {{ x['signed_by_name'] }}{% endfor %}
+
+FIXME https://sources.debian.net/src/{{ k }}/latest/debian/changelog/
+{% endfor %}
+tests.reproducible-builds.org
+-----------------------
+
+Misc.
+-----
+
+This week's edition was written by {{ author }} and reviewed by a bunch of Reproducible builds folks on IRC.
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/blog.git
More information about the Reproducible-commits
mailing list