[Reproducible-commits] [misc] 01/01: Add generate-draft from blog.git
Chris Lamb
chris at chris-lamb.co.uk
Sun Aug 14 16:27:43 UTC 2016
This is an automated email from the git hooks/post-receive script.
lamby pushed a commit to branch master
in repository misc.
commit 59bb2d729f2d14d15c4fafbeab5ba015bbf479d8
Author: Chris Lamb <lamby at debian.org>
Date: Sun Aug 14 17:27:08 2016 +0100
Add generate-draft from blog.git
---
reports/bin/generate-draft | 231 ++++++++++++++++++++++++++++++++++++
reports/bin/generate-draft.template | 72 +++++++++++
2 files changed, 303 insertions(+)
diff --git a/reports/bin/generate-draft b/reports/bin/generate-draft
new file mode 100755
index 0000000..48eebc1
--- /dev/null
+++ b/reports/bin/generate-draft
@@ -0,0 +1,231 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import sys
+import time
+import jinja2
+import pickle
+import calendar
+import datetime
+import requests
+import debianbts
+import subprocess
+import collections
+
+PROJECTS = (
+ 'diffoscope',
+ 'strip-nondeterminism',
+ 'disorderfs',
+ 'reprotest',
+)
+
+def main(*args):
+ week = int(args[0])
+
+ data = get_data(week)
+
+ env = jinja2.Environment(
+ loader=jinja2.FileSystemLoader(os.path.dirname(__file__))
+ )
+ print(env.get_template('generate-draft.template').render(**data))
+
+ return 0
+
+def log(msg, *args, **kwargs):
+ print("I: " + msg.format(*args, **kwargs), file=sys.stderr)
+
+def get_data(week, max_age=3600):
+ filename = '/tmp/generate-draft-{}.pickle'.format(week)
+
+ try:
+ mtime = os.path.getmtime(filename)
+
+ if mtime >= time.time() - max_age:
+ log("Using cache from {}", filename)
+
+ with open(filename, 'rb') as f:
+ return pickle.load(f)
+ except (EOFError, OSError):
+ pass
+
+ log("Getting new data")
+
+ week_start = datetime.date(2015, 4, 19) + datetime.timedelta(days=week * 7)
+ week_end = week_start + datetime.timedelta(days=6)
+
+ data = {x: y(week_start, week_end) for x, y in (
+ ('author', get_author),
+ ('commits', get_commits),
+ ('uploads', get_uploads),
+ ('patches', get_patches),
+ ('ftbfs_bugs', get_ftbfs_bugs),
+ ('issues_yml', get_issues_yml),
+ ('packages_yml', get_packages_yml),
+ )}
+
+ data.update({
+ 'week': week,
+ 'week_start': week_start,
+ 'week_end': week_end,
+ 'projects': PROJECTS,
+ })
+
+ log("Saving cache to {}", filename)
+
+ with open(filename, 'wb') as f:
+ pickle.dump(data, f)
+
+ return data
+
+def get_author(week_start, week_end):
+ return os.environ.get('DEBFULLNAME', 'FIXME')
+
+def get_ftbfs_bugs(week_start, week_end):
+ return bugs(
+ week_start,
+ week_end,
+ "bugs_usertags.tag = '{}'".format('ftbfs'),
+ )
+
+def get_patches(week_start, week_end):
+ return bugs(
+ week_start,
+ week_end,
+ "id IN (SELECT id FROM bugs_tags WHERE tag = 'patch')",
+ )
+
+def bugs(week_start, week_end, extra="true"):
+ log("Querying UDD for usertagged bugs with filter: {}", extra)
+
+ fields = (
+ 'id',
+ 'source',
+ 'submitter',
+ 'submitter_name',
+ 'title',
+ 'arrival',
+ )
+
+ sql = """
+ SELECT
+ {fields}
+ FROM
+ bugs
+ INNER JOIN
+ bugs_usertags USING (id)
+ WHERE
+ bugs_usertags.email = 'reproducible-builds at lists.alioth.debian.org'
+ AND
+ {extra}
+ AND
+ CAST(arrival AS DATE) BETWEEN '{week_start}' AND '{week_end}'
+ """.format(**{
+ 'fields': ', '.join(fields),
+ 'extra': extra,
+ 'week_start': week_start.strftime('%F'),
+ 'week_end': week_end.strftime('%F'),
+ })
+
+ result = {}
+ for x in udd(sql, fields):
+ result.setdefault(x['submitter_name'], []).append(x)
+
+ return result
+
+def get_uploads(week_start, week_end):
+ log("Querying UDD for uploads")
+
+ fields = (
+ 'source',
+ 'version',
+ 'distribution',
+ 'signed_by_name',
+ )
+
+ data = udd("""
+ SELECT
+ {fields}
+ FROM
+ upload_history
+ WHERE
+ source IN ({sources})
+ AND
+ CAST(date AS date) BETWEEN '{week_start}' AND '{week_end}'
+ ORDER BY
+ date
+ """.format(**{
+ 'fields': ', '.join(fields),
+ 'sources': ', '.join("'{}'".format(x) for x in PROJECTS),
+ 'week_start': week_start.strftime('%F'),
+ 'week_end': week_end.strftime('%F'),
+ }), fields)
+
+ result = {}
+ for x in data:
+ result.setdefault(x['source'], []).append(x)
+
+ return result
+
+def udd(query, fields):
+ lines = subprocess.check_output("""
+ echo "{}" | ssh alioth.debian.org psql --no-psqlrc service=udd
+ """.format(query), shell=True)
+
+ data = []
+
+ for line in lines.splitlines()[2:]:
+ split = line.decode('utf-8').split('|')
+
+ if len(split) != len(fields):
+ continue
+
+ row = dict(zip(fields, [x.strip() for x in split]))
+
+ data.append(row)
+
+ return data
+
+def get_commits(week_start, week_end):
+ return {x: commits(week_start, week_end, x) for x in PROJECTS}
+
+def get_issues_yml(week_start, week_end):
+ return commits(week_start, week_end, 'notes', 'issues.yml')
+
+def get_packages_yml(week_start, week_end):
+ return commits(week_start, week_end, 'notes', 'packages.yml')
+
+def commits(week_start, week_end, project, path='.'):
+ # Assume its in the parent dir
+ git_dir = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
+ project,
+ '.git',
+ )
+
+ output = subprocess.check_output((
+ 'git',
+ 'log',
+ 'origin/master',
+ '--since', week_start.strftime('%F'),
+ '--until', week_end.strftime('%F'),
+ '--pretty=format:%an\t%h\t%s',
+ '--',
+ path,
+ ), env={
+ 'GIT_DIR': git_dir,
+ }).decode('utf-8')
+
+ result = collections.defaultdict(list)
+ for x in output.splitlines():
+ author, sha, title = x.split('\t', 2)
+
+ result[author].append({
+ 'sha': sha,
+ 'title': title,
+ })
+
+ return result
+
+if __name__ == '__main__':
+ sys.exit(main(*sys.argv[1:]))
diff --git a/reports/bin/generate-draft.template b/reports/bin/generate-draft.template
new file mode 100644
index 0000000..a5f58a9
--- /dev/null
+++ b/reports/bin/generate-draft.template
@@ -0,0 +1,72 @@
+[[!meta title="Reproducible builds: week {{ week }} in Stretch cycle"]]
+
+What happened in the [Reproducible
+Builds](https://wiki.debian.org/ReproducibleBuilds) effort between Sunday {{ week_start.strftime('%B') }} {{ week_start.day }} and Saturday {{ week_end.strftime('%B') }} {{ week_end.day }} {{ week_end.year }}:
+
+Media coverage
+--------------
+
+FIXME
+
+GSoC and Outreachy updates
+--------------------------
+
+FIXME
+
+Documentation update
+--------------------
+
+FIXME
+
+Reproducible work in other projects
+-----------------------------------
+
+FIXME
+
+Packages reviewed and fixed, and bugs filed
+-------------------------------------------
+
+Patches have been submitted by:
+{% for x, ys in patches.items()|sort %}
+- {{ x }}:{% for y in ys %}
+ - [{{ y['title'] }}](https://bugs.debian.org/{{ y['id'] }}){% endfor %}
+{% endfor %}
+
+Package reviews
+---------------
+
+FIXME reviews have been added, FIXME have been updated and FIXME have been removed in this week,
+adding to our knowledge about [identified issues](https://tests.reproducible-builds.org/debian/index_issues.html).
+{% for _, xs in packages_yml.items()|sort %}{% for x in xs %}
+- [{{ x['title'] }}](https://anonscm.debian.org/git/notes.git/commit/?id={{ x['sha'] }}){% endfor %}{% endfor %}
+
+Issue types have been updated:
+{% for _, xs in issues_yml.items()|sort %}{% for x in xs %}
+- [{{ x['title'] }}](https://anonscm.debian.org/git/notes.git/commit/?id={{ x['sha'] }}){% endfor %}{% endfor %}
+
+Weekly QA work
+--------------
+
+FTBFS bugs have been reported by:
+{% for k, v in ftbfs_bugs.items()|sort %}
+ - {{ k }} ({{ v|length }}){% endfor %}
+
+{% for project in projects %}
+{{ project }} development
+------------{{ "-" * project|length }}
+{% for x in uploads[project] %}
+- {{ project }} {{ x['version'] }} was uploaded to {{ x['distribution'] }} by {{ x['signed_by_name'] }}.{% endfor %}
+{% for x, ys in commits[project].items() %}- {{ x }}:{% for y in ys %}
+ - [{{ y['title'] }}](https://anonscm.debian.org/git/reproducible/{{ project }}.git/commit/?id={{ y['sha'] }}){% endfor %}
+{% endfor %}
+{% endfor %}
+
+tests.reproducible-builds.org
+-----------------------
+
+FIXME
+
+Misc.
+-----
+
+This week's edition was written by {{ author }} and reviewed by a bunch of Reproducible builds folks on IRC.
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/misc.git
More information about the Reproducible-commits
mailing list