[Reproducible-commits] [misc] 01/01: reports: various improvements

Ximin Luo infinity0 at debian.org
Mon Aug 22 14:00:08 UTC 2016


This is an automated email from the git hooks/post-receive script.

infinity0 pushed a commit to branch master
in repository misc.

commit 9239de8766dbe243b2699e3af8a60a490b9b8ce1
Author: Ximin Luo <infinity0 at debian.org>
Date:   Mon Aug 22 15:57:01 2016 +0200

    reports: various improvements
    
    - Check git dirs exist before using them
    - Use the prev week if not given on command line
    - Figure out git root dynamically instead of hard-coding it
    - Use epoch timestamps in python (datetime uses local tz)
    - `git fetch` before reading logs
    - Set cwd to gitdir rather than wiping the whole env
---
 reports/bin/generate-draft          | 63 ++++++++++++++++++++++---------------
 reports/bin/generate-draft.template |  2 +-
 reports/bin/get-latest-data         |  2 +-
 3 files changed, 39 insertions(+), 28 deletions(-)

diff --git a/reports/bin/generate-draft b/reports/bin/generate-draft
index 915171f..8416054 100755
--- a/reports/bin/generate-draft
+++ b/reports/bin/generate-draft
@@ -13,6 +13,9 @@ import debianbts
 import subprocess
 import collections
 
+REPO_PARENT = None
+WEEK_1_END = 1430611200 # May 3 2015, 00:00 UTC, Sunday
+
 PROJECTS = (
     'diffoscope',
     'strip-nondeterminism',
@@ -21,7 +24,9 @@ PROJECTS = (
 )
 
 def main(*args):
-    week = int(args[0])
+    for d in map(sibling_repo_gitdir, list(PROJECTS) + ["notes"]):
+        ensure_dir(d)
+    week = int(args[0]) if len(args) > 0 else prev_week()
 
     data = get_data(week)
 
@@ -35,6 +40,21 @@ def main(*args):
 def log(msg, *args, **kwargs):
     print("I: " + msg.format(*args, **kwargs), file=sys.stderr)
 
+def prev_week():
+    now = int(time.time())
+    return ((now - WEEK_1_END) // (7*24*3600) + 1)
+
+def sibling_repo_gitdir(path):
+    global REPO_PARENT
+    if not REPO_PARENT:
+        REPO_PARENT = os.path.dirname(subprocess.check_output(
+            ["git", "rev-parse", "--show-toplevel"]).decode("utf-8"))
+    return os.path.join(REPO_PARENT, path, ".git")
+
+def ensure_dir(path):
+    if not os.path.isdir(path):
+        raise ValueError("not a directory: %s" % path)
+
 def get_data(week, max_age=3600):
     filename = '/tmp/generate-draft-{}.pickle'.format(week)
 
@@ -51,8 +71,8 @@ def get_data(week, max_age=3600):
 
     log("Getting new data")
 
-    week_start = datetime.date(2015, 4, 19) + datetime.timedelta(days=week * 7)
-    week_end = week_start + datetime.timedelta(days=6)
+    week_end = WEEK_1_END + (week - 1)*7*24*3600 # exclusive
+    week_start = week_end - 7*24*3600 # inclusive
 
     data = {x: y(week_start, week_end) for x, y in (
         ('author', get_author),
@@ -66,8 +86,8 @@ def get_data(week, max_age=3600):
 
     data.update({
         'week': week,
-        'week_start': week_start,
-        'week_end': week_end,
+        'week_start': datetime.datetime.utcfromtimestamp(week_start),
+        'week_end': datetime.datetime.utcfromtimestamp(week_end-1),
         'projects': PROJECTS,
     })
 
@@ -119,12 +139,12 @@ def bugs(week_start, week_end, extra="true"):
         AND
             {extra}
         AND
-            CAST(arrival AS DATE) BETWEEN '{week_start}' AND '{week_end}'
+            CAST(arrival AS DATE) BETWEEN to_timestamp({week_start}) AND to_timestamp({week_end})
     """.format(**{
         'fields': ', '.join(fields),
         'extra': extra,
-        'week_start': week_start.strftime('%F'),
-        'week_end': week_end.strftime('%F'),
+        'week_start': '@%s'%week_start,
+        'week_end': '@%s'%week_end,
     })
 
     result = {}
@@ -151,14 +171,14 @@ def get_uploads(week_start, week_end):
         WHERE
             source IN ({sources})
         AND
-            CAST(date AS date) BETWEEN '{week_start}' AND '{week_end}'
+            CAST(date AS date) BETWEEN to_timestamp({week_start}) AND to_timestamp({week_end})
         ORDER BY
             date
     """.format(**{
         'fields': ', '.join(fields),
         'sources': ', '.join("'{}'".format(x) for x in PROJECTS),
-        'week_start': week_start.strftime('%F'),
-        'week_end': week_end.strftime('%F'),
+        'week_start': '@%s'%week_start,
+        'week_end': '@%s'%week_end,
     }), fields)
 
     result = {}
@@ -195,31 +215,22 @@ def get_issues_yml(week_start, week_end):
 def get_packages_yml(week_start, week_end):
     return commits(week_start, week_end, 'notes', 'packages.yml')
 
-def dirname_n(filename, n):
-    for _ in range(n):
-        filename = os.path.dirname(filename)
-    return filename
-
 def commits(week_start, week_end, project, path='.'):
     # Assume its in the parent dir
-    git_dir = os.path.join(
-        dirname_n(os.path.abspath(__file__), 4),
-        project,
-        '.git',
-    )
+    git_dir = sibling_repo_gitdir(project)
 
+    subprocess.check_call(('git', 'fetch', 'origin'), cwd=git_dir)
     output = subprocess.check_output((
         'git',
         'log',
         'origin/master',
-        '--since', week_start.strftime('%F'),
-        '--until',  week_end.strftime('%F'),
+        '--since', '@%s'%week_start,
+        '--until',  '@%s'%week_end,
         '--pretty=format:%an\t%h\t%s',
         '--',
         path,
-    ), env={
-        'GIT_DIR': git_dir,
-    }).decode('utf-8')
+    ), cwd=git_dir
+    ).decode('utf-8')
 
     result = collections.defaultdict(list)
     for x in output.splitlines():
diff --git a/reports/bin/generate-draft.template b/reports/bin/generate-draft.template
index 2bef69c..a415d53 100644
--- a/reports/bin/generate-draft.template
+++ b/reports/bin/generate-draft.template
@@ -1,7 +1,7 @@
 [[!meta title="Reproducible Builds: week {{ week }} in Stretch cycle"]]
 
 What happened in the [Reproducible
-Builds](https://wiki.debian.org/ReproducibleBuilds) effort between Sunday {{ week_start.strftime('%B') }} {{ week_start.day }} and Saturday {{ week_end.strftime('%B') }} {{ week_end.day }} {{ week_end.year }}:
+Builds](https://wiki.debian.org/ReproducibleBuilds) effort between {{ week_start.strftime('%A %B') }} {{ week_start.day }} and {{ week_end.strftime('%A %B') }} {{ week_end.day }} {{ week_end.year }}:
 
 Media coverage
 --------------
diff --git a/reports/bin/get-latest-data b/reports/bin/get-latest-data
index 6c9fbd3..2096ec1 100755
--- a/reports/bin/get-latest-data
+++ b/reports/bin/get-latest-data
@@ -101,7 +101,7 @@ echo >&2 "- bug reports that were modified (will ssh to alioth.debian.org)"
 query_select="SELECT DISTINCT bugs.id FROM bugs_usertags, bugs"
 query_filter="bugs_usertags.email = 'reproducible-builds at lists.alioth.debian.org' \
 AND bugs.id = bugs_usertags.id \
-AND bugs.last_modified > to_timestamp($week_start) \
+AND bugs.last_modified >= to_timestamp($week_start) \
 AND bugs.last_modified < to_timestamp($week_end)"
 query_order="ORDER BY bugs.id"
 udd_query="ssh alioth.debian.org psql service=udd -t"

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/misc.git



More information about the Reproducible-commits mailing list