[Reproducible-commits] [misc] 01/01: reports: various improvements
Ximin Luo
infinity0 at debian.org
Mon Aug 22 14:00:08 UTC 2016
This is an automated email from the git hooks/post-receive script.
infinity0 pushed a commit to branch master
in repository misc.
commit 9239de8766dbe243b2699e3af8a60a490b9b8ce1
Author: Ximin Luo <infinity0 at debian.org>
Date: Mon Aug 22 15:57:01 2016 +0200
reports: various improvements
- Check git dirs exist before using them
- Use the prev week if not given on command line
- Figure out git root dynamically instead of hard-coding it
- Use epoch timestamps in python (datetime uses local tz)
- `git fetch` before reading logs
- Set cwd to gitdir rather than wiping the whole env
---
reports/bin/generate-draft | 63 ++++++++++++++++++++++---------------
reports/bin/generate-draft.template | 2 +-
reports/bin/get-latest-data | 2 +-
3 files changed, 39 insertions(+), 28 deletions(-)
diff --git a/reports/bin/generate-draft b/reports/bin/generate-draft
index 915171f..8416054 100755
--- a/reports/bin/generate-draft
+++ b/reports/bin/generate-draft
@@ -13,6 +13,9 @@ import debianbts
import subprocess
import collections
+REPO_PARENT = None
+WEEK_1_END = 1430611200 # May 3 2015, 00:00 UTC, Sunday
+
PROJECTS = (
'diffoscope',
'strip-nondeterminism',
@@ -21,7 +24,9 @@ PROJECTS = (
)
def main(*args):
- week = int(args[0])
+ for d in map(sibling_repo_gitdir, list(PROJECTS) + ["notes"]):
+ ensure_dir(d)
+ week = int(args[0]) if len(args) > 0 else prev_week()
data = get_data(week)
@@ -35,6 +40,21 @@ def main(*args):
def log(msg, *args, **kwargs):
print("I: " + msg.format(*args, **kwargs), file=sys.stderr)
+def prev_week():
+ now = int(time.time())
+ return ((now - WEEK_1_END) // (7*24*3600) + 1)
+
+def sibling_repo_gitdir(path):
+ global REPO_PARENT
+ if not REPO_PARENT:
+ REPO_PARENT = os.path.dirname(subprocess.check_output(
+ ["git", "rev-parse", "--show-toplevel"]).decode("utf-8"))
+ return os.path.join(REPO_PARENT, path, ".git")
+
+def ensure_dir(path):
+ if not os.path.isdir(path):
+ raise ValueError("not a directory: %s" % path)
+
def get_data(week, max_age=3600):
filename = '/tmp/generate-draft-{}.pickle'.format(week)
@@ -51,8 +71,8 @@ def get_data(week, max_age=3600):
log("Getting new data")
- week_start = datetime.date(2015, 4, 19) + datetime.timedelta(days=week * 7)
- week_end = week_start + datetime.timedelta(days=6)
+ week_end = WEEK_1_END + (week - 1)*7*24*3600 # exclusive
+ week_start = week_end - 7*24*3600 # inclusive
data = {x: y(week_start, week_end) for x, y in (
('author', get_author),
@@ -66,8 +86,8 @@ def get_data(week, max_age=3600):
data.update({
'week': week,
- 'week_start': week_start,
- 'week_end': week_end,
+ 'week_start': datetime.datetime.utcfromtimestamp(week_start),
+ 'week_end': datetime.datetime.utcfromtimestamp(week_end-1),
'projects': PROJECTS,
})
@@ -119,12 +139,12 @@ def bugs(week_start, week_end, extra="true"):
AND
{extra}
AND
- CAST(arrival AS DATE) BETWEEN '{week_start}' AND '{week_end}'
+ CAST(arrival AS DATE) BETWEEN to_timestamp({week_start}) AND to_timestamp({week_end})
""".format(**{
'fields': ', '.join(fields),
'extra': extra,
- 'week_start': week_start.strftime('%F'),
- 'week_end': week_end.strftime('%F'),
+ 'week_start': '@%s'%week_start,
+ 'week_end': '@%s'%week_end,
})
result = {}
@@ -151,14 +171,14 @@ def get_uploads(week_start, week_end):
WHERE
source IN ({sources})
AND
- CAST(date AS date) BETWEEN '{week_start}' AND '{week_end}'
+ CAST(date AS date) BETWEEN to_timestamp({week_start}) AND to_timestamp({week_end})
ORDER BY
date
""".format(**{
'fields': ', '.join(fields),
'sources': ', '.join("'{}'".format(x) for x in PROJECTS),
- 'week_start': week_start.strftime('%F'),
- 'week_end': week_end.strftime('%F'),
+ 'week_start': '@%s'%week_start,
+ 'week_end': '@%s'%week_end,
}), fields)
result = {}
@@ -195,31 +215,22 @@ def get_issues_yml(week_start, week_end):
def get_packages_yml(week_start, week_end):
return commits(week_start, week_end, 'notes', 'packages.yml')
-def dirname_n(filename, n):
- for _ in range(n):
- filename = os.path.dirname(filename)
- return filename
-
def commits(week_start, week_end, project, path='.'):
# Assume its in the parent dir
- git_dir = os.path.join(
- dirname_n(os.path.abspath(__file__), 4),
- project,
- '.git',
- )
+ git_dir = sibling_repo_gitdir(project)
+ subprocess.check_call(('git', 'fetch', 'origin'), cwd=git_dir)
output = subprocess.check_output((
'git',
'log',
'origin/master',
- '--since', week_start.strftime('%F'),
- '--until', week_end.strftime('%F'),
+ '--since', '@%s'%week_start,
+ '--until', '@%s'%week_end,
'--pretty=format:%an\t%h\t%s',
'--',
path,
- ), env={
- 'GIT_DIR': git_dir,
- }).decode('utf-8')
+ ), cwd=git_dir
+ ).decode('utf-8')
result = collections.defaultdict(list)
for x in output.splitlines():
diff --git a/reports/bin/generate-draft.template b/reports/bin/generate-draft.template
index 2bef69c..a415d53 100644
--- a/reports/bin/generate-draft.template
+++ b/reports/bin/generate-draft.template
@@ -1,7 +1,7 @@
[[!meta title="Reproducible Builds: week {{ week }} in Stretch cycle"]]
What happened in the [Reproducible
-Builds](https://wiki.debian.org/ReproducibleBuilds) effort between Sunday {{ week_start.strftime('%B') }} {{ week_start.day }} and Saturday {{ week_end.strftime('%B') }} {{ week_end.day }} {{ week_end.year }}:
+Builds](https://wiki.debian.org/ReproducibleBuilds) effort between {{ week_start.strftime('%A %B') }} {{ week_start.day }} and {{ week_end.strftime('%A %B') }} {{ week_end.day }} {{ week_end.year }}:
Media coverage
--------------
diff --git a/reports/bin/get-latest-data b/reports/bin/get-latest-data
index 6c9fbd3..2096ec1 100755
--- a/reports/bin/get-latest-data
+++ b/reports/bin/get-latest-data
@@ -101,7 +101,7 @@ echo >&2 "- bug reports that were modified (will ssh to alioth.debian.org)"
query_select="SELECT DISTINCT bugs.id FROM bugs_usertags, bugs"
query_filter="bugs_usertags.email = 'reproducible-builds at lists.alioth.debian.org' \
AND bugs.id = bugs_usertags.id \
-AND bugs.last_modified > to_timestamp($week_start) \
+AND bugs.last_modified >= to_timestamp($week_start) \
AND bugs.last_modified < to_timestamp($week_end)"
query_order="ORDER BY bugs.id"
udd_query="ssh alioth.debian.org psql service=udd -t"
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/misc.git
More information about the Reproducible-commits
mailing list