[Reproducible-commits] [misc] 01/01: reports: move network stuff from review-uploads to get-latest-data, and make it work for non-DDs
Ximin Luo
infinity0 at debian.org
Tue May 17 10:44:18 UTC 2016
This is an automated email from the git hooks/post-receive script.
infinity0 pushed a commit to branch master
in repository misc.
commit ed3b752fdef208f6db4c321dd10b8d549e2b59fe
Author: Ximin Luo <infinity0 at debian.org>
Date: Tue May 17 12:43:08 2016 +0200
reports: move network stuff from review-uploads to get-latest-data, and make it work for non-DDs
---
reports/README | 1 -
reports/bin/get-latest-data | 45 +++++++++++++++++++++++++++++++++------
reports/bin/review-uploads | 28 ++++++------------------
reports/bin/sync-master-to-alioth | 7 ++++++
4 files changed, 51 insertions(+), 30 deletions(-)
diff --git a/reports/README b/reports/README
index bbda5c3..597d2b9 100644
--- a/reports/README
+++ b/reports/README
@@ -50,7 +50,6 @@ Process
3. Look at all uploads for the past week.
- $ . ../misc/reports/latest/variables
$ bin/review-uploads
If you can't use that script, the archive is available on the web:
diff --git a/reports/bin/get-latest-data b/reports/bin/get-latest-data
index 76719c7..d66797e 100755
--- a/reports/bin/get-latest-data
+++ b/reports/bin/get-latest-data
@@ -1,7 +1,13 @@
#!/bin/bash
+# Get latest data relevant to the current week, or a week given by -w.
+#
+# If you don't have access to master.debian.org, ask someone who does to run
+# bin/sync-master-to-alioth. After they have done that, you can re-run this
+# script with "-m alioth.debian.org:/home/groups/reproducible/mirror"
+#
+# If you want to use torsocks, give this script TORSOCKS=torsocks
+
set -e
-# if you want to use torsocks:
-# export TORSOCKS=torsocks
scriptdir="$(readlink -f "$(dirname "$0")")"
week_1_end=1430611200 # May 3 2015, 00:00 UTC, Sunday
@@ -11,15 +17,21 @@ now="$(date +%s)"
prev_week=$(((now - week_1_end) / (7*24*3600) + 1))
-while getopts 'w:' opt; do
+while getopts 'w:m:' opt; do
case $opt in
w)
- week=$OPTARG
+ week="$OPTARG"
+ ;;
+ m)
+ mailbase="$OPTARG"
;;
esac
done
+shift `expr $OPTIND - 1`
week="${week:-$prev_week}"
+mailbase="${mailbase:-master.debian.org:/srv/mail-archives/lists}"
+
week_end=$((week_1_end + (week - 1)*7*24*3600))
week_start=$((week_end - 7*24*3600))
week_path="week_${week}_ending_$(date -u -d@${week_end} +%Y-%m-%d)"
@@ -50,8 +62,25 @@ for url in $("$scriptdir/newly-reproducible" -w "$week" | sed -n -e 's,.*<,,;s,>
echo
done
-
-echo >&2 "- bug reports that were modified (via ssh to alioth)"
+echo >&2 "- mails describing uploads to debian (will ssh to $mailbase)"
+current_month="$(date -u +%Y%m)"
+month_at_end="$(date -u +%Y%m -d@$week_end)"
+month_at_start="$(date -u +%Y%m -d@$week_start)"
+
+get_emails() {
+ if [ "$1" = "$current_month" ]; then
+ rsync -v "$mailbase/debian-devel-changes/debian-devel-changes.$1" "debian-devel-changes.$1.snapshot"
+ cat "debian-devel-changes.$1.snapshot" >> uploads.mbox
+
+ else
+ rsync -v "$mailbase/debian-devel-changes/debian-devel-changes.$1.xz" "debian-devel-changes.$1.xz"
+ xzcat "debian-devel-changes.$1.xz" >> uploads.mbox
+ fi
+}
+test "$month_at_start" != "$month_at_end" && get_emails "$month_at_start"
+get_emails "$month_at_end"
+
+echo >&2 "- bug reports that were modified (will ssh to alioth.debian.org)"
echo "SELECT DISTINCT bugs.id FROM bugs_usertags, bugs WHERE email = 'reproducible-builds at lists.alioth.debian.org' AND bugs.id = bugs_usertags.id AND bugs.last_modified > to_timestamp($week_start) AND bugs.last_modified < to_timestamp($week_end) ORDER BY bugs.id;" \
| ssh alioth.debian.org psql service=udd -t \
| awk '/[0-9]/ { print $1 }' > bugs
@@ -67,4 +96,6 @@ fi
)
-rm -f latest && ln -sf "data/$week_path" latest
+if [ "$prev_week" = "$week" ]; then
+ rm -f latest && ln -sf "data/$week_path" latest
+fi
diff --git a/reports/bin/review-uploads b/reports/bin/review-uploads
index 796e737..3ff066d 100755
--- a/reports/bin/review-uploads
+++ b/reports/bin/review-uploads
@@ -3,33 +3,17 @@
# Assumes you have an account on master.debian.org.
set -e
-pattern="${1:-reproduc\|SOURCE_DATE_EPOCH}"
+mbox="${1:-latest/uploads.mbox}"
+pattern="${2:-reproduc\|SOURCE_DATE_EPOCH}"
-quote() { printf "'%s'" "${1//\'/\'\\\'\'}"; }
-mailbase=/srv/mail-archives/lists/debian-devel-changes/debian-devel-changes
+if [ -f "$(dirname "$mbox")/variables" ]; then . "$(dirname "$mbox")/variables"; fi
+if [ -z "$RB_REPORT_WEEK_END" ]; then echo >&2 "abort: RB_REPORT_WEEK_END not set"; exit 1; fi
# get the relevant months
-if [ -z "$RB_REPORT_WEEK_END" ]; then echo >&2 "abort: RB_REPORT_WEEK_END not set"; exit 1; fi
current_month="$(date -u +%Y%m)"
month="$(date -u +%Y%m -d@$RB_REPORT_WEEK_END)"
month_at_start="$(date -u +%Y%m -d@$RB_REPORT_WEEK_START)"
-get_emails=
-append_emails() {
- if [ "$1" = "$current_month" ]; then
- get_emails="${get_emails}cat $mailbase.$1 >> ~/.tmp.mutt-review; "
- else
- get_emails="${get_emails}xzcat $mailbase.$1.xz >> ~/.tmp.mutt-review; "
- fi
-}
-append_emails "$month"
-if [ "$month_at_start" != "$month" ]; then
- append_emails "$month_at_start"
-fi
-
-date_pattern="$(date -u -d"@$RB_REPORT_WEEK_START" +"%d/%m/%y")-$(date -u -d"@$RB_REPORT_WEEK_END" +"%d/%m/%y")"
-command="rm -f ~/.tmp.mutt-review;
-${get_emails}env TZ=UTC mutt -e \"push 'l ~d $date_pattern ~b $pattern<Enter>';\" -R -f ~/.tmp.mutt-review;
-rm -f ~/.tmp.mutt-review;"
+date_pattern="$(date -u -d"@$RB_REPORT_WEEK_START" +"%d/%m/%y")-$(date -u -d"@$((RB_REPORT_WEEK_END - 1))" +"%d/%m/%y")"
-ssh -t master.debian.org sh -c "$(quote "$command")"
+env TZ=UTC mutt -e "push 'l ~d $date_pattern ~b $pattern<Enter>';" -R -f "$mbox"
diff --git a/reports/bin/sync-master-to-alioth b/reports/bin/sync-master-to-alioth
new file mode 100755
index 0000000..ed524c9
--- /dev/null
+++ b/reports/bin/sync-master-to-alioth
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+mkdir -p data/debian-devel-changes
+rsync -av --delete --progress -f '- *19????.?z' -f '- *200???.?z' -f '- *201[0-4]??.?z' \
+ master.debian.org:/srv/mail-archives/lists/debian-devel-changes/ data/debian-devel-changes/
+rsync -av --delete --progress -f '- *19????.?z' -f '- *200???.?z' -f '- *201[0-4]??.?z' \
+ data/debian-devel-changes/ alioth.debian.org:/home/groups/reproducible/mirror/debian-devel-changes/
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/misc.git
More information about the Reproducible-commits
mailing list