[Reproducible-commits] [misc] 03/05: clean-notes: query udd to get usertagged bugs, and be sure we include them in the notes (optional using -d/--db flag)

[Mattia Rizzolo]

This is an automated email from the git hooks/post-receive script.

mapreri-guest pushed a commit to branch master
in repository misc.

commit 82f38fc61d59253321161506d0e107d576c2ec0e
Author: Mattia Rizzolo <mattia at mapreri.org>
Date:   Tue Jan 6 22:39:00 2015 +0100

    clean-notes: query udd to get usertagged bugs, and be sure we include them in the notes (optional using -d/--db flag)
---
 clean-notes | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 98 insertions(+), 1 deletion(-)

diff --git a/clean-notes b/clean-notes
index 1296aea..bde7c15 100755
--- a/clean-notes
+++ b/clean-notes
@@ -13,6 +13,7 @@ import json
 import yaml
 import apt_pkg
 import argparse
+import psycopg2
 import requests
 
 reproducible_json = 'https://reproducible.debian.net/userContent/reproducible.json'
@@ -33,6 +34,28 @@ args = parser.parse_args()
 def warning(*objs):
     print("WARNING: ", *objs, file=sys.stderr)
 
+def start_udd_connection():
+    username = "public-udd-mirror"
+    password = "public-udd-mirror"
+    host = "public-udd-mirror.xvm.mit.edu"
+    port = 5432
+    db = "udd"
+    try:
+        conn = psycopg2.connect("dbname=" + db +
+                               " user=" + username +
+                               " host=" + host +
+                               " password=" + password)
+    except:
+        warning("erorr connecting to the UDD database replica")
+        raise
+    conn.set_client_encoding('utf8')
+    return conn
+
+def query_udd(query):
+    cursor = conn.cursor()
+    cursor.execute(query)
+    return cursor.fetchall()
+
 def load_reproducible_status():
     try:
         with open('reproducible.json') as fd:
@@ -72,6 +95,80 @@ def find_old_notes(testedpkgs, notes):
                     print("The package " + pkg + " has a new tested version")
     return toremove
 
+def is_virtual_package(package):
+    rows = query_udd("""SELECT source FROM sources WHERE source='%s'""" % package)
+    if len(rows) > 0:
+        return False
+    return True
+
+def get_bugs():
+    rows = query_udd("""SELECT * FROM bugs_usertags WHERE
+                        email='reproducible-builds at lists.alioth.debian.org'""")
+    bugs = {}
+    for tag in rows:
+        try:
+            bugs[tag[2]].append(tag[1])
+        except KeyError:
+            bugs[tag[2]] = [tag[1]]
+    return bugs
+
+def parse_bugs(bugs):
+    """
+    This function return a dict:
+    { "package_name": {
+        "bugs": [bug1, bug2],
+         "usertags": ["usertag1", "usertag2"]
+       }
+    }
+
+    The `bugs` argument is {bug_number: ["usertag1", "usertag2"]}
+    """
+    packages = {}
+    ids = ''
+    bugs_list = sorted(bugs.keys())
+    for bug in bugs_list[:-1]:
+        ids += 'id=' + str(bug) + ' OR '
+    ids += 'id=' + str(bugs_list[-1])
+    query = """SELECT id, source, done FROM bugs WHERE %s""" % ids
+    rows = query_udd(query)
+    for item in rows:
+        if item[2]: # do not consider closed bugs
+            continue
+        if is_virtual_package(item[1]):
+            continue
+        try:
+            packages[item[1]]['bugs'].append(item[0])
+        except KeyError:
+            try:
+                packages[item[1]]['bugs'] = [item[0]]
+            except KeyError:
+                packages[item[1]] = {}
+                packages[item[1]]['bugs'] = [item[0]]
+        for tag in bugs[item[0]]:
+            try:
+                packages[item[1]]['usertags'].append(tag)
+            except KeyError:
+                packages[item[1]]['usertags'] = [tag]
+    return packages
+
+def join_notes_bugs(notes, bugs):
+    for package in bugs:
+        for bug in bugs[package]['bugs']:
+            try:
+                if 'bugs' in notes[package]:
+                    if bug not in notes[package]['bugs']:
+                        warning("bug #" + str(bug) + " in package " +
+                                str(package) + " is not listed in notes.git.")
+                        notes[package]['bugs'].append(bug)
+                else:
+                    notes[package]['bugs'] = [bug]
+            except KeyError:
+                warning("bug #" + str(bug) + " in package " + str(package) +
+                        " is not listed in notes.git.")
+                notes[package] = {}
+                notes[package]['bugs'] = [bug]
+    return notes
+
 def cleanup_notes(notes, toremove):
     for pkg in toremove:
         del notes[pkg]
@@ -109,7 +206,7 @@ if __name__ == '__main__':
     toremove = find_old_notes(testedpkgs, notes)
     notes = cleanup_notes(notes, toremove)
     if args.db:
-        conn = start_db_connection()
+        conn = start_udd_connection()
         bugs = get_bugs()
         bugs = parse_bugs(bugs)
         notes = join_notes_bugs(notes, bugs)

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/misc.git