[Reproducible-commits] [misc] 05/06: move database connection into a module
Chris West
faux-guest at moszumanska.debian.org
Mon Dec 14 12:06:58 UTC 2015
This is an automated email from the git hooks/post-receive script.
faux-guest pushed a commit to branch master
in repository misc.
commit 3498c3071dea9c86815208f34e11a0c52189d30c
Author: Chris West (Faux) <git at goeswhere.com>
Date: Mon Dec 14 11:35:00 2015 +0000
move database connection into a module
---
clean-notes | 74 +++++++++----------------------------------------------------
remote.py | 58 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 68 insertions(+), 64 deletions(-)
diff --git a/clean-notes b/clean-notes
index 07db71c..bb6bf1a 100755
--- a/clean-notes
+++ b/clean-notes
@@ -12,12 +12,11 @@ from apt_pkg import version_compare
import apt
import argparse
-import psycopg2
import yaml
import yamlfiles
import logger
-from remote import load_reproducible_status, RB_SITE
+import remote
# "apt" is to avoid adding an ugly "apt_pkg.init()" call to fix a "ValueError:
# _system not initialized" error.
@@ -143,59 +142,6 @@ if __name__ == '__main__':
yaml.composer.Composer.compose_mapping_node = compose_mapping_node
-def start_udd_connection():
- username = "public-udd-mirror"
- password = "public-udd-mirror"
- host = "public-udd-mirror.xvm.mit.edu"
- port = 5432
- db = "udd"
- try:
- try:
- log.debug("Starting connection to the UDD database")
- conn = psycopg2.connect(
- database=db,
- user=username,
- host=host,
- port=port,
- password=password,
- connect_timeout=5,
- )
- except psycopg2.OperationalError as err:
- if str(err) == 'timeout expired\n':
- log.error('Connection to the UDD database replice timed out. '
- 'Maybe the machine is offline or just unavailable.')
- log.error('Failing nicely anyway, all queries will return an '
- 'empty response.')
- return None
- else:
- raise
- except:
- log.exception('Erorr connecting to the UDD database replica.' +
- 'The full error is:')
- log.error('Failing nicely anyway, all queries will return an empty ' +
- 'response.')
- return None
- conn.set_client_encoding('utf8')
- return conn
-
-
-def query_udd(query):
- if not conn_udd:
- log.error('There has been an error connecting to the UDD database. ' +
- 'Please look for a previous error for more information.')
- log.error('Failing nicely anyway, returning an empty response.')
- return []
- cursor = conn_udd.cursor()
- try:
- cursor.execute(query)
- except:
- log.exception('The UDD server encountered a issue while executing the'
- ' query. The full error is:')
- log.error('Failing nicely anyway, returning an empty response.')
- return []
- return cursor.fetchall()
-
-
def check_notes_validity(notes, testedpkgs):
for pkg in sorted(notes):
if pkg not in testedpkgs:
@@ -224,7 +170,7 @@ def check_bugs(notes):
for bug in bugs_list[:-1]:
ids += '%s OR id=' % bug
ids += str(bugs_list[-1])
- rows = query_udd("""SELECT id FROM bugs_usertags WHERE
+ rows = remote.query_udd(conn_udd, """SELECT id FROM bugs_usertags WHERE
email='reproducible-builds at lists.alioth.debian.org' AND (
%s )""" % ids)
for bug in bugs_list:
@@ -258,7 +204,7 @@ def find_old_notes(testedpkgs, notes):
'timestamps_in_png' not in notes[pkg]['issues']) and \
'fixed-magically' not in args.disable:
log.info("%s/%s has a note for the version %s but that version is "
- "reproducible", RB_SITE, pkg, item['version'])
+ "reproducible", remote.RB_SITE, pkg, item['version'])
if item['status'] == 'reproducible' and \
notes[pkg].get('version') and \
'now-fixed' not in args.disable and \
@@ -280,7 +226,7 @@ def find_old_notes(testedpkgs, notes):
def get_bugs():
query = 'SELECT * FROM bugs_usertags ' + \
"WHERE email='reproducible-builds at lists.alioth.debian.org'"
- rows = query_udd(query)
+ rows = remote.query_udd(conn_udd, query)
# returns a list of tuples (email, tag, id)
bugs = {}
for tag in rows:
@@ -295,14 +241,14 @@ def detect_archived_bugs(notes):
query = 'SELECT u.id ' \
'FROM bugs_usertags AS u JOIN archived_bugs AS a ON u.id=a.id ' \
"WHERE u.email='reproducible-builds at lists.alioth.debian.org'"
- rows = [x[0] for x in query_udd(query)]
+ rows = [x[0] for x in remote.query_udd(conn_udd, query)]
for pkg in sorted(notes.keys()):
try:
for bug in notes[pkg]['bugs']:
if bug in rows:
log.warning(
'https://bugs.debian.org/%s in %s/%s is archived',
- bug, RB_SITE, pkg)
+ bug, remote.RB_SITE, pkg)
notes[pkg]['bugs'].remove(bug)
except KeyError:
pass
@@ -336,7 +282,7 @@ def parse_bugs(bugs):
query += "FROM bugs AS b JOIN sources AS s ON b.source=s.source "
query += "WHERE %s""" % ids
log.debug(query)
- rows = query_udd(query)
+ rows = remote.query_udd(conn_udd, query)
log.info('%d bugs found', len(rows))
for item in rows:
if item[2]: # do not consider closed bugs
@@ -396,10 +342,10 @@ if __name__ == '__main__':
if 'not-usertagged' not in args.disable or \
'missing-usertagged' not in args.disable or \
'archived-bugs' not in args.disable:
- conn_udd = start_udd_connection()
+ conn_udd = remote.start_udd_connection()
notes = yamlfiles.load_notes()
if not args.sort_only:
- testedpkgs = load_reproducible_status()
+ testedpkgs = remote.load_reproducible_status()
check_notes_validity(notes, testedpkgs)
toremove = find_old_notes(testedpkgs, notes)
else:
@@ -417,4 +363,4 @@ if __name__ == '__main__':
yamlfiles.write_out(notes)
else:
log.info("Don't write out the %s file, as requested (dry-run).",
- NOTES_YAML)
+ yamlfiles.NOTES_YAML)
diff --git a/remote.py b/remote.py
index 2d64d5a..8f90d32 100644
--- a/remote.py
+++ b/remote.py
@@ -3,12 +3,17 @@
import json
+import psycopg2
import requests
+import logger
+
RB_SITE = 'https://reproducible.debian.net'
REPRODUCIBLE_JSON = '{}/reproducible.json'.format(RB_SITE)
# [{package: xxx, suite: sid, version: 0.0.0, status: reproducible}, {...}]
+log = logger.setup_logging(__name__)
+
def load_reproducible_status():
status = {}
@@ -44,3 +49,56 @@ def load_reproducible_status():
return status
+
+def start_udd_connection():
+ username = "public-udd-mirror"
+ password = "public-udd-mirror"
+ host = "public-udd-mirror.xvm.mit.edu"
+ port = 5432
+ db = "udd"
+ try:
+ try:
+ log.debug("Starting connection to the UDD database")
+ conn = psycopg2.connect(
+ database=db,
+ user=username,
+ host=host,
+ port=port,
+ password=password,
+ connect_timeout=5,
+ )
+ except psycopg2.OperationalError as err:
+ if str(err) == 'timeout expired\n':
+ log.error('Connection to the UDD database replice timed out. '
+ 'Maybe the machine is offline or just unavailable.')
+ log.error('Failing nicely anyway, all queries will return an '
+ 'empty response.')
+ return None
+ else:
+ raise
+ except:
+ log.exception('Erorr connecting to the UDD database replica.' +
+ 'The full error is:')
+ log.error('Failing nicely anyway, all queries will return an empty ' +
+ 'response.')
+ return None
+ conn.set_client_encoding('utf8')
+ return conn
+
+
+def query_udd(conn_udd, query):
+ if not conn_udd:
+ log.error('There has been an error connecting to the UDD database. ' +
+ 'Please look for a previous error for more information.')
+ log.error('Failing nicely anyway, returning an empty response.')
+ return []
+ cursor = conn_udd.cursor()
+ try:
+ cursor.execute(query)
+ except:
+ log.exception('The UDD server encountered a issue while executing the'
+ ' query. The full error is:')
+ log.error('Failing nicely anyway, returning an empty response.')
+ return []
+ return cursor.fetchall()
+
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/reproducible/misc.git
More information about the Reproducible-commits
mailing list