[Collab-qa-commits] r2378 - udd/udd
Andreas Tille
tille at alioth.debian.org
Fri Jun 15 09:10:57 UTC 2012
Author: tille
Date: 2012-06-15 09:10:57 +0000 (Fri, 15 Jun 2012)
New Revision: 2378
Modified:
udd/udd/blends_prospective_gatherer.py
Log:
There is no point in leaving out the tests applied to not yet uploaded packages but leave them out for packages inside Debian. So write potential problems into logfile also for official Debian packages and do not stop parsing once a package is in UDD.
Modified: udd/udd/blends_prospective_gatherer.py
===================================================================
--- udd/udd/blends_prospective_gatherer.py 2012-06-15 03:01:04 UTC (rev 2377)
+++ udd/udd/blends_prospective_gatherer.py 2012-06-15 09:10:57 UTC (rev 2378)
@@ -105,24 +105,9 @@
sources.append(re.sub("\.changelog", "", file))
for source in sources:
cur.execute("EXECUTE check_source (%s)", (source,))
+ prospective = True
if cur.fetchone()[0] > 0:
- # print "Source %s is in DB. Ignore for prospective packages" % source
- ufile=upath+'/'+source+'.upstream'
- if not exists(ufile):
- continue
- cur.execute("EXECUTE check_reference (%s)", (source,))
- if cur.fetchone()[0] > 0:
- # UDD seems to contain the references specified in source.upstream file
- continue
- upstream = upstream_reader(ufile, source, self.log)
- if not upstream.references:
- # There are no valid references found in this upstream file or it is no valid YAML
- continue
- self.log.warning("%s has upstream file but no references in UDD" % (source, ))
- upstream.parse()
- for ref in upstream.get_bibrefs():
- bibrefs.append(ref)
- continue
+ prospective = False
sprosp = {}
# Read Vcs fields
@@ -185,70 +170,71 @@
sprosp['wnpp_type'] = ''
sprosp['wnpp_desc'] = ''
# if match: # try to make sure we are really dealing with ITPs
- for iwnpp in sprosp['closes']:
- if iwnpp == 12345: # that seems to be a fake ITP
- self.log.debug("Fake WNPP no. 12345 in changelog of '%s' of %s" % (source, sprosp['blend']))
- continue
- elif iwnpp > 0:
- sprosp['wnpp'] = iwnpp
- cur.execute("EXECUTE check_itp (%s)", (iwnpp,))
- if cur.rowcount > 0:
- wnppbug = RowDictionaries(cur)[0]
- itpmatch = parse_itp_re.search(wnppbug['title'])
- if itpmatch:
- sprosp['wnpp_type'] = itpmatch.groups()[0]
- sprosp['wnpp_desc'] = itpmatch.groups()[2]
- if source != itpmatch.groups()[1]:
- self.log.info("Source name of '%s' of %s differs from name in %s bug: package name = %s, short description = %s" % (source, sprosp['blend'], itpmatch.groups()[0], itpmatch.groups()[1], itpmatch.groups()[2]))
+ if prospective:
+ for iwnpp in sprosp['closes']:
+ if iwnpp == 12345: # that seems to be a fake ITP
+ self.log.debug("Fake WNPP no. 12345 in changelog of '%s' of %s" % (source, sprosp['blend']))
+ continue
+ elif iwnpp > 0:
+ sprosp['wnpp'] = iwnpp
+ cur.execute("EXECUTE check_itp (%s)", (iwnpp,))
+ if cur.rowcount > 0:
+ wnppbug = RowDictionaries(cur)[0]
+ itpmatch = parse_itp_re.search(wnppbug['title'])
+ if itpmatch:
+ sprosp['wnpp_type'] = itpmatch.groups()[0]
+ sprosp['wnpp_desc'] = itpmatch.groups()[2]
+ if source != itpmatch.groups()[1]:
+ self.log.info("Source name of '%s' of %s differs from name in %s bug: package name = %s, short description = %s" % (source, sprosp['blend'], itpmatch.groups()[0], itpmatch.groups()[1], itpmatch.groups()[2]))
+ else:
+ self.log.warning("Cannot parse ITP bug %i for package '%s' of %s: `%s`" % (iwnpp, source, sprosp['blend'], wnppbug['title']))
else:
- self.log.warning("Cannot parse ITP bug %i for package '%s' of %s: `%s`" % (iwnpp, source, sprosp['blend'], wnppbug['title']))
- else:
- self.log.debug("ITP bug %i for package '%s' of %s is not open any more or can otherwise not be found" % (iwnpp, source, sprosp['blend']))
- continue # Try other bug number if exists
- break
- match = find_itp_re.search(changes)
- if not match and sprosp['wnpp_type'] == '' and len(sprosp['closes']) > 0 and sprosp['wnpp'] > 0:
- sprosp['wnpp'] = 0
- self.log.warning("Bug %s closed in changelog of package '%s' of %s does not seem to be an ITP" % (str(sprosp['closes']), source, sprosp['blend']))
-
- # Read Copyright file if specifying Format in the first line
- cprfile = upath+'/'+source+'.copyright'
- try:
- cpr = open(cprfile,'r')
- except:
- self.log.debug("Unable to open Copyright file for source '%s' of %s (%s)" % (source, sprosp['blend'], cprfile))
- cpr = None
- linenr = 0
- found_files = False
- sprosp['license'] = ''
- if cpr:
- for line in cpr.readlines():
- line = line.strip()
- if line == '':
- if found_files:
- found_files = False
- break # We might leave the 'Files: *' paragraph again
- continue
- try:
- (field,value) = line.split(': ')
- except ValueError:
- # either no DEP5 file or no line we want to read here
- continue
- if linenr == 0:
- if field != 'Format':
- self.log.debug("Copyright file for source '%s' of %s does not seem to regard DEP5. Found line `%s`" % (source, sprosp['blend'], line.strip()))
- found_files = True # one flag is enough to control this - we do not need another warning in the logs
+ self.log.debug("ITP bug %i for package '%s' of %s is not open any more or can otherwise not be found" % (iwnpp, source, sprosp['blend']))
+ continue # Try other bug number if exists
+ break
+ match = find_itp_re.search(changes)
+ if not match and sprosp['wnpp_type'] == '' and len(sprosp['closes']) > 0 and sprosp['wnpp'] > 0:
+ sprosp['wnpp'] = 0
+ self.log.warning("Bug %s closed in changelog of package '%s' of %s does not seem to be an ITP" % (str(sprosp['closes']), source, sprosp['blend']))
+
+ # Read Copyright file if specifying Format in the first line
+ cprfile = upath+'/'+source+'.copyright'
+ try:
+ cpr = open(cprfile,'r')
+ except:
+ self.log.debug("Unable to open Copyright file for source '%s' of %s (%s)" % (source, sprosp['blend'], cprfile))
+ cpr = None
+ linenr = 0
+ found_files = False
+ sprosp['license'] = ''
+ if cpr:
+ for line in cpr.readlines():
+ line = line.strip()
+ if line == '':
+ if found_files:
+ found_files = False
+ break # We might leave the 'Files: *' paragraph again
+ continue
+ try:
+ (field,value) = line.split(': ')
+ except ValueError:
+ # either no DEP5 file or no line we want to read here
+ continue
+ if linenr == 0:
+ if field != 'Format':
+ self.log.debug("Copyright file for source '%s' of %s does not seem to regard DEP5. Found line `%s`" % (source, sprosp['blend'], line.strip()))
+ found_files = True # one flag is enough to control this - we do not need another warning in the logs
+ break
+ linenr += 1
+ field = field.strip()
+ value = value.strip()
+ if field == 'Files' and value == '*':
+ found_files = True
+ if field == 'License' and found_files:
+ sprosp['license'] = value
break
- linenr += 1
- field = field.strip()
- value = value.strip()
- if field == 'Files' and value == '*':
- found_files = True
- if field == 'License' and found_files:
- sprosp['license'] = value
- break
- if not found_files:
- self.log.debug("No 'Files: *' specification found in copyright file for source '%s' of %s" % (source, sprosp['blend']))
+ if not found_files:
+ self.log.debug("No 'Files: *' specification found in copyright file for source '%s' of %s" % (source, sprosp['blend']))
# Try to read debian/control
ctrl = None
@@ -257,74 +243,74 @@
ctrl = open(ctrlfile,'r')
except:
self.log.warning("Unable to open control file for source '%s' of %s (%s)" % (source, sprosp['blend'], ctrlfile))
- # FIXME: This part is deactivated via 1==0 due to the fact that iter_paragraphs does not seem to work for debian/control files
if ctrl:
- ictrl = deb822.Deb822.iter_paragraphs(ctrl)
- src = ictrl.next()
- # print 'SOURCE:', src # print Source stanza
- if src.has_key('source'):
- if source != src['source']:
- self.log.error("Something is wrong with control data of package '%s' of %s. Changelog says source = '%s'." % (source, sprosp['blend'], src['Source']))
- else:
- self.log.warning("Control file for source '%s' of %s is lacking source field" % (source, sprosp['blend']))
- if src.has_key('vcs-browser'):
- if sprosp['vcs_browser'] != src['vcs-browser']:
- tmp_prosp = re.sub('/$', '', sprosp['vcs_browser']) # ignore forgotten '/' at end of Vcs-Browser
- tmp_src = re.sub('/$', '', src['vcs-browser']) # same with string in debian/control to enable comparison after further changes below
- tmp_src = re.sub('\.git;a=summary$', '.git', tmp_src)
- tmp_src = re.sub('/viewsvn/', '/wsvn/', tmp_src) # machine-readable gatherer implies /wsvn/ but specifying /viewsvn/ does no harm
- tmp_src = re.sub('/anonscm.debian.org/gitweb/\?p=', '/git.debian.org/?p=', tmp_src)
- tmp_src = re.sub('/anonscm.debian.org/git/([^?])', '/git.debian.org/?p=\\1', tmp_src) # Add missing '?p='
- tmp_src = re.sub('/anonscm\.debian\.org/viewvc', '/svn.debian.org/wsvn', tmp_src) # FIXME: is it correct to assume SVN here??? - probably not
- tmp_src = re.sub('\.debian\.org/viewvc', '.debian.org/wsvn', tmp_src) # somehow there seem some viewvc calls to remain
- tmp_src = re.sub('/\?op=log', '', tmp_src) # Some SVN URLs specify this parameter which should not be regarded here
- if tmp_prosp != tmp_src:
- tmp_src = re.sub('^git:', 'http:', tmp_src) # check for usual error in specifying Vcs-Browser by just leaving 'git:' as protocol
- if tmp_src == sprosp['vcs_browser']:
- self.log.error("%s of %s - Wrong Vcs-Browser: Use 'http:' instead of 'git:' in '%s' ('%s')." % (source, sprosp['blend'], src['Vcs-Browser'], sprosp['vcs_browser']))
- else:
- tmp_prosp = re.sub('/trunk/?$', '', tmp_prosp) # sometimes the trailing trunk/ is forgotten which is no real problem
- tmp_src = re.sub('/trunk/?$', '', tmp_src) # also in tmp_src there is sometimes a remaining /trunk
- if tmp_prosp != tmp_src:
- self.log.warning("%s of %s - Differing Vcs-Browser: Obtained from Vcs-Browser='%s' <-> control has '%s'." % (source, sprosp['blend'], sprosp['vcs_browser'], src['Vcs-Browser']))
- else:
- self.log.debug("Control file for source '%s' of %s is lacking Vcs-Browser field" % (source, sprosp['blend']))
+ ictrl = deb822.Deb822.iter_paragraphs(ctrl)
+ src = ictrl.next()
+ # print 'SOURCE:', src # print Source stanza
+ if src.has_key('source'):
+ if source != src['source']:
+ self.log.error("Something is wrong with control data of package '%s' of %s. Changelog says source = '%s'." % (source, sprosp['blend'], src['Source']))
+ else:
+ self.log.warning("Control file for source '%s' of %s is lacking source field" % (source, sprosp['blend']))
+ if src.has_key('vcs-browser'):
+ if sprosp['vcs_browser'] != src['vcs-browser']:
+ tmp_prosp = re.sub('/$', '', sprosp['vcs_browser']) # ignore forgotten '/' at end of Vcs-Browser
+ tmp_src = re.sub('/$', '', src['vcs-browser']) # same with string in debian/control to enable comparison after further changes below
+ tmp_src = re.sub('\.git;a=summary$', '.git', tmp_src)
+ tmp_src = re.sub('/viewsvn/', '/wsvn/', tmp_src) # machine-readable gatherer implies /wsvn/ but specifying /viewsvn/ does no harm
+ tmp_src = re.sub('/anonscm.debian.org/gitweb/\?p=', '/git.debian.org/?p=', tmp_src)
+ tmp_src = re.sub('/anonscm.debian.org/git/([^?])', '/git.debian.org/?p=\\1', tmp_src) # Add missing '?p='
+ tmp_src = re.sub('/anonscm\.debian\.org/viewvc', '/svn.debian.org/wsvn', tmp_src) # FIXME: is it correct to assume SVN here??? - probably not
+ tmp_src = re.sub('\.debian\.org/viewvc', '.debian.org/wsvn', tmp_src) # somehow there seem some viewvc calls to remain
+ tmp_src = re.sub('/\?op=log', '', tmp_src) # Some SVN URLs specify this parameter which should not be regarded here
+ if tmp_prosp != tmp_src:
+ tmp_src = re.sub('^git:', 'http:', tmp_src) # check for usual error in specifying Vcs-Browser by just leaving 'git:' as protocol
+ if tmp_src == sprosp['vcs_browser']:
+ self.log.error("%s of %s - Wrong Vcs-Browser: Use 'http:' instead of 'git:' in '%s' ('%s')." % (source, sprosp['blend'], src['Vcs-Browser'], sprosp['vcs_browser']))
+ else:
+ tmp_prosp = re.sub('/trunk/?$', '', tmp_prosp) # sometimes the trailing trunk/ is forgotten which is no real problem
+ tmp_src = re.sub('/trunk/?$', '', tmp_src) # also in tmp_src there is sometimes a remaining /trunk
+ if tmp_prosp != tmp_src:
+ self.log.warning("%s of %s - Differing Vcs-Browser: Obtained from Vcs-Browser='%s' <-> control has '%s'." % (source, sprosp['blend'], sprosp['vcs_browser'], src['Vcs-Browser']))
+ else:
+ self.log.debug("Control file for source '%s' of %s is lacking Vcs-Browser field" % (source, sprosp['blend']))
- if src.has_key('Maintainer'):
- sprosp['maintainer'] = src['maintainer']
- (name, email) = parse_email(src['maintainer'])
- sprosp['maintainer_name'] = name
- sprosp['maintainer_email'] = email
- else:
- self.log.info("Control file for source '%s' of %s is lacking Maintainer field" % (source, sprosp['blend']))
+ if src.has_key('Maintainer'):
+ sprosp['maintainer'] = src['maintainer']
+ (name, email) = parse_email(src['maintainer'])
+ sprosp['maintainer_name'] = name
+ sprosp['maintainer_email'] = email
+ else:
+ self.log.info("Control file for source '%s' of %s is lacking Maintainer field" % (source, sprosp['blend']))
- for prop in ('homepage', 'priority', 'section', 'uploaders', ):
- if src.has_key(prop):
- if prop == 'section':
- if src['section'].startswith('non-free'):
- sprosp['component'] = 'non-free'
- (dummy,sprosp['section']) = src['section'].split('/')
- if sprosp['license'] == '':
- sprosp['license'] = 'non-free'
- elif src['section'].startswith('contrib'):
- sprosp['component'] = 'contrib'
- (dummy,sprosp['section']) = src['section'].split('/')
- if sprosp['license'] == '':
- sprosp['license'] = 'free'
- else:
- sprosp['component'] = 'main'
- sprosp['section'] = src['section']
- if sprosp['license'] == '':
- sprosp['license'] = 'free'
+ for prop in ('homepage', 'priority', 'section', 'uploaders', ):
+ if src.has_key(prop):
+ if prop == 'section':
+ if src['section'].startswith('non-free'):
+ sprosp['component'] = 'non-free'
+ (dummy,sprosp['section']) = src['section'].split('/')
+ if sprosp['license'] == '':
+ sprosp['license'] = 'non-free'
+ elif src['section'].startswith('contrib'):
+ sprosp['component'] = 'contrib'
+ (dummy,sprosp['section']) = src['section'].split('/')
+ if sprosp['license'] == '':
+ sprosp['license'] = 'free'
else:
- sprosp[prop] = src[prop]
+ sprosp['component'] = 'main'
+ sprosp['section'] = src['section']
+ if sprosp['license'] == '':
+ sprosp['license'] = 'free'
else:
- sprosp[prop] = ''
- if prop != 'uploaders':
- self.log.warning("Control file for source '%s' of %s is lacking %s field" % (source, sprosp['blend'], prop))
- else:
- self.log.debug("Control file for source '%s' of %s is lacking %s field" % (source, sprosp['blend'], prop))
+ sprosp[prop] = src[prop]
+ else:
+ sprosp[prop] = ''
+ if prop != 'uploaders':
+ self.log.warning("Control file for source '%s' of %s is lacking %s field" % (source, sprosp['blend'], prop))
+ else:
+ self.log.debug("Control file for source '%s' of %s is lacking %s field" % (source, sprosp['blend'], prop))
+ if prospective:
pkg = ictrl.next()
valid_pkg_info = True
while pkg:
@@ -333,22 +319,22 @@
pprosp[sprop] = sprosp[sprop]
if pkg.has_key('package'):
- pprosp['package'] = pkg['package']
+ pprosp['package'] = pkg['package']
else:
- self.log.warning("Control file for source '%s' od %s is lacking Package field" % (source, sprosp['blend']))
+ self.log.warning("Control file for source '%s' od %s is lacking Package field" % (source, sprosp['blend']))
if pkg.has_key('description'):
- if len(pkg['description'].split("\n",1)) > 1:
- pprosp['long_description'] = pkg['description'].split("\n",1)[1]
- else:
- pprosp['long_description'] = ''
- pprosp['description'] = pkg['description'].split("\n",1)[0].strip()
+ if len(pkg['description'].split("\n",1)) > 1:
+ pprosp['long_description'] = pkg['description'].split("\n",1)[1]
+ else:
+ pprosp['long_description'] = ''
+ pprosp['description'] = pkg['description'].split("\n",1)[0].strip()
else:
- if pprosp.has_key('package'):
- self.log.warning("Control file for source '%s' of %s has no desription for Package %s" % (source, sprosp['blend'], pprosp['package']))
- else:
- # self.log.error("Control file for source '%s' of %s seems to miss package information" % (source, sprosp['blend']))
- self.log.info("Control file for source '%s' of %s seems to contain some comments which can not be parsed/ignored with this python-debian version" % (source, sprosp['blend']))
- valid_pkg_info = False
+ if pprosp.has_key('package'):
+ self.log.warning("Control file for source '%s' of %s has no desription for Package %s" % (source, sprosp['blend'], pprosp['package']))
+ else:
+ # self.log.error("Control file for source '%s' of %s seems to miss package information" % (source, sprosp['blend']))
+ self.log.info("Control file for source '%s' of %s seems to contain some comments which can not be parsed/ignored with this python-debian version" % (source, sprosp['blend']))
+ valid_pkg_info = False
if valid_pkg_info:
pkgs.append(pprosp)
try:
@@ -356,7 +342,7 @@
valid_pkg_info = True
except:
break
- # Try to read debian/control
+
upstream = None
ufile = upath+'/'+source+'.upstream'
if exists(ufile):
@@ -364,6 +350,9 @@
if cur.fetchone()[0] == 0:
upstream = upstream_reader(ufile, source, self.log)
if upstream.references:
+ if not prospective:
+ # Valid references found in this upstream file and it is valid YAML even if it was not fetched by bibref gatherer
+ self.log.warning("%s of %s has upstream file but no references in UDD" % (source, sprosp['blend']))
upstream.parse()
for ref in upstream.get_bibrefs():
bibrefs.append(ref)
More information about the Collab-qa-commits
mailing list