[SCM] pkg-kde-jenkins packaging branch, master, updated. d811dd13490c8b97d4a79be80e4cfa992874630d
Maximiliano Curia
maxy at moszumanska.debian.org
Mon Jun 6 22:10:47 UTC 2016
Gitweb-URL: http://git.debian.org/?p=pkg-kde/pkg-kde-jenkins.git;a=commitdiff;h=d811dd1
The following commit has been merged in the master branch:
commit d811dd13490c8b97d4a79be80e4cfa992874630d
Author: Maximiliano Curia <maxy at gnuservers.com.ar>
Date: Thu Jun 2 21:43:08 2016 +0200
Add a topological view, and abandon any idea about elegance
---
tools.py | 279 +++++++++++++++++++++++++++++++++++++++++++++++++++++----------
1 file changed, 238 insertions(+), 41 deletions(-)
diff --git a/tools.py b/tools.py
index bfed277..6e3f9ad 100755
--- a/tools.py
+++ b/tools.py
@@ -3,6 +3,7 @@
import argparse
import configparser
+import heapq
import logging
import subprocess
import sys
@@ -11,6 +12,9 @@ import debian.debian_support as debian_support
import jenkins
+Version = debian_support.Version
+
+
def process_options():
kw = {
@@ -18,13 +22,16 @@ def process_options():
}
arg_parser = argparse.ArgumentParser(
- description='List jenkins jobs unders certain criterias')
+ description='List jenkins jobs unders certain criterias',
+ fromfile_prefix_chars='@')
arg_parser.add_argument('-c', '--config', default='jenkins.ini')
arg_parser.add_argument('--debug', action='store_true')
arg_parser.add_argument('-D', '--distribution', default='unstable')
arg_parser.add_argument('--mode', default='todo',
- choices=['todo', 'fix', 'unreleased'])
+ choices=['todo', 'fix', 'unreleased', 'topsort',
+ 'trigger'])
arg_parser.add_argument('--trigger', action='store_true')
+ arg_parser.add_argument('--packages', nargs='+')
args = arg_parser.parse_args()
if args.debug:
@@ -111,10 +118,13 @@ def get_build_infos(server, job_info, distributions):
build_info = server.get_build_info(job_info['name'], build['number'])
parameters = get_parameters(build_info)
value = get_distribution_value(parameters)
+ # Probably "building"
+ if 'result' not in build_info or build_info['result'] is None:
+ continue
if value in distributions and value not in infos:
infos[value] = build_info
- if len(distributions) == infos:
+ if len(distributions) == len(infos):
break
return infos
@@ -125,8 +135,10 @@ def latest_build(server, job, distribution):
job_info = server.get_job_info(name)
result = {}
- latest_infos = get_build_infos(server, job_info,
- set(('unreleased', distribution)))
+ distributions = {'unreleased', distribution}
+ if distribution == 'experimental':
+ distributions.add('unstable')
+ latest_infos = get_build_infos(server, job_info, distributions)
for distribution, info in latest_infos.items():
result[distribution] = {}
result[distribution]['status'] = info.get('result', 'FAILURE')
@@ -159,19 +171,36 @@ def tests_unstable(package):
return _status('prepare') and _status('build') and _unstable('test')
-def check_deps(package, packages, distribution):
+def latest_package(package, packages, distributions):
+ max_version = Version('0')
+ at_distribution = {}
+ for d in distributions:
+ p = packages.get(package, {}).get(d, {})
+ version = Version(p.get('version', '0'))
+ if version > max_version:
+ at_distribution = p
+ max_version = version
+ return at_distribution
+
+
+def check_deps(package, packages, distributions):
for dep in package.get('deps', set()):
- at_distribution = packages.get(dep, {}).get(distribution, {})
+ at_distribution = latest_package(dep, packages, distributions)
if not (status(at_distribution) or tests_unstable(at_distribution)):
return False
return True
-def check_deps_version(package, packages, distribution):
+def check_deps_version(package, packages, distributions):
for dep in package.get('deps', set()):
dep_package = packages.get(dep, {})
- at_distribution = dep_package.get(distribution, {})
+ at_distribution = latest_package(dep, packages, distributions)
if 'version' not in at_distribution:
+ if 'debian_version' not in dep_package:
+ return False
+ at_unreleased = packages[dep].get('unreleased', {}).get('version', '0')
+ if dep_package['debian_version'] >= Version(at_unreleased):
+ continue
return False
new_version = debian_support.Version(at_distribution['version'])
@@ -197,20 +226,29 @@ def get_packages(server, distribution):
d[dist][part] = value
if 'source_name' in value and 'source_name' not in d:
d['source_name'] = value['source_name']
- if 'version' in value and 'version' not in d[dist]:
+ if part == 'prepare' and 'version' in value and 'version' not in d[dist]:
d[dist]['version'] = value['version']
return packages
-def version_at_distribution(source_name):
+def version_at_distribution(source_name, distributions):
+ # cmd = ['rmadison', '-u', 'udd', source_name]
cmd = ['rmadison', source_name]
output = subprocess.check_output(cmd, universal_newlines=True)
- version = None
+ version = '0'
for line in output.split('
'):
if '|' not in line:
continue
fields = line.split('|')
- new_version = debian_support.Version(fields[1].strip())
+ if 'source' not in fields[3]:
+ continue
+ d = fields[2].strip()
+ if 'new' == d:
+ continue
+ if 'experimental' not in distributions and \
+ 'experimental' in d:
+ continue
+ new_version = Version(fields[1].strip())
if not version:
version = new_version
elif version < new_version:
@@ -218,7 +256,7 @@ def version_at_distribution(source_name):
return version
-def get_ready(packages, distribution):
+def get_ready(packages, distributions):
ready = {}
# process packages and collect the packages for the second_phase
second_phase = set()
@@ -228,11 +266,12 @@ def get_ready(packages, distribution):
tests_unstable(package.get('unreleased', {}))):
continue
# print(package_name)
- if not check_deps(package, packages, distribution):
+ if not check_deps(package, packages, distributions):
continue
- version = version_at_distribution(package['source_name'])
- epochless = debian_support.Version(version)
+ version = version_at_distribution(package['source_name'],
+ distributions)
+ epochless = Version(version)
epochless.epoch = None
# print(version, epochless)
package['debian_version'] = epochless
@@ -246,13 +285,14 @@ def get_ready(packages, distribution):
if new_version <= package['debian_version']:
continue
- if not check_deps_version(package, packages, distribution):
+ if not check_deps_version(package, packages, distributions):
continue
- status_ok = status(package.get(distribution, {}))
- status_unstable = tests_unstable(package.get(distribution, {}))
+ p = latest_package(package_name, packages, distributions)
+ status_ok = status(p)
+ status_unstable = tests_unstable(p)
newer = (status_ok or status_unstable) and (debian_support.Version(
- package[distribution]['version']) > epochless)
+ p['version']) > epochless)
if status_ok and newer:
ready.setdefault('upload', set()).add(package_name)
@@ -273,31 +313,79 @@ def get_ready(packages, distribution):
return ready
+def topsort_tiers(packages, ready):
+ ready['all'] = ready.get('upload', set()) | \
+ ready.get('unstable', set()) | \
+ ready.get('build', set())
+ heap = []
+ # Prepare rdeps to complete the graph
+ for name in ready['all']:
+ package = packages[name]
+ ready_deps = package.get('deps', set()) & ready['all']
+ ready_deps -= {name}
+ for dep in ready_deps:
+ packages[dep].setdefault('rdeps', set()).add(name)
+ packages[name]['ready_deps'] = ready_deps
+ packages[name]['deps_required'] = len(ready_deps)
+ packages[name]['tier'] = 0
+ if packages[name]['deps_required'] == 0:
+ heapq.heappush(heap, (packages[name]['tier'], name))
+ tiers = []
+ sanity_check = 0
+ while heap:
+ tier, name = heapq.heappop(heap)
+ sanity_check += 1
+ package = packages[name]
+ if tier >= len(tiers):
+ tiers.append(set())
+ tiers[tier].add(name)
+ for rdep_name in package.get('rdeps', set()):
+ rdep = packages[rdep_name]
+ rdep['ready_deps'].remove(name)
+ rdep['deps_required'] -= 1
+ rdep['tier'] = package['tier'] + 1
+ if rdep['deps_required'] == 0:
+ heapq.heappush(heap, (rdep['tier'], rdep_name))
+ if len(ready['all']) != sanity_check:
+ print('ERROR: We processed {} of {}, something is '
+ 'wrong'.format(sanity_check, len(ready['all'])))
+ for name in ready['all']:
+ package = packages[name]
+ if package['ready_deps']:
+ print('name: {} missing {}'.format(
+ name, package['ready_deps']))
+ return tiers
+
+
def list_todo_distribution(server, packages, trigger, distribution):
# Obtain jobs
# for each job, obtain the builds
# check if the build parameter DISTRIBUTION matches the distribution, and
# keep the information of the newest matching job.
- ready = get_ready(packages, distribution)
-
- print('
###
# Upload
###')
- if 'upload' in ready:
- for package in ready['upload']:
- print(package)
- print('
###
# UNSTABLE
###')
- if 'unstable' in ready:
- for package in ready['unstable']:
- print(package)
- print('
###
# BUILD
###')
- if 'build' in ready:
- for package in ready['build']:
- print(package)
- if not trigger:
- return
- if 'build' in ready:
- for package in ready['build']:
- job_name = '{}_prepare'.format(package)
- server.build_job(job_name, {'DISTRIBUTION': distribution})
+ distributions = {distribution}
+ if distribution == 'experimental':
+ distributions.add('unstable')
+ ready = get_ready(packages, distributions)
+ tiers = topsort_tiers(packages, ready)
+
+ for tier, items in enumerate(tiers):
+ print('
###
# Tier {}
###'.format(tier + 1))
+ print('
###
# Upload
###')
+ if 'upload' in ready:
+ for package in ready['upload'] & items:
+ print(package)
+ print('
###
# UNSTABLE
###')
+ if 'unstable' in ready:
+ for package in ready['unstable'] & items:
+ print(package)
+ print('
###
# BUILD
###')
+ if 'build' in ready:
+ for package in ready['build'] & items:
+ print(package)
+ if tier == 0 and trigger and 'build' in ready:
+ for package in ready['build'] & items:
+ job_name = '{}_prepare'.format(package)
+ server.build_job(job_name, {'DISTRIBUTION': distribution})
def list_fix(packages):
@@ -334,12 +422,119 @@ def list_unreleased(packages):
_status(p, 'test')))
+def list_topsort(packages, distribution):
+
+ distributions = {distribution}
+ if distribution == 'experimental':
+ distributions.add('unstable')
+
+ heap = []
+ for name, package in packages.items():
+ deps = package.get('deps', set()) - {name}
+ deps &= packages.keys()
+ for dep in deps:
+ packages[dep].setdefault('rdeps', set()).add(name)
+ packages[name]['deps_required'] = deps
+ packages[name]['tier'] = 0
+ if len(deps) == 0:
+ heapq.heappush(heap, (0, name))
+ tiers = []
+ sanity_check = 0
+ while heap:
+ tier, name = heapq.heappop(heap)
+ package = packages[name]
+ sanity_check += 1
+ for i in range(len(tiers) - 1, tier + 1):
+ tiers.append(set())
+ tiers[tier].add(name)
+ for rdep_name in package.get('rdeps', set()):
+ rdep = packages[rdep_name]
+ rdep['deps_required'].remove(name)
+ rdep['tier'] = package['tier'] + 1
+ if len(rdep['deps_required']) == 0:
+ heapq.heappush(heap, (rdep['tier'], rdep_name))
+ if len(packages) != sanity_check:
+ print('ERROR: We processed {} of {}, something is '
+ 'wrong'.format(sanity_check, len(packages)))
+ for name, package in packages.items():
+ if package['deps_required']:
+ print('name: {} missing {}'.format(
+ name, package['deps_required']))
+
+ second_phase = set()
+ for name, package in packages.items():
+ package['status'] = ''
+ if not (status(package.get('unreleased', {})) or
+ tests_unstable(package.get('unreleased', {}))):
+ package['status'] = 'unreleased build'
+ continue
+ if not check_deps(package, packages, {'unreleased'}):
+ package['status'] = 'unreleased deps'
+ continue
+ version = version_at_distribution(package['source_name'], distributions)
+ epochless = debian_support.Version(version)
+ epochless.epoch = None
+ package['debian_version'] = epochless
+ new_version = debian_support.Version(package['unreleased']['version'])
+ if new_version < package['debian_version']:
+ package['status'] = 'unreleased old'
+ continue
+ elif new_version == package['debian_version']:
+ package['status'] = 'uploaded'
+ continue
+ second_phase.add(name)
+ for name in second_phase:
+ package = packages[name]
+ if not check_deps_version(package, packages, distributions):
+ package['status'] = 'missing dependency'
+ continue
+ p = latest_package(name, packages, distributions)
+ status_ok = status(p)
+ status_unstable = tests_unstable(p)
+ newer = (status_ok or status_unstable) and (debian_support.Version(
+ p['version']) > package['debian_version'])
+ if status_ok and newer:
+ package['status'] = 'upload'
+ elif status_unstable and newer:
+ package['status'] = 'unstable'
+ else:
+ package['status'] = 'build'
+
+ for tier, items in enumerate(tiers):
+ print('
###
# Tier {}
###'.format(tier + 1))
+ xs = sorted(items)
+ for name in xs:
+ package = packages[name]
+ versions = {}
+ if 'debian_version' in package:
+ versions['debian'] = package['debian_version']
+ for d in distributions | {'unreleased'}:
+ if d not in package:
+ continue
+ if 'version' in package[d]:
+ versions[d] = package[d]['version']
+
+ print('{name}: {status} {versions}'.format(name=name,
+ status=package['status'],
+ versions=versions))
+
+
+def trigger(server, options):
+ for package in options.packages:
+ job_name = '{}_prepare'.format(package)
+ server.build_job(job_name, {'DISTRIBUTION': options.distribution})
+
+
def main():
options = process_options()
server = connect(options)
if options.mode == 'unreleased':
options.distribution = 'unreleased'
+ elif options.mode == 'trigger':
+ trigger(server, options)
+ sys.exit(0)
+
packages = get_packages(server, options.distribution)
if options.mode == 'todo':
@@ -349,6 +544,8 @@ def main():
list_fix(packages)
elif options.mode == 'unreleased':
list_unreleased(packages)
+ elif options.mode == 'topsort':
+ list_topsort(packages, options.distribution)
# test(server)
sys.exit(0)
--
pkg-kde-jenkins packaging
More information about the pkg-kde-commits
mailing list