[SCM] ci-tooling packaging branch, master, updated. e9a9e375222ea0f5990ad527a18704d0a538d84f
Harald Sitter
apachelogger-guest at moszumanska.debian.org
Fri Feb 6 14:41:24 UTC 2015
Gitweb-URL: http://git.debian.org/?p=pkg-kde/ci-tooling.git;a=commitdiff;h=e9a9e37
The following commit has been merged in the master branch:
commit e9a9e375222ea0f5990ad527a18704d0a538d84f
Author: Harald Sitter <sitter at kde.org>
Date: Fri Feb 6 15:34:34 2015 +0100
throw contained folder away, ci-tooling by default is assumed to be
---
contained/daily-promote.rb | 277 ---------------------------------------------
contained/lib | 1 -
contained/ppa/promoter.py | 94 ---------------
contained/ppa/sync.py | 218 -----------------------------------
4 files changed, 590 deletions(-)
diff --git a/contained/daily-promote.rb b/contained/daily-promote.rb
deleted file mode 100755
index 0ed08be..0000000
--- a/contained/daily-promote.rb
+++ /dev/null
@@ -1,277 +0,0 @@
-#!/usr/bin/env ruby
-
-ENV['PATH'] = '/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin'
-Dir.chdir('/tmp/') # Containers by default have a bugged out pwd, force /tmp.
-
-if Process.uid == '0'
- exit 1 unless system("apt-get update")
- exit 1 unless system("apt-get install -y ruby ruby-dev")
- exit 1 unless system("gem install jenkins_api_client oauth")
-end
-
-require 'thwait'
-require_relative 'lib/jenkins'
-require_relative 'lib/lp'
-
-Project = Struct.new(:series, :stability)
-# get basename, distro series, unstable/stable
-raise 'Did not get valid argumenst' unless ARGV.size == 2
-project = Project.new(ARGV[0], ARGV[1])
-
-jenkins = new_jenkins
-job_names = jenkins.job.list("^#{project.series}_#{project.stability}_.*")
-
-job_name_queue = Queue.new
-job_names.each do |name|
- job_name_queue << name
-end
-
-# Check status of the jobs through a very simply threaded queue.
-# This allows $threadcount concurrent connections which is heaps
-# faster than doing this sequentially. In particular when run
-# outside localhost.
-threads = []
-# 16.times do
-# threads << Thread.new do
-# jenkins = new_jenkins
-# while name = job_name_queue.pop(true) do
-# begin
-# if jenkins.job.status(name) != 'success' \
-# or jenkins.job.status(name) != 'unstable' \
-# or jenkins.queue.list.include?(name)
-# puts "E: #{name} does not qualify for snapshot. Aborting."
-# exit 1
-# end
-# rescue
-# retry
-# end
-# end
-# end
-# end
-
-# ThreadsWait.all_waits(threads)
-
-puts "I: All Jenkins jobs seem jolly alright"
-
-class CiPPA
- attr_reader :type
- attr_reader :series
-
- def initialize(type, series)
- @type = type
- @series = series
- @added = false
- @packages = {}
- end
-
- def add
- return if @added
- `echo "force-unsafe-io" > /etc/dpkg/dpkg.cfg.d/02apt-speedup`
- system('apt-get install -y --no-install-recommends software-properties-common eatmydata')
- system("apt-add-repository -y ppa:kubuntu-ci/#{@type}")
- system('apt-get update')
- @added = true
- end
-
- def remove
- # Always remove even if !@added to make sure it is really gone.
- system('apt-get install -y --no-install-recommends software-properties-common eatmydata')
- system("apt-add-repository -y -r ppa:kubuntu-ci/#{@type}")
- system('apt-get update')
- @added = false
- end
-
- def packages
- return @packages unless @packages.empty?
-
-# # TODO: this presently does not take multiarch into account.
-# Dir.glob("/var/lib/apt/lists/ppa.launchpad.net_kubuntu-ci_#{@type}_*_Packages") do |cache_file|
-# file = File.open(cache_file, 'r')
-# while !file.eof?
-# line = file.readline
-# match = line.match(/^Package: (.*)$/)
-# next unless match and match.size == 2 # fullmatch + name match
-# @packages << match[1]
-# end
-# end
-# @packages.uniq!
-# @packages.delete_if { |p| p.end_with?('-dbg') || p.end_with?('-data') }
-# @packages.sort!
-
- ubuntu = Launchpad::Rubber::from_json(Net::HTTP.get(URI('https://api.launchpad.net/1.0/ubuntu')))
- series = nil
- ubuntu.series.each do |_series|
- next unless _series.name == @series
- series = _series
- break
- end
-
- reference_distro_arch_series_link = nil
- series.architectures.each do |_arch|
- # FIXME: arch harcode
- next unless _arch.architecture_tag == 'amd64'
- reference_distro_arch_series_link = _arch.self_link
- break
- end
-
- ppas = Launchpad::Rubber::from_json(Net::HTTP.get(URI('https://api.launchpad.net/1.0/~kubuntu-ci/ppas'))).entries
- ppa = nil
- ppas.each do |_ppa|
- next unless _ppa.name == @type
- ppa = _ppa
- break
- end
-
- packages = {}
- # ppa.getPublishedBinaries(:ordered => false,
- # :status => 'Published',
- # :distro_arch_series => reference_distro_arch_series_link).each do |binary|
- # packages[binary.binary_package_name] = binary.binary_package_version
- # end
- ppa.getPublishedSources(:status => 'Published',
- :distro_series => series.self_link).each do |source|
- binaries = source.getPublishedBinaries()
- binaries.each do |binary|
-# puts "archspecific: #{binary.architecture_specific} . #{binary.binary_package_name} . #{binary.binary_package_version}"
- next if binary.binary_package_name.end_with?('-dbg')
- if binary.architecture_specific
- next unless binary.distro_arch_series_link == reference_distro_arch_series_link
- end
- packages[binary.binary_package_name] = binary.binary_package_version
- end
- end
-
- return @packages = packages
- end
-
- def install
- p packages.map{|k,v| "#{k}='#{v}'"}.join(' ')
- pin!
- return system("apt-get install --force-yes -y -o 'Debug::pkgProblemResolver=true' #{packages.map{|k,v| "#{k}='#{v}'"}.join(' ')}")
- end
-
- def purge
- return system("apt-get purge --force-yes -y #{packages.keys().join(' ')}")
- end
-
-private
- def pin!
- File.open('/etc/apt/preferences.d/superpin', 'w') do |file|
- file << "Package: *
"
- file << "Pin: release o=LP-PPA-kubuntu-ci-#{@type}
"
- file << "Pin-Priority: 999
"
- end
- end
-end
-
-# Disable invoke-rc.d because it is crap and (sometimes) fails to detect
-# upstart making postinst that call it fail for no good reason at all.
-# This primarily affects modemmanager.
-# http://kci.pangea.pub/job/mgmt_daily_promotion_utopic_unstable/91/console
-File.open("/usr/sbin/invoke-rc.d", "w") { |f| f.write("#!/bin/sh
") }
-
-daily_ppa = CiPPA.new("#{project.stability}-daily", project.series)
-live_ppa = CiPPA.new("#{project.stability}", project.series)
-live_ppa.remove() # remove live before attempting to use daily.
-
-# Add the present daily snapshot, install everything.
-# If this fails then the current snapshot is kaputsies....
-# daily_ppa.add()
-daily_ppa.add()
-unless daily_ppa.install()
- puts "I: daily ppa failed to install."
- daily_purged = daily_ppa.purge()
- unless daily_purged
- puts "I: daily failed to install and then failed to purge. Maybe check maintscripts?"
- end
-end
-
-# NOTE: If daily failed to install, no matter if we can upgrade live it is
-# an improvement just as long as it can be installed...
-# So we purged daily again, and even if that fails we try to install live
-# to see what happens. If live is ok we are good, otherwise we would fail anyway.
-
-live_ppa.add()
-unless live_ppa.install()
- puts "E: all is vain! live PPA is not installing!"
- exit 1
-end
-
-# All is lovely. Let's make sure all live packages uninstall again (maintscripts!)
-# and then start the promotion.
-unless live_ppa.purge()
- puts "E: live PPA installed just fine, but can't be uninstalled again. Maybe check maintscripts?"
- exit 1
-end
-
-# uri = URI('https://api.launchpad.net/devel/')
-# http = Net::HTTP.new(uri.host, uri.port)
-# http.use_ssl = true
-# http.verify_mode = OpenSSL::SSL::VERIFY_NONE # read into this
-# http.set_debug_output($stdout)
-# wadl_d =nil
-# http.start {
-# r = Net::HTTP::Get.new('/devel/', 'Accept' => 'application/vd.sun.wadl+xml')
-# r = http.request(r) # Net::HTTPResponse object
-# wadl_d = r.body
-# }
-# w = WADL::Application.from_wadl(wadl_d)
-
-# p r = @access_token.post('/1.0/~apachelogger',
-# 'ws.op' => 'createPPA',
-# 'name' => 'kittens')
-# exit 0
-
-ubuntu = Launchpad::Rubber::from_json(Net::HTTP.get(URI('https://api.launchpad.net/1.0/ubuntu')))
-series = nil
-ubuntu.series.each do |_series|
- next unless _series.name == 'utopic'
- series = _series
- break
-end
-
-class Archive
- attr_accessor :ppa
-
- def initialize(name, series)
- @ppa = Launchpad::Rubber::from_json($token.get("/devel/~kubuntu-ci/+archive/ubuntu/#{name}").body)
- @series = series
- end
-
- def wipe
- @ppa.getPublishedSources(:distro_series => @series.self_link).each do |source|
- begin
- source.requestDeletion!()
- rescue
- puts "failed to request deletion, retry"
- sleep 1
- retry
- end
- end
- end
-
- def copy(from_ppa)
- from_ppa.getPublishedSources(:status => 'Published',
- :distro_series => @series.self_link).each do |source|
- begin
- @ppa.copyPackage!(:from_archive => from_ppa.self_link,
- :source_name => source.source_package_name,
- :version => source.source_package_version,
- :to_pocket => 'Release',
- :include_binaries => true)
- rescue
- puts "failed to request copy, retry"
- sleep 1
- retry
- end
- end
- end
-end
-
-live = Archive.new('unstable', series)
-daily = Archive.new('unstable-daily', series)
-
-daily.wipe
-daily.copy(live.ppa)
-
-exit 0
diff --git a/contained/lib b/contained/lib
deleted file mode 120000
index dc598c5..0000000
--- a/contained/lib
+++ /dev/null
@@ -1 +0,0 @@
-../lib
\ No newline at end of file
diff --git a/contained/ppa/promoter.py b/contained/ppa/promoter.py
deleted file mode 100755
index a2e885f..0000000
--- a/contained/ppa/promoter.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (C) 2013 Harald Sitter <sitter at kde.org>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import os
-import re
-import sys
-import datetime
-from launchpadlib.launchpad import Launchpad
-
-import argparse
-parser = argparse.ArgumentParser(description="lunchpadlib turd")
-parser.add_argument("-c", "--credentials", help="Location of the credenticals file")
-parser.add_argument("--chain", action='store_true', help="Chain promote ppa->daily->weekly")
-args = parser.parse_args()
-
-if args.credentials:
- lp = Launchpad.login_with("kubuntu-dev-tools", "production", credentials_file=args.credentials)
-else:
- lp = Launchpad.login_with("kubuntu-dev-tools", "production")
-
-def getPPAObject(owner, PPA):
- return lp.people[owner].getPPAByName(name=PPA)
-
-chain = False
-promote = False
-
-print datetime.datetime.now()
-
-# There are two operation modes:
-# - chain: copies from live kf5 ppa to daily *and* weekly, this is used by
-# iso promotion automation
-# - ppa description trigger: needs PROMOTE in description of daily to trigger
-# a promotion from daily to weekly
-if args.chain:
- print "Chain promoting from kf5 to daily to weekly"
- dailyPPA = getPPAObject("neon", "kf5")
- chain = True
- promote = True
-else:
- dailyPPA = getPPAObject("neon", "kf5-snapshot-daily")
- promote_match = re.search('PROMOTE', dailyPPA.description)
- if promote_match:
- print "Promoting daily to weekly. Kopypackage to the resuce!"
- promote = True
-
-if not promote:
- print "N: No promotion requested. Good day."
- sys.exit(0)
-
-print "------------------ Removing Promotion Stamp -------------------"
-
-desc = dailyPPA.description
-dailyPPA.description = re.sub("PROMOTE", "", desc)
-dailyPPA.lp_save()
-
-print "-------------- Removing Promotion Stamp:COMPLETE --------------"
-print "------------------------ Kopypackages -------------------------"
-
-# We want the checkout in the same dir as the PWD may be /root/ when called
-# via su etc.
-print "changing dir to %r" % os.path.dirname(os.path.abspath(__file__))
-os.chdir(os.path.dirname(os.path.abspath(__file__)))
-print "bzr checkout"
-os.system("bzr co lp:kubuntu-dev-tools")
-print "bzr up"
-os.system("cd kubuntu-dev-tools && bzr up")
-
-# Depending on promotion mode we either copy from kf5 or daily
-if chain:
- print "kopy kf5 to daily"
- os.system("cd kubuntu-dev-tools/bin && PYTHONPATH=../pylib/ python kopypackages -a ppa:neon/kf5 all ppa:neon/kf5-snapshot-daily all")
- print "kopy kf5 to weekly"
- os.system("cd kubuntu-dev-tools/bin && PYTHONPATH=../pylib/ python kopypackages -a ppa:neon/kf5 all ppa:neon/kf5-snapshot-weekly all")
-else:
- print "kopy daily to weekly"
- os.system("cd kubuntu-dev-tools/bin && PYTHONPATH=../pylib/ python kopypackages -a ppa:neon/kf5-snapshot-daily all ppa:neon/kf5-snapshot-weekly all")
-
-print "------------------- Kopypackages: COMPLETE ---------------------"
-print "Promotion Done. Thanks for flying Python"
-
diff --git a/contained/ppa/sync.py b/contained/ppa/sync.py
deleted file mode 100755
index 40325ac..0000000
--- a/contained/ppa/sync.py
+++ /dev/null
@@ -1,218 +0,0 @@
-#!/usr/bin/env python
-#
-# Script to copy sources from one PPA to another.
-#
-# Copyright (C) 2013 Rohan Garg <rohan at kde.org>
-# Copyright (C) 2013 Harald Sitter <sitter at kde.org>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-import re
-import sys
-import os
-from lazr.restfulclient.errors import HTTPError
-from optparse import OptionParser
-from launchpadlib.launchpad import Launchpad
-import datetime
-
-import argparse
-parser = argparse.ArgumentParser(description="lunchpadlib turd")
-parser.add_argument("-c", "--credentials", help="Location of the credenticals file")
-args = parser.parse_args()
-
-if args.credentials:
- lp = Launchpad.login_with("kubuntu-dev-tools", "production", credentials_file=args.credentials)
-else:
- lp = Launchpad.login_with("kubuntu-dev-tools", "production")
-
-date_today = datetime.datetime.utcnow().date()
-message_prefix = "Requesting copy of"
-
-def getPPAObject(owner, PPA):
- return lp.people[owner].getPPAByName(name=PPA)
-
-def getRelease(source):
- return source.distro_series_link[source.distro_series_link.rfind('/') +1:]
-
-def hasBuiltToday():
- buildPPA = getPPAObject("neon", "kf5")
- stamp_match = re.search('Last-Autobuild: (.*)$', buildPPA.description)
- if not stamp_match:
- print "E: No Last-Autobuild stamp found, considering no build done."
- return False
- else:
- date_string = stamp_match.group(1)
- date_string_list = date_string.split('-')
- date_stamp = datetime.date(int(date_string_list[0]), int(date_string_list[1]), int(date_string_list[2]))
- if date_stamp >= date_today:
- return True
- return False
-
-def hasSyncedToday():
- syncPPA = getPPAObject("neon", "kf5-snapshot-daily")
- stamp_match = re.search('Last-Update: (.*)', syncPPA.description)
- if not stamp_match:
- print "I: No Last-Update stamp found, assuming snapshot created and stamping for today."
- syncPPA.description = syncPPA.description + "
Last-Update: " + date_today.isoformat()
- syncPPA.lp_save()
- sys.exit(0)
- else:
- date_string = stamp_match.group(1)
- date_string_list = date_string.split('-')
- date_stamp = datetime.date(int(date_string_list[0]), int(date_string_list[1]), int(date_string_list[2]))
- if date_stamp >= date_today:
- return True
- return False
-
-if not hasBuiltToday():
- print "I: Apparently autobuild was not triggered yet, not attempting to create a snapshot."
- sys.exit(0)
-else:
- print "I: Autobuild attempted, attempting sync."
-
-if hasSyncedToday():
- print "I: Already created a snapshot today; aborting."
- sys.exit(0)
-else:
- print "I: No sync attempted today, continuing"
-
-sourcePPA = getPPAObject("neon", "kf5")
-targetPPA = getPPAObject("neon", "kf5-snapshot-daily")
-
-print "------------------------- Recipe Check ------------------------"
-
-# Note that querying over people[neon] or .recipes will almost always yield
-# wrong results or simply cause HTTP-503. Therefore we query over branches of
-# project-neon5, which in turn requires all kf5 recipes to be associated to
-# branches of project-neon5.
-project = lp.projects['project-neon5']
-branches = project.getBranches()
-for branch in branches:
- print "::%r::" % branch.name
-
- for recipe in branch.recipes:
- print "%r (%r)" % (recipe.name, recipe.base_branch.bzr_identity)
- if 'project-neon5-' not in recipe.name:
- print "N: skipping %r; not a neon5 recipe" % recipe.name
- continue # Not a neon5 recipe
- for pending_build in recipe.pending_builds:
- print "E: recipe %r has pending builds! NOT syncing" % recipe.name
- sys.exit(1) # having pending builds on daily jobs blocks sync
- if not recipe.build_daily:
- print "N: skipping %r; not built daily" % recipe.name
- continue # Not a daily build -> has no pending builds, so we are ok with it
- if recipe.last_build.datebuilt and recipe.base_branch.last_mirrored:
- # New change in branch AND not built today ~= hasn't got a build scheduled yet.
- if recipe.last_build.datebuilt < recipe.base_branch.last_mirrored and (recipe.last_build.datebuilt.date() != date_today):
- print "E: recipe %r still needs a build for new data (last: %r)" % (recipe.name, recipe.last_build.datebuilt)
- sys.exit(1) # hasn't gotten a build scheduled yet
- else:
- print "N: recipe %r doesn't have a recent build nor does it need one; ignoring" % recipe.name
-
- # TODO: technically we should also check:
- # * all configured series have had a build today
- # * that all of them are buildstate 'Successfully built'
-
-print "-------------------- Recipe Check: COMPLETE -------------------"
-print "------------------------ Archive Check ------------------------"
-
-# ----- If we get here all recipes have been schedule and have built successfully
-
-# getPublishedSources accepts
-# * Pending
-# * Published
-# * Superseded
-# * Deleted
-# * Obsolte
-# we only care about the first two though as the others have no baring on whether
-# or not we want to sync.
-
-# Pending comes before Published, a source that is Pending is definitely not
-# built, so abort syncing if we encounter even one Pending source.
-for source in sourcePPA.getPublishedSources(status='Pending'):
- print "E: Got at least one pending source (%r)! NOT syncing" % source.source_package_name
- sys.exit(1)
-
-# Published sources are all sources that are considered 'active' meaning they
-# are presently supplying binaries or have pending builds.
-# These are the sources that we want to copy.
-for source in sourcePPA.getPublishedSources(status='Published'):
- print("::%r (%r) [%r]::" % (source.source_package_name, source.source_package_version, source.status))
-
- # 1. source status is 'Published'
- if source.status != "Published":
- print("E: %r is not completely published! NOT syncing" % (source.source_package_name))
- sys.exit(1)
-
- # 2. all builds of source have 'Successfully built'
- for build in source.getBuilds():
- if build.buildstate != "Successfully built":
- print "E: %r [%r] did not successfully build (yet)! NOT syncing" % (source.source_package_name, build.arch_tag)
- sys.exit(1)
-
- # 3. all binaries of source are not 'Pending'
- # As with source publication we disregard !Pending !Published; the
- # other states can only ever be reached by their source having that state.
- for binary in source.getPublishedBinaries():
- if binary.status == "Pending":
- print "E: %r has a pending binary publication for %r" % (source.source_package_name, binary.binary_package_name)
- sys.exit(1)
-
-print "------------------- Archive Check: COMPLETE -------------------"
-print "----------------------------- Sync ----------------------------"
-
-# Note that we do a two-pass iteration to make sure all criteria are met before
-# trying to copy stuff around.
-for source in sourcePPA.getPublishedSources(status='Published'):
- release = getRelease(source)
- sys.stdout.write("%s %s from %s [%s] to %s [%s]...
"
- % (message_prefix,
- source.source_package_name + ' ' + source.source_package_version,
- sourcePPA.name,
- release,
- targetPPA.name,
- release))
- sys.stdout.flush()
-
- # We retry sync requests per package 15 times, after that we either abort
- # and move on with other packages (hoping stuff will still work),
- # or we submitted the sync successfully and can move on earlier than that.
- counter = 0
- sync_submitted = False
- while not sync_submitted and counter != 15:
- counter += 1
- try:
- targetPPA.syncSource(from_archive=sourcePPA,
- include_binaries=True,
- source_name=source.source_package_name,
- to_pocket='Release',
- to_series=release,
- version=source.source_package_version)
- except HTTPError, e:
- print(" FAILED:")
- print("[HTTP %s]: %s" % (e.response['status'], e.content[e.content.find('(') + 1:e.content.find(')')]))
- else:
- sync_submitted = True
- print("Done")
-
-print "----------------------- Sync: COMPLETE ------------------------"
-print "------------------- Updating Snapshot Stamp -------------------"
-
-desc = targetPPA.description
-targetPPA.description = re.sub("(Last-Update:) (.*)", "\g<1> " + date_today.isoformat(), desc)
-targetPPA.lp_save()
-
-print "-------------- Updating Snapshot Stamp:COMPLETE ---------------"
-print "Snapshot Done. Thanks for flying Python"
-
--
ci-tooling packaging
More information about the pkg-kde-commits
mailing list