[SCM] ci-tooling packaging branch, master, updated. 39985e1a3a4812ec26f07c58c4417fcc1f91da42
Harald Sitter
apachelogger-guest at moszumanska.debian.org
Wed Feb 25 10:53:45 UTC 2015
Gitweb-URL: http://git.debian.org/?p=pkg-kde/ci-tooling.git;a=commitdiff;h=10e956d
The following commit has been merged in the master branch:
commit 10e956d44100147ed2d31a5bdeeb241614f8dd55
Author: Harald Sitter <sitter at kde.org>
Date: Wed Feb 25 11:44:26 2015 +0100
refactor binary list building in install test to use a threadpool
there's some 100-170 sources per series that are published, for each of
those we need to do a getPublishedBinaries which according to profiling
takes anywhere between 0.5 and 3 seconds (KCI always is in the upper
half of that due to inherit traffic delay to australia).
so we are looking at well above 1 minute just spent querying. in practice
on KCI we are more in the range of upwards of 3 minutes.
---
kci/daily-qa.rb | 117 +++++++++++++++++++++------------------------------
kci/install_check.rb | 35 +++++++++++----
2 files changed, 75 insertions(+), 77 deletions(-)
diff --git a/kci/daily-qa.rb b/kci/daily-qa.rb
index 7c4c84f..df57cae 100755
--- a/kci/daily-qa.rb
+++ b/kci/daily-qa.rb
@@ -8,8 +8,11 @@ require_relative "lib/install_and_require"
install_and_require "logger"
install_and_require "thwait"
install_and_require "logger/colors"
+
require_relative "lib/jenkins"
require_relative "lib/lp"
+require_relative 'lib/retry'
+require_relative 'lib/thread_pool'
$logger = Logger.new(STDERR)
$logger.level = Logger::INFO
@@ -87,78 +90,56 @@ class CiPPA
end
def packages
- return @packages unless @packages.empty?
-
- $logger.info "Building package list for PPA #{@type}."
-
-# # TODO: this presently does not take multiarch into account.
-# Dir.glob("/var/lib/apt/lists/ppa.launchpad.net_kubuntu-ci_#{@type}_*_Packages") do |cache_file|
-# file = File.open(cache_file, 'r')
-# while !file.eof?
-# line = file.readline
-# match = line.match(/^Package: (.*)$/)
-# next unless match and match.size == 2 # fullmatch + name match
-# @packages << match[1]
-# end
-# end
-# @packages.uniq!
-# @packages.delete_if { |p| p.end_with?('-dbg') || p.end_with?('-data') }
-# @packages.sort!
-
- ubuntu = Launchpad::Rubber::from_json(Net::HTTP.get(URI('https://api.launchpad.net/1.0/ubuntu')))
- series = nil
- ubuntu.series.each do |_series|
- next unless _series.name == @series
- series = _series
- break
- end
-
- reference_distro_arch_series_link = nil
- series.architectures.each do |_arch|
- # FIXME: arch harcode
- next unless _arch.architecture_tag == 'amd64'
- reference_distro_arch_series_link = _arch.self_link
- break
- end
-
- ppas = Launchpad::Rubber::from_json(Net::HTTP.get(URI('https://api.launchpad.net/1.0/~kubuntu-ci/ppas'))).entries
- p ppas
- ppa = nil
- ppas.each do |_ppa|
- p _ppa.name
- next unless _ppa.name == @type
- ppa = _ppa
- break
+ return @packages unless @packages.empty?
+
+ logger = $logger
+ logger.info "Building package list for PPA #{@type}."
+
+ series = Launchpad::Rubber.from_url("https://api.launchpad.net/devel/ubuntu/#{@series}")
+ # FIXME: arch hardcode
+ reference_distro_arch_series_link =
+ Launchpad::Rubber.from_url("#{series.self_link}/amd64")
+ ppa = Launchpad::Rubber.from_url("https://api.launchpad.net/devel/~kubuntu-ci/+archive/ubuntu/#{@type}")
+
+ packages = {}
+
+ sources = ppa.getPublishedSources(status: 'Published',
+ distro_series: series)
+ source_queue = Queue.new
+ sources.each { |s| source_queue << s }
+ binary_queue = Queue.new
+ BlockingThreadPool.run(8) do
+ until source_queue.empty?
+ source = source_queue.pop(true)
+ Retry.retry_it do
+ binary_queue << source.getPublishedBinaries
+ end
end
+ end
- packages = {}
- # ppa.getPublishedBinaries(:ordered => false,
- # :status => 'Published',
- # :distro_arch_series => reference_distro_arch_series_link).each do |binary|
- # packages[binary.binary_package_name] = binary.binary_package_version
- # end
- ppa.getPublishedSources(:status => 'Published',
- :distro_series => series).each do |source|
- binaries = source.getPublishedBinaries()
- binaries.each do |binary|
-# puts "archspecific: #{binary.architecture_specific} . #{binary.binary_package_name} . #{binary.binary_package_version}"
- next if binary.binary_package_name.end_with?('-dbg')
- # Do not include known to conflict packages
- next if binary.binary_package_name == 'libqca2-dev'
- # Do not include udebs, this unfortunately cannot be determined easily via api
- next if binary.binary_package_name.start_with?('oem-config')
- # Unresolvable as that requires more PPAs...
- next if binary.binary_package_name == 'kubuntu-ci-live'
- next if binary.binary_package_name == 'kubuntu-plasma5-desktop'
- if binary.architecture_specific
- next unless binary.distro_arch_series_link == reference_distro_arch_series_link
- end
- packages[binary.binary_package_name] = binary.binary_package_version
- end
+ until binary_queue.empty?
+ binaries = binary_queue.pop(true)
+ binaries.each do |binary|
+ # Do not include debug packages, they can't conflict anyway, and if
+ # they did we still wouldn't care.
+ next if binary.binary_package_name.end_with?('-dbg')
+ # Do not include known to conflict packages
+ next if binary.binary_package_name == 'libqca2-dev'
+ # Do not include udebs, this unfortunately cannot be determined
+ # easily via the API.
+ next if binary.binary_package_name.start_with?('oem-config')
+ # Unresolvable as that requires more PPAs...
+ next if binary.binary_package_name == 'kubuntu-ci-live'
+ next if binary.binary_package_name == 'kubuntu-plasma5-desktop'
+ if binary.architecture_specific
+ next unless binary.distro_arch_series_link == reference_distro_arch_series_link
+ end
+ packages[binary.binary_package_name] = binary.binary_package_version
end
+ end
- $logger.info "Built package list: #{packages.keys().join(", ")}"
- return @packages = packages
+ logger.info "Built package list: #{packages.keys.join(', ')}"
+ @packages = packages
end
def install
diff --git a/kci/install_check.rb b/kci/install_check.rb
index da1dc35..7b38699 100755
--- a/kci/install_check.rb
+++ b/kci/install_check.rb
@@ -6,9 +6,10 @@ ENV['PATH'] = '/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin'
require 'logger'
require 'logger/colors'
require 'thwait'
-require_relative 'lib/lp'
-require 'benchmark'
+require_relative 'lib/lp'
+require_relative 'lib/retry'
+require_relative 'lib/thread_pool'
LOG = Logger.new(STDERR)
LOG.level = Logger::INFO
@@ -52,18 +53,34 @@ class CiPPA
ppa = Launchpad::Rubber.from_path("~kubuntu-ci/+archive/ubuntu/#{@type}")
packages = {}
- ppa.getPublishedSources(status: 'Published',
- distro_series: series).each do |source|
- binaries = source.getPublishedBinaries
+
+ sources = ppa.getPublishedSources(status: 'Published',
+ distro_series: series)
+ source_queue = Queue.new
+ sources.each { |s| source_queue << s }
+ binary_queue = Queue.new
+ BlockingThreadPool.run(8) do
+ until source_queue.empty?
+ source = source_queue.pop(true)
+ Retry.retry_it do
+ binary_queue << source.getPublishedBinaries
+ end
+ end
+ end
+
+ until binary_queue.empty?
+ binaries = binary_queue.pop(true)
binaries.each do |binary|
- # Do not include debug packages, they can't conflict anyway, and if they
- # did we still wouldn't care.
+ # Do not include debug packages, they can't conflict anyway, and if
+ # they did we still wouldn't care.
next if binary.binary_package_name.end_with?('-dbg')
# Do not include known to conflict packages
next if binary.binary_package_name == 'libqca2-dev'
- # Do not include udebs, this unfortunately cannot be determined easily
- # via the API.
+ # Do not include udebs, this unfortunately cannot be determined
+ # easily via the API.
next if binary.binary_package_name.start_with?('oem-config')
+ next if binary.binary_package_name == 'kubuntu-ci-live'
+ next if binary.binary_package_name == 'kubuntu-plasma5-desktop'
if binary.architecture_specific
next unless binary.distro_arch_series_link == host_arch
end
--
ci-tooling packaging
More information about the pkg-kde-commits
mailing list