[SCM] ci-tooling packaging branch, master, updated. fcb5e71a1731be1b272c2fdc68623c4becd4013a

[Harald Sitter]

Gitweb-URL: http://git.debian.org/?p=pkg-kde/ci-tooling.git;a=commitdiff;h=db88ccf

The following commit has been merged in the master branch:
commit db88ccfa1a15aaac9d2b727a6dc4b243780965d2
Author: Harald Sitter <sitter at kde.org>
Date:   Fri May 8 12:42:12 2015 +0200

    try out a new approach to ppa promotion and launchpad oops on copies
    
    instead of sequencing the entire copy which makes a whole run take about
    an hour or two per type and series let's copy all on 8 threads and then
    verify that they have appeared in the target PPA. if not sleep 5 minutes
    and try again. this can be repated up to 10 times (i.e. 50 minutes worth
    of waiting).
    
    this commit also sees the introduction of two new helpers that might move
    elsewhere...
    - a Queue version that can init from an array and convert into
      one (latter unfortunately twice as inefficient as former).
    - a module to prepend on PPA rubbers to easily verify package presence.
      this is mostly just convenience and threading isolation there isn't
      any actual business logic. it's possibly that the lp api will need some
      stronger type coercion/reflection than the generic rubber and grow actual
      REST api design (i.e. ReflectionType + ConnectionType + RouteType)
---
 kci/ppa_promote.rb | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 86 insertions(+), 6 deletions(-)

diff --git a/kci/ppa_promote.rb b/kci/ppa_promote.rb
index 2bf27f8..7e2c57b 100755
--- a/kci/ppa_promote.rb
+++ b/kci/ppa_promote.rb
@@ -1,5 +1,6 @@
 #!/usr/bin/env ruby
 
+require 'date'
 require 'json'
 require 'logger'
 require 'logger/colors'
@@ -14,9 +15,11 @@ require_relative 'lib/retry'
 # Since waiting 20 and then doing it again would be slower than just copying one
 # by one we instead do that by limiting our thread pool.
 # https://bugs.launchpad.net/launchpad/+bug/1314569
-THREAD_COUNT = 1
+THREAD_COUNT = 8
 # Wiping is unaffected as far as we can tell.
 WIPE_THREAD_COUNT = 8
+# Polling isn't either.
+POLL_THREAD_COUNT = WIPE_THREAD_COUNT
 
 # TODO: wait for wipe and publish to be done. fail if publish takes >40 minutes
 
@@ -28,6 +31,58 @@ LOG.warn 'PPA Promote'
 Project = Struct.new(:series, :type)
 project = Project.new(ENV.fetch('DIST'), ENV.fetch('TYPE'))
 
+class Queue
+  alias_method :super_init, :initialize
+
+  def initialize(array = nil)
+    super_init
+    return if array.nil?
+    fail 'Queue can only be constructed from an Array' unless array.is_a?(Array)
+    array.each { |i| self << i }
+  end
+
+  def to_a
+    # Queue isn't exactly the most nejoable thing in the world as it doesn't
+    # allow for random access so you cannot iterate over it, and it doesn't
+    # implement dup nor clone so you can't deep copy it either.
+    # Now since we need to iterate queue to convert it in an array and iteration
+    # means destructive popping we first need to pop it into an Array and then
+    # iterate over the array to push the values back into the queue. Quite mad.
+    ret = []
+    ret << pop until empty?
+    ret.each { |i| self << i }
+    ret
+  end
+end
+
+module VerifablePPA
+  EXISTING_STATES = %w(Pending Published)
+
+  # @param packages [Hash<String, String>] Hash of package-versions
+  # @return [Array<String>] array of source names that are existing
+  def existing_sources(packages)
+    source_queue = Queue.new(packages.to_a)
+    exist_queue = Queue.new
+    BlockingThreadPool.run(POLL_THREAD_COUNT) do
+      until source_queue.empty?
+        entry = source_queue.pop(true)
+        exist_queue << entry[0] if source_exist?(*entry)
+      end
+    end
+    exist_queue.to_a
+  end
+
+  def source_exist?(name, version)
+    sources = getPublishedSources(source_name: name, version: version)
+    sources.reject! { |s| !EXISTING_STATES.include?(s.status) }
+    return true if sources.size == 1
+    if sources.size > 1
+      fail "Found more than one matching source for #{name}=#{version}"
+    end
+    false
+  end
+end
+
 class Archive
   attr_accessor :ppa
 
@@ -42,8 +97,7 @@ class Archive
     %i(Pending Published Superseded Obsolete).each do |status|
       sources = @ppa.getPublishedSources(status: status,
                                          distro_series: @series)
-      source_queue = Queue.new
-      sources.each { |s| source_queue << s }
+      source_queue = Queue.new(sources)
       BlockingThreadPool.run(WIPE_THREAD_COUNT) do
         until source_queue.empty?
           source = source_queue.pop(true)
@@ -59,8 +113,35 @@ class Archive
   end
 
   def copy(packages, from_ppa)
-    source_queue = Queue.new
-    packages.each_pair { |a| source_queue << a }
+    # We are modifying, dup it first.
+    packages = packages.dup
+    make_ppa_verifable!
+    Retry.retry_it(times: 10, sleep: (60 * 5)) do
+      copy_internal(packages, from_ppa)
+      existing_sources = @ppa.existing_sources(packages)
+      packages.reject! do |p|
+        next true if existing_sources.include?(p)
+        LOG.warn "Package #{p} hasn't been copied. Going to try again shortly."
+        false
+      end
+      unless packages.empty?
+        fail "Not all packages copied successfully #{packages}"
+      end
+    end
+  end
+
+  private
+
+  def make_ppa_verifable!
+    eigenclass = (class << @ppa; self; end)
+    return if eigenclass.included_modules.include?(VerifablePPA)
+    class << @ppa
+      include VerifablePPA
+    end
+  end
+
+  def copy_internal(packages, from_ppa)
+    source_queue = Queue.new(packages.to_a)
     BlockingThreadPool.run(THREAD_COUNT) do
       until source_queue.empty?
         entry = source_queue.pop(true)
@@ -74,7 +155,6 @@ class Archive
                             version: version,
                             to_pocket: 'Release',
                             include_binaries: true)
-          sleep 5 # Force sleep to avoid a race in launchpad, see THREAD_COUNT.
         end
       end
     end

-- 
ci-tooling packaging