[SCM] ci-tooling packaging branch, master, updated. 49d2097d47c89494d193b3d0b292a8c7c164b80c
Harald Sitter
apachelogger-guest at moszumanska.debian.org
Wed Feb 18 10:29:31 UTC 2015
Gitweb-URL: http://git.debian.org/?p=pkg-kde/ci-tooling.git;a=commitdiff;h=49d2097
The following commit has been merged in the master branch:
commit 49d2097d47c89494d193b3d0b292a8c7c164b80c
Author: Harald Sitter <sitter at kde.org>
Date: Wed Feb 18 11:29:18 2015 +0100
move builder into kci folder (was previously duplicated there)
---
builder.rb | 366 ---------------------------------------------------
kci/builder.rb | 32 ++---
kci/daily-promote.rb | 18 +--
3 files changed, 24 insertions(+), 392 deletions(-)
diff --git a/builder.rb b/builder.rb
deleted file mode 100644
index f1bcd49..0000000
--- a/builder.rb
+++ /dev/null
@@ -1,366 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'date'
-require 'fileutils'
-require 'timeout'
-require_relative 'lib/debian/changelog'
-
-ENV['HOME'] = '/var/lib/jenkins'
-ENV['PATH'] = '/usr/local/bin:/usr/local/sbin:/usr/bin:/usr/sbin:/bin:/sbin'
-ENV['GNUPGHOME'] = '/var/lib/jenkins/tooling/gnupg'
-KEYID = 'AB3CE70E'
-
-Project = Struct.new(:series, :stability, :name)
-
-# get basename, distro series, unstable/stable
-components = ARGV[0].split('_')
-raise 'Did not get a valid project identifier via ARGV0' unless components.size == 3
-project = Project.new(components[0], components[1], components[2])
-
-PPA = "ppa:kubuntu-ci/#{project.stability}"
-
-File.open('/etc/apt/apt.conf.d/apt-cacher', 'w') { |file| file.puts('Acquire::http { Proxy "http://10.0.3.1:3142"; };') }
-
-# PWD
-raise 'Could not change dir to ARGV1' unless Dir.chdir(ARGV[1])
-
-WORKSPACE_PATH = ARGV[1]
-
-# install deps
-`apt-get install -y xz-utils dpkg-dev ruby dput debhelper pkg-kde-tools devscripts python-launchpadlib ubuntu-dev-tools git`
-
-# version
-Dir.chdir('packaging') do
- $changelog = Changelog.new
-end
-
-# Note that the kubuntu version needs to be part of the *base* version as otherwise
-# different series uploads can end up with exactly the same tar name and launchpad
-# freaks out. So, kubuntu version in base/suffix not revision/suffix.
-eval %x[grep VERSION_ID /etc/os-release].strip
-
-source_name = $changelog.name
-version_suffix = "+git#{DateTime.now().strftime('%Y%m%d.%H%M')}+#{VERSION_ID}"
-version = "#{$changelog.version(Changelog::EPOCH | Changelog::BASE)}#{version_suffix}" # <-- needs git version and bzr version possibly
-tar_version = "#{$changelog.version(Changelog::BASE)}#{version_suffix}"
-version += "-0ubuntu0"
-
-# copy sources around
-FileUtils.rm_r('build') if File.exist?('build')
-FileUtils.mkpath('build/source/')
-`cp -rv source/* build/source/`
-
-Dir.chdir('build/source') do
- FileUtils.rm_rf(Dir.glob('**/.bzr'))
- FileUtils.rm_rf(Dir.glob('**/.git'))
- FileUtils.rm_rf(Dir.glob('**/.svn'))
-end
-
-# create orig tar
-Dir.chdir('build/') do
- tar = "#{source_name}_#{tar_version}.orig.tar"
- raise 'Failed to create a tarball' unless system("tar -cf #{tar} source")
- raise 'Failed to compress the tarball' unless system("xz -6 #{tar}")
-end
-
-# Copy some more
-`cp -rv packaging/* build/source/`
-
-# Create changelog entry
-Dir.chdir('build/source/') do
- raise 'Failed to create changelog entry' unless system({'DEBFULLNAME' => 'Kubuntu CI',
- 'DEBEMAIL' => 'kubuntu-ci at lists.launchpad.net'},
- "dch -b -v #{version} -D #{project.series} 'Automatic Kubuntu Build'")
-end
-
-# Rip out locale install
-Dir.chdir('build/source/') do
- Dir.glob('debian/*.install').each do | install_file_path |
- # Strip localized manpages
- # e.g. usr /share /man / * /man 7 /kf5options.7
- subbed = File.open(install_file_path).read().gsub(/^.*usr\/share\/man\/(\*|\w+)\/man\d\/.*$/, '')
- File.open(install_file_path, 'w') do |f|
- f << subbed
- end
-
- # FIXME: bloody workaround for kconfigwidgets and kdelibs4support containing legit locale data
- next if source_name == 'kconfigwidgets' or source_name == 'kdelibs4support'
- subbed = File.open(install_file_path).read().gsub(/^.*usr\/share\/locale.*$/, '')
- File.open(install_file_path, 'w') do |f|
- f << subbed
- end
- end
- # If the package is now empty, lintian override the empty warning to avoid false positives
- Dir.glob('debian/*.install').each do | install_file_path |
- next unless File.open(install_file_path, 'r').read.strip.empty?
- package_name = File.basename(install_file_path, '.install')
- lintian_overrides_path = install_file_path.gsub('.install', '.lintian-overrides')
- puts "#{package_name} is now empty, trying to add lintian override"
- fopen_flag = 'w'
- if File.exist?(lintian_overrides_path)
- fopen_flag = 'a'
- end
- File.open(lintian_overrides_path, fopen_flag) { |file| file.write("#{package_name}: empty-binary-package
") }
- end
-end
-
-# dpkg-buildpackage
-Dir.chdir('build/source/') do
- system('update-maintainer')
- raise 'Failed to build source package' unless system("dpkg-buildpackage -S -k#{KEYID}")
-end
-
-# Upload
-def timeout_spawn(cmd, timeout)
- pid = Process.spawn(cmd, {:pgroup => true})
- begin
- Timeout.timeout(timeout) do
- Process.waitpid(pid, 0)
- return ($?.exitstatus == 0)
- end
- rescue Timeout::Error
- Process.kill(15, -Process.getpgid(pid))
- return false
- end
-end
-Dir.chdir('build/source/') do
- changelog = Changelog.new
- # Upload likes to get stuck, so we do timeout control to prevent all of
- # builder from getting stuck.
- # We try to dput two times in a row giving it first 30 minutes and then
- # 15 minutes to complete. If it didn't manage to upload after that we
- # ignore the package and move on.
- dput = "dput #{PPA} ../#{changelog.name}_#{changelog.version(Changelog::BASE | Changelog::BASESUFFIX)}*.changes"
- success = false
- 2.times do |count|
- # Note: count starts at 0 ;)
- if timeout_spawn(dput, 60 * (30.0/(count+1)))
- success = true
- break
- end
- end
- raise ' !!!!!!!!!!!!!!!! dput failed two times !!!!!!!!!!!!!!!!' unless success
- Dir.chdir('../..') do # main dir
- raise 'PPA Interaction Failed' unless system("#{File.expand_path(File.dirname(__FILE__))}/ppa-wait.py #{changelog.name} #{changelog.version} #{project.stability}")
- end
-end
-
-exit 0 unless File.exist?('logs/i386.log.gz')
-
-def segmentify(data, start_marker, end_marker)
- start_index = data.index(start_marker)
- end_index = data.index(end_marker)
- return [] unless start_index and end_index
- data = data.slice(start_index..end_index).split("
")
- data.shift # Ditch start line
- data.pop # Ditch end line
- return data
-end
-
-def puts_kci(type, str)
- puts "KCI-#{type} :: #{str}"
-end
-
-def puts_error(str)
- puts_kci('E', str)
-end
-
-def puts_warning(str)
- puts_kci('W', str)
-end
-
-def puts_info(str)
- puts_kci('I', str)
-end
-
-def cmake_parse_optional!(data)
- missing = []
- start_line = false
- while not data.empty?
- line = data.shift
- if !start_line and line.empty?
- start_line = true
- next
- elsif start_line and not line.empty?
- next if line.strip.empty?
- match = line.match(/^ *\* (.*)$/)
- if match and match.size > 1
- missing << match[1]
- end
- else
- # Line is empty and the start conditions didn't match.
- # Either the block is not valid or we have reached the end.
- # In any case, break here.
- break
- end
- end
- return missing
-end
-
-def cmake_parse_package(line)
- _package='Could not find a package configuration file provided by'
- match = line.match(/^\s+#{_package}\s+"(.+)"/)
- if match and match.size > 1
- return [match[1]]
- end
- return []
-end
-
-def cmake_parse_warning(line)
- return [] unless line.include?('CMake Warning')
- # Lines coming from MacroOptionalFindPackage (from old parsing).
- return [] if line.include?('CMake Warning at /usr/share/kde4/apps/cmake/modules/MacroOptionalFindPackage.cmake')
- # Lines coming from find_package (from old parsing).
- return [] if line.match(/CMake Warning at [^ :]+:\d+ \(find_package\)/)
- # Lines coming from warnings inside the actual CMakeLists.txt as those can be arbitrary.
- # ref: "CMake Warning at src/worker/CMakeLists.txt:33 (message):"
- return [] if line.match(/CMake Warning at [^ :]*CMakeLists.txt:\d+ \(message\)/)
- return [] if line.start_with?('CMake Warning (dev)')
- return [] if line.start_with?('CMake Warning:')
- return [line]
-end
-
-def puts_cmake(data, source_name)
- data = segmentify(data, "dh_auto_configure", "dh_auto_build")
- return if data.empty?
-
- missing = []
- warnings = []
- while not data.empty?
- line = data.shift
- if line.include?('The following OPTIONAL packages have not been found')
- missing += cmake_parse_optional!(data)
- elsif line.include?('Could not find a package configuration file provided by')
- missing += cmake_parse_package(line)
- elsif line.include?('CMake Warning')
- warnings += cmake_parse_warning(line)
- end
- end
-
- missing.uniq!
- warnings.uniq!
-
- puts "KCI::CMAKE"
-
- ignore_missing = []
- cmake_ignore_path = "#{WORKSPACE_PATH}/packaging/debian/meta/cmake-ignore"
- if File.exist?(cmake_ignore_path)
- ignore_missing += File.read(cmake_ignore_path).strip.split("
")
- end
-
- missing.each do |dep|
- p dep
- p ignore_missing
- ignore = false
- ignore_missing.each do |possibly_ignore|
- p possibly_ignore == dep
- p dep.start_with?(possibly_ignore)
- p dep.start_with?(possibly_ignore.chomp('*'))
- if possibly_ignore == dep or dep.start_with?(possibly_ignore) or dep.start_with?(possibly_ignore.chomp('*'))
- ignore = true
- break
- end
- end
- next if ignore
- puts_warning("Missing Dep: #{dep}")
- end
-
- warnings.each do |warning|
- puts_warning(warning)
- end
-end
-
-def puts_list_missing(data)
- data = segmentify(data, "=== Start list-missing
", "=== End list-missing
")
- return if data.empty?
- puts "KCI::MISSING"
- data.each do |line|
- # Missing files are always considered errors
- puts_error(line)
- end
-end
-
-def puts_lintian(data, args = {})
- data = segmentify(data, "=== Start lintian
", "=== End lintian
")
- return if data.empty?
- puts "KCI::LINTIAN"
- data.each do |line|
- next if line.start_with?('warning: ') # random warnings from lintian itself
-
- # Package names can easily go beyond what shit can suck on, so gag it.
- next if line.include?('source-package-component-has-long-file-name')
- next if line.include?('package-has-long-file-name')
-
- # We really do not care about standards versions for now. They only ever get
- # bumped by the pkg-kde team anyway.
- next if line.include?('out-of-date-standards-version')
- next if line.include?('newer-standards-version')
-
- # Do not print symbols warnings if we already auto-updated.
- if args[:updated_symbols] and line.include?('symbols-file-contains-current-version-with-debian-revision')
- next
- end
-
- case line[0..1]
- when 'W:'
- puts_warning(line)
- next
- when 'E:'
- puts_error(line)
- next
- end
- puts_info(line)
- end
-end
-
-architectures_with_log = []
-Dir.chdir('logs/') do
- `gunzip *.log.gz`
- Dir.glob('*.log').each do |log|
- architectures_with_log << File.basename(log, '.log')
- end
-end
-
-archindep = File.read('archindep').strip rescue 'amd64' # Get archindep from ppa script.
-log_data = File.open("logs/#{archindep}.log").read
-
-updated_symbols = false
-# FIXME: stability wtf
-if project.stability == 'unstable' && log_data.match(/dpkg-gensymbols: warning: (.*)\/symbols doesn't match completely debian\/(.*).symbols/)
- puts 'KCI::SYMBOLS'
- if log_data.include?('dpkg-gensymbols: warning: some new symbols appeared')
- match = log_data.match(/--- debian\/(.*).symbols/)
- if match and match.size > 1
- Dir.chdir("#{WORKSPACE_PATH}/packaging") do
- system('git config core.sparsecheckout')
- system('git checkout -f remotes/packaging/kubuntu_unstable')
- system('git branch -a')
- system('git status')
- system('git reset --hard')
- captures = match.captures
- captures.each do |lib_package|
- puts "pkgkde-symbolshelper batchpatch -v #{tar_version} -c #{architectures_with_log.join(',')} #{WORKSPACE_PATH}/logs/*.log"
- system("pkgkde-symbolshelper batchpatch -v #{tar_version} -c #{architectures_with_log.join(',')} #{WORKSPACE_PATH}/logs/*.log")
- updated_symbols = ($? == 0)
- puts_info("Auto-updated symbols of #{lib_package}")
- end
- # Username et al apparently is somehow coming from .git or something apparently
- system('git status')
- system('git diff')
- system('git commit -a -m "Automatic symbol update"')
- end
- else
- puts_error('Failed to update symbols as the package name(s) could not be parsed.')
- end
- else
- puts_error('It would very much appear that symbols have been retracted')
- end
-end
-
-puts_cmake(log_data, source_name)
-puts_list_missing(log_data)
-puts_lintian(log_data, :updated_symbols => updated_symbols)
-
-# TODO: try to figure out when a qml dep is needed and make sure it is there...
-
-
diff --git a/kci/builder.rb b/kci/builder.rb
index 5c561e5..f1bcd49 100644
--- a/kci/builder.rb
+++ b/kci/builder.rb
@@ -29,7 +29,6 @@ WORKSPACE_PATH = ARGV[1]
# install deps
`apt-get install -y xz-utils dpkg-dev ruby dput debhelper pkg-kde-tools devscripts python-launchpadlib ubuntu-dev-tools git`
-
# version
Dir.chdir('packaging') do
$changelog = Changelog.new
@@ -128,16 +127,17 @@ Dir.chdir('build/source/') do
changelog = Changelog.new
# Upload likes to get stuck, so we do timeout control to prevent all of
# builder from getting stuck.
- # Also add some timing logic to increase chances of it suddenly working.
+ # We try to dput two times in a row giving it first 30 minutes and then
+ # 15 minutes to complete. If it didn't manage to upload after that we
+ # ignore the package and move on.
dput = "dput #{PPA} ../#{changelog.name}_#{changelog.version(Changelog::BASE | Changelog::BASESUFFIX)}*.changes"
success = false
- 4.times do |count|
+ 2.times do |count|
# Note: count starts at 0 ;)
if timeout_spawn(dput, 60 * (30.0/(count+1)))
success = true
break
end
- sleep(60) # Sleep for a minute
end
raise ' !!!!!!!!!!!!!!!! dput failed two times !!!!!!!!!!!!!!!!' unless success
Dir.chdir('../..') do # main dir
@@ -223,7 +223,7 @@ end
def puts_cmake(data, source_name)
data = segmentify(data, "dh_auto_configure", "dh_auto_build")
return if data.empty?
-
+
missing = []
warnings = []
while not data.empty?
@@ -236,18 +236,18 @@ def puts_cmake(data, source_name)
warnings += cmake_parse_warning(line)
end
end
-
+
missing.uniq!
warnings.uniq!
-
+
puts "KCI::CMAKE"
-
+
ignore_missing = []
cmake_ignore_path = "#{WORKSPACE_PATH}/packaging/debian/meta/cmake-ignore"
if File.exist?(cmake_ignore_path)
ignore_missing += File.read(cmake_ignore_path).strip.split("
")
end
-
+
missing.each do |dep|
p dep
p ignore_missing
@@ -264,7 +264,7 @@ def puts_cmake(data, source_name)
next if ignore
puts_warning("Missing Dep: #{dep}")
end
-
+
warnings.each do |warning|
puts_warning(warning)
end
@@ -286,21 +286,21 @@ def puts_lintian(data, args = {})
puts "KCI::LINTIAN"
data.each do |line|
next if line.start_with?('warning: ') # random warnings from lintian itself
-
+
# Package names can easily go beyond what shit can suck on, so gag it.
next if line.include?('source-package-component-has-long-file-name')
next if line.include?('package-has-long-file-name')
-
+
# We really do not care about standards versions for now. They only ever get
# bumped by the pkg-kde team anyway.
next if line.include?('out-of-date-standards-version')
next if line.include?('newer-standards-version')
-
+
# Do not print symbols warnings if we already auto-updated.
if args[:updated_symbols] and line.include?('symbols-file-contains-current-version-with-debian-revision')
next
end
-
+
case line[0..1]
when 'W:'
puts_warning(line)
@@ -325,7 +325,7 @@ archindep = File.read('archindep').strip rescue 'amd64' # Get archindep from ppa
log_data = File.open("logs/#{archindep}.log").read
updated_symbols = false
-# FIXME stability wtf
+# FIXME: stability wtf
if project.stability == 'unstable' && log_data.match(/dpkg-gensymbols: warning: (.*)\/symbols doesn't match completely debian\/(.*).symbols/)
puts 'KCI::SYMBOLS'
if log_data.include?('dpkg-gensymbols: warning: some new symbols appeared')
@@ -362,3 +362,5 @@ puts_list_missing(log_data)
puts_lintian(log_data, :updated_symbols => updated_symbols)
# TODO: try to figure out when a qml dep is needed and make sure it is there...
+
+
diff --git a/kci/daily-promote.rb b/kci/daily-promote.rb
index 136bb14..b0a8365 100644
--- a/kci/daily-promote.rb
+++ b/kci/daily-promote.rb
@@ -50,7 +50,7 @@ class Archive
to_pocket: 'Release',
include_binaries: true)
rescue => e
- LOG.warn "failed to request copy, retry"
+ LOG.warn 'failed to request copy, retry'
puts e
sleep 1
retry
@@ -64,15 +64,11 @@ packages = JSON.parse(File.read('package-list.json'))
Launchpad.authenticate
live = Archive.new(project.stability, project.series)
-
-LOG.info "Working on #{project.stability}-daily"
-daily = Archive.new("#{project.stability}-daily", project.series)
-daily.wipe
-daily.copy(packages, live.ppa)
-
-# LOG.info "Working on unstable-weekly"
-# weekly = Archive.new("unstable-weekly", project.series)
-# weekly.wipe
-# weekly.copy(live.ppa)
+%w(daily weekly).each do |snapshot|
+ LOG.info "Working on #{project.stability}-#{snapshot}"
+ ppa = Archive.new("#{project.stability}-#{snapshot}", project.series)
+ ppa.wipe
+ ppa.copy(packages, live.ppa)
+end
exit 0
--
ci-tooling packaging
More information about the pkg-kde-commits
mailing list