[game-data-packager] 23/25: Move all package-building logic into PackageSystem
Simon McVittie
smcv at debian.org
Sun Oct 9 21:26:07 UTC 2016
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit 6816de96d6bf8792c385a6f727ceba7d954a3ed9
Author: Simon McVittie <smcv at debian.org>
Date: Sun Oct 9 20:31:34 2016 +0100
Move all package-building logic into PackageSystem
---
game_data_packager/build.py | 156 ++-----------------------------
game_data_packager/packaging/__init__.py | 20 ++++
game_data_packager/packaging/arch.py | 42 ++++++++-
game_data_packager/packaging/deb.py | 55 ++++++++++-
game_data_packager/packaging/rpm.py | 62 +++++++++++-
5 files changed, 183 insertions(+), 152 deletions(-)
diff --git a/game_data_packager/build.py b/game_data_packager/build.py
index 5dfd104..2f07b8c 100644
--- a/game_data_packager/build.py
+++ b/game_data_packager/build.py
@@ -25,7 +25,6 @@ import shutil
import stat
import subprocess
import tempfile
-import time
import urllib.request
import zipfile
@@ -50,7 +49,6 @@ from .util import (AGENT,
copy_with_substitutions,
lang_score,
mkdir_p,
- normalize_permissions,
rm_rf,
recursive_utime,
which)
@@ -237,13 +235,6 @@ class PackagingTask(object):
# None or an existing directory in which to save downloaded files.
self.save_downloads = None
- # How to compress the .deb:
- # True: dpkg-deb's default
- # False: -Znone
- # str: -Zstr (gzip, xz or none)
- # list: arbitrary options (e.g. -z9 -Zgz -Sfixed)
- self.compress_deb = game.compress_deb
-
# Factory for a progress report (or None).
self.progress_factory = lambda info=None: None
@@ -2008,30 +1999,18 @@ class PackagingTask(object):
packages = set()
for package in ready:
- arch = package.architecture
- if arch != 'all':
- arch = self.packaging.get_architecture(arch)
- arch = self.packaging.ARCH_DECODE.get(arch, arch)
-
if not self.check_complete(package, log=True):
raise SystemExit(1)
- destdir = os.path.join(self.get_workdir(),
- '%s.DESTDIR' % package.name)
+ per_package_dir = os.path.join(self.get_workdir(),
+ '%s.d' % package.name)
+ destdir = os.path.join(per_package_dir, 'DESTDIR')
self.fill_dest_dir(package, destdir)
- if self.packaging.derives_from('deb'):
- pkg = self.build_deb(destdir, package, arch, destination,
- compress=compress)
- elif self.packaging.derives_from('rpm'):
- pkg = self.build_rpm(destdir, package, arch, compress=compress)
- elif self.packaging.derives_from('arch'):
- pkg = self.build_arch(destdir, package, arch, destination,
- compress=compress)
-
- if pkg is None:
- raise SystemExit(1)
-
+ self.__check_component(package)
+ pkg = self.packaging.build_package(per_package_dir, self.game,
+ package, destination, compress=compress)
+ assert pkg is not None
packages.add(pkg)
return packages
@@ -2144,9 +2123,11 @@ class PackagingTask(object):
self.game.shortname, path)
yield path
- def check_component(self, package):
+ def __check_component(self, package):
# redistributable packages are redistributable as long as their
# optional license file is present
+ # FIXME: only do this for .deb?
+ # FIXME: shouldn't modify package.component in-place
if package.component == 'local':
return
for f in package.optional_files:
@@ -2158,123 +2139,6 @@ class PackagingTask(object):
return
return
- def build_deb(self, destdir, package, arch, destination, compress=True):
- self.check_component(package)
- self.packaging.fill_dest_dir_deb(game, package, destdir)
- normalize_permissions(destdir)
-
- # it had better have a /usr and a DEBIAN directory or
- # something has gone very wrong
- assert os.path.isdir(os.path.join(destdir, 'usr')), destdir
- assert os.path.isdir(os.path.join(destdir, 'DEBIAN')), destdir
-
- deb_basename = '%s_%s_%s.deb' % (package.name, package.version, arch)
-
- outfile = os.path.join(os.path.abspath(destination), deb_basename)
-
- # only compress if the caller says we should, the YAML
- # says it's worthwhile, and this isn't a ripped CD (Vorbis
- # is already compressed)
- if not compress or not self.compress_deb or package.rip_cd:
- dpkg_deb_args = ['-Znone']
- elif self.compress_deb is True:
- dpkg_deb_args = []
- elif isinstance(self.compress_deb, str):
- dpkg_deb_args = ['-Z' + self.compress_deb]
- elif isinstance(self.compress_deb, list):
- dpkg_deb_args = self.compress_deb
-
- try:
- logger.info('generating package %s', package.name)
- check_output(['fakeroot', 'dpkg-deb'] +
- dpkg_deb_args +
- ['-b', '%s.deb.d' % package.name, outfile],
- cwd=self.get_workdir())
- except subprocess.CalledProcessError as cpe:
- print(cpe.output)
- raise
-
- rm_rf(destdir)
- return outfile
-
-
- def build_arch(self, destdir, package, arch, destination, compress=True):
- self.packaging.fill_dest_dir_arch(game, package, destdir, compress,
- arch)
- normalize_permissions(destdir)
-
- assert os.path.isdir(os.path.join(destdir, 'usr')), destdir
- assert os.path.isfile(os.path.join(destdir, '.PKGINFO')), destdir
- assert os.path.isfile(os.path.join(destdir, '.MTREE')), destdir
-
- pkg_basename = '%s-%s-1-%s.pkg.tar.xz' % (package.name, package.version, arch)
- outfile = os.path.join(os.path.abspath(destination), pkg_basename)
-
- try:
- logger.info('generating package %s', package.name)
- files = set()
- for dirpath, dirnames, filenames in os.walk(destdir):
- for fn in filenames:
- full = os.path.join(dirpath, fn)
- full = full[len(destdir)+1:]
- files.add(full)
- check_output(['fakeroot', 'bsdtar', 'cfJ', outfile] + sorted(files),
- cwd=destdir, env={'LANG':'C'})
- except subprocess.CalledProcessError as cpe:
- print(cpe.output)
- raise
-
- rm_rf(destdir)
- return outfile
-
- def build_rpm(self, destdir, package, arch, compress=True):
- if arch == 'noarch':
- setarch = []
- else:
- setarch = ['setarch', arch]
-
- # increase local 'release' number on repacking
- if not self.packaging.is_installed(package.name):
- release = '0'
- elif Version(package.version) > Version(self.packaging.current_version(package.name)):
- release = '0'
- else:
- try:
- release = check_output(['rpm', '-q', '--qf' ,'%{RELEASE}',
- package.name]).decode('ascii')
- if (self.packaging.distro is not None and
- release.endswith('.' + self.packaging.distro)):
- release = release[:-(len(self.packaging.distro) + 1)]
- release = str(int(release) + 1)
- except (subprocess.CalledProcessError, ValueError):
- release = '0'
-
- if self.packaging.distro is not None:
- release = release + '.' + self.packaging.distro
-
- if compress:
- compress = self.compress_deb
-
- specfile = self.packaging.fill_dest_dir_rpm(game, package,
- self.get_workdir(), destdir, compress, arch, release)
- normalize_permissions(destdir)
-
- assert os.path.isdir(os.path.join(destdir, 'usr')), destdir
-
- try:
- logger.info('generating package %s', package.name)
- check_output(setarch + ['rpmbuild',
- '--buildroot', destdir,
- '-bb', '-v', specfile],
- cwd=self.get_workdir())
- except subprocess.CalledProcessError as cpe:
- print(cpe.output)
- raise
-
- return(os.path.expanduser('~/rpmbuild/RPMS/') + arch + '/'
- + package.name + '-'
- + package.version + '-' + release + '.' + arch + '.rpm')
-
def check_unpacker(self, wanted):
if not wanted.unpack:
return True
diff --git a/game_data_packager/packaging/__init__.py b/game_data_packager/packaging/__init__.py
index 0e04501..207561c 100644
--- a/game_data_packager/packaging/__init__.py
+++ b/game_data_packager/packaging/__init__.py
@@ -296,6 +296,26 @@ class PackagingSystem(metaclass=ABCMeta):
return (short_desc, long_desc)
+ def get_effective_architecture(self, package):
+ arch = package.architecture
+ if arch != 'all':
+ arch = self.get_architecture(arch)
+ return self.ARCH_DECODE.get(arch, arch)
+
+ @abstractmethod
+ def build_package(self, per_package_dir, game, package,
+ destination, compress=True):
+ """Build the .deb or equivalent in destination, and return its
+ filename.
+
+ per_package_dir may be used as scratch space. It already contains
+ a subdirectory named DESTDIR which has been populated with the files
+ to be packaged (so it contains DESTDIR/usr, etc.)
+
+ game and package are a GameData and a GameDataPackage respectively.
+ """
+ raise NotImplementedError
+
def get_packaging_system(format, distro=None):
mod = 'game_data_packager.packaging.{}'.format(format)
return importlib.import_module(mod).get_packaging_system(distro)
diff --git a/game_data_packager/packaging/arch.py b/game_data_packager/packaging/arch.py
index b1e9627..8c5576c 100644
--- a/game_data_packager/packaging/arch.py
+++ b/game_data_packager/packaging/arch.py
@@ -19,9 +19,15 @@
import logging
import os
import subprocess
+import time
from . import (PackagingSystem)
-from ..util import (run_as_root, check_output)
+from ..util import (
+ check_output,
+ normalize_permissions,
+ rm_rf,
+ run_as_root,
+ )
logger = logging.getLogger(__name__)
@@ -109,7 +115,7 @@ class ArchPackaging(PackagingSystem):
return self.rename_package(pr.package)
- def fill_dest_dir_arch(self, game, package, destdir, compress, arch):
+ def __fill_dest_dir_arch(self, game, package, destdir, compress, arch):
PKGINFO = os.path.join(destdir, '.PKGINFO')
short_desc, _ = self.generate_description(game, package)
size = check_output(['du','-bs','.'], cwd=destdir)
@@ -148,5 +154,37 @@ class ArchPackaging(PackagingSystem):
'--options=!all,use-set,type,uid,gid,mode,time,size,md5,sha256,link']
+ sorted(files), env={'LANG':'C'}, cwd=destdir)
+ def build_package(self, per_package_dir, game, package, destination,
+ compress=True):
+ destdir = os.path.join(per_package_dir, 'DESTDIR')
+ arch = self.get_effective_architecture(package)
+
+ self.__fill_dest_dir_arch(game, package, destdir, compress, arch)
+ normalize_permissions(destdir)
+
+ assert os.path.isdir(os.path.join(destdir, 'usr')), destdir
+ assert os.path.isfile(os.path.join(destdir, '.PKGINFO')), destdir
+ assert os.path.isfile(os.path.join(destdir, '.MTREE')), destdir
+
+ pkg_basename = '%s-%s-1-%s.pkg.tar.xz' % (package.name, package.version, arch)
+ outfile = os.path.join(os.path.abspath(destination), pkg_basename)
+
+ try:
+ logger.info('generating package %s', package.name)
+ files = set()
+ for dirpath, dirnames, filenames in os.walk(destdir):
+ for fn in filenames:
+ full = os.path.join(dirpath, fn)
+ full = full[len(destdir)+1:]
+ files.add(full)
+ check_output(['fakeroot', 'bsdtar', 'cfJ', outfile] + sorted(files),
+ cwd=destdir, env={'LANG':'C'})
+ except subprocess.CalledProcessError as cpe:
+ print(cpe.output)
+ raise
+
+ rm_rf(destdir)
+ return outfile
+
def get_packaging_system(distro=None):
return ArchPackaging()
diff --git a/game_data_packager/packaging/deb.py b/game_data_packager/packaging/deb.py
index a2b28b2..036449e 100644
--- a/game_data_packager/packaging/deb.py
+++ b/game_data_packager/packaging/deb.py
@@ -18,17 +18,27 @@
import logging
import os
+import stat
import subprocess
try:
+ from debian.deb822 import Deb822
from debian.debian_support import Version
except ImportError:
# make check
from distutils.version import LooseVersion as Version
+ Deb822 = None
from . import (PackagingSystem)
-from ..util import (check_output, mkdir_p, run_as_root)
+from ..data import (HashedFile)
+from ..paths import (DATADIR)
+from ..util import (
+ check_output,
+ mkdir_p,
+ normalize_permissions,
+ rm_rf,
+ run_as_root)
logger = logging.getLogger(__name__)
@@ -337,7 +347,7 @@ class DebPackaging(PackagingSystem):
return control
- def fill_dest_dir_deb(self, game, package, destdir):
+ def __fill_dest_dir_deb(self, game, package, destdir):
if package.component == 'local':
self.override_lintian(destdir, package.name,
'unknown-section', 'local/%s' % package.section)
@@ -372,5 +382,46 @@ class DebPackaging(PackagingSystem):
fd=open(control, 'wb'), encoding='utf-8')
os.chmod(control, 0o644)
+ def build_package(self, per_package_dir, game, package, destination,
+ compress=True):
+ destdir = os.path.join(per_package_dir, 'DESTDIR')
+ arch = self.get_effective_architecture(package)
+ self.__fill_dest_dir_deb(game, package, destdir)
+ normalize_permissions(destdir)
+
+ # it had better have a /usr and a DEBIAN directory or
+ # something has gone very wrong
+ assert os.path.isdir(os.path.join(destdir, 'usr')), destdir
+ assert os.path.isdir(os.path.join(destdir, 'DEBIAN')), destdir
+
+ deb_basename = '%s_%s_%s.deb' % (package.name, package.version, arch)
+
+ outfile = os.path.join(os.path.abspath(destination), deb_basename)
+
+ # only compress if the caller says we should, the YAML
+ # says it's worthwhile, and this isn't a ripped CD (Vorbis
+ # is already compressed)
+ if not compress or not game.compress_deb or package.rip_cd:
+ dpkg_deb_args = ['-Znone']
+ elif game.compress_deb is True:
+ dpkg_deb_args = []
+ elif isinstance(game.compress_deb, str):
+ dpkg_deb_args = ['-Z' + game.compress_deb]
+ elif isinstance(game.compress_deb, list):
+ dpkg_deb_args = game.compress_deb
+
+ try:
+ logger.info('generating package %s', package.name)
+ check_output(['fakeroot', 'dpkg-deb'] +
+ dpkg_deb_args +
+ ['-b', 'DESTDIR', outfile],
+ cwd=per_package_dir)
+ except subprocess.CalledProcessError as cpe:
+ print(cpe.output)
+ raise
+
+ rm_rf(destdir)
+ return outfile
+
def get_packaging_system(distro=None):
return DebPackaging()
diff --git a/game_data_packager/packaging/rpm.py b/game_data_packager/packaging/rpm.py
index 128d972..4012a42 100644
--- a/game_data_packager/packaging/rpm.py
+++ b/game_data_packager/packaging/rpm.py
@@ -19,9 +19,15 @@
import logging
import os
import subprocess
+import time
+from distutils.version import LooseVersion as Version
from . import (PackagingSystem)
-from ..util import (check_output, run_as_root)
+from ..util import (
+ check_output,
+ normalize_permissions,
+ run_as_root,
+ )
logger = logging.getLogger(__name__)
@@ -101,7 +107,7 @@ class RpmPackaging(PackagingSystem):
return self.rename_package(pr.package)
- def fill_dest_dir_rpm(self, game, package, workdir, destdir, compress,
+ def __fill_dest_dir_rpm(self, game, package, workdir, destdir, compress,
architecture, release):
specfile = os.path.join(workdir, '%s.spec' % package.name)
short_desc, long_desc = self.generate_description(game, package)
@@ -224,6 +230,58 @@ class RpmPackaging(PackagingSystem):
return specfile
+ def build_package(self, per_package_dir, game, package, destination,
+ compress=True):
+ destdir = os.path.join(per_package_dir, 'DESTDIR')
+ arch = self.get_effective_architecture(package)
+
+ if arch == 'noarch':
+ setarch = []
+ else:
+ setarch = ['setarch', arch]
+
+ # increase local 'release' number on repacking
+ if not self.is_installed(package.name):
+ release = '0'
+ elif Version(package.version) > Version(self.current_version(package.name)):
+ release = '0'
+ else:
+ try:
+ release = check_output(['rpm', '-q', '--qf' ,'%{RELEASE}',
+ package.name]).decode('ascii')
+ if (self.distro is not None and
+ release.endswith('.' + self.distro)):
+ release = release[:-(len(self.distro) + 1)]
+ release = str(int(release) + 1)
+ except (subprocess.CalledProcessError, ValueError):
+ release = '0'
+
+ if self.distro is not None:
+ release = release + '.' + self.distro
+
+ if compress:
+ compress = game.compress_deb
+
+ specfile = self.__fill_dest_dir_rpm(game, package,
+ per_package_dir, destdir, compress, arch, release)
+ normalize_permissions(destdir)
+
+ assert os.path.isdir(os.path.join(destdir, 'usr')), destdir
+
+ try:
+ logger.info('generating package %s', package.name)
+ check_output(setarch + ['rpmbuild',
+ '--buildroot', destdir,
+ '-bb', '-v', specfile],
+ cwd=per_package_dir)
+ except subprocess.CalledProcessError as cpe:
+ print(cpe.output)
+ raise
+
+ return(os.path.expanduser('~/rpmbuild/RPMS/') + arch + '/'
+ + package.name + '-'
+ + package.version + '-' + release + '.' + arch + '.rpm')
+
# XXX: dnf is written in python3 and has a stable public api,
# it is likely faster to use it instead of calling 'dnf' pgm.
#
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list