[game-data-packager] 09/11: Distinguish between expanded and unexpanded install, optional, provides
Simon McVittie
smcv at debian.org
Tue Nov 3 23:14:49 UTC 2015
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit 752b02affa4d394e71ac90114a8f1f7706797b92
Author: Simon McVittie <smcv at debian.org>
Date: Tue Nov 3 10:53:27 2015 +0000
Distinguish between expanded and unexpanded install, optional, provides
This is a step towards being able to round-trip file info through
objects, and more specifically making make_template work with objects
instead of its own cut-down version of GameData.
install, optional, provides are still sets of filenames or group names,
but they now have groups unexpanded.
install_files, optional_files, provides_files are sets of WantedFile
objects, with groups expanded as before.
---
game_data_packager/__init__.py | 82 +++++++++++++++++------------
game_data_packager/build.py | 114 ++++++++++++++++++++++-------------------
2 files changed, 110 insertions(+), 86 deletions(-)
diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index c5144f7..ba52588 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -66,6 +66,7 @@ class WantedFile(HashedFile):
self.license = False
self._look_for = []
self._provides = set()
+ self.provides_files = None
self._size = None
self.unpack = None
self.unsuitable = None
@@ -114,7 +115,7 @@ class WantedFile(HashedFile):
def provides(self, value):
self._provides = set(value)
- def to_yaml(self):
+ def to_yaml(self, expand=True):
ret = {
'distinctive_name': self.distinctive_name,
'name': self.name,
@@ -126,7 +127,6 @@ class WantedFile(HashedFile):
'executable',
'license',
'look_for',
- 'provides',
'skip_hash_matching',
):
v = getattr(self, k)
@@ -136,6 +136,13 @@ class WantedFile(HashedFile):
else:
ret[k] = v
+ if expand:
+ if self.provides_files:
+ ret['provides'] = sorted(f.name for f in self.provides_files)
+ else:
+ if self.provides:
+ ret['provides'] = sorted(self.provides)
+
for k in (
'download',
'group_members',
@@ -230,6 +237,12 @@ class GameDataPackage(object):
# set of names of WantedFile instances to be optionally installed
self._optional = set()
+ # set of WantedFile instances for install, with groups expanded
+ # only available after load_file_data()
+ self.install_files = None
+ # set of WantedFile instances for optional, with groups expanded
+ self.optional_files = None
+
self.version = GAME_PACKAGE_VERSION
# if not None, install every file provided by the files with
@@ -316,7 +329,7 @@ class GameDataPackage(object):
assert type(value) is str
self.langs = [value]
- def to_yaml(self):
+ def to_yaml(self, expand=True):
ret = {
'architecture': self.architecture,
'component': self.component,
@@ -333,9 +346,7 @@ class GameDataPackage(object):
'demo_for',
'dotemu',
'gog',
- 'install',
'install_to_docdir',
- 'optional',
'origin',
'rip_cd',
'steam',
@@ -348,6 +359,17 @@ class GameDataPackage(object):
else:
ret[k] = v
+ if expand and self.install_files is not None:
+ if self.install_files:
+ ret['install'] = sorted(f.name for f in self.install_files)
+ if self.optional_files:
+ ret['optional'] = sorted(f.name for f in self.optional_files)
+ else:
+ if self.install:
+ ret['install'] = sorted(self.install)
+ if self.optional:
+ ret['optional'] = sorted(self.optional)
+
for k in (
'better_version',
'copyright',
@@ -664,7 +686,7 @@ class GameData(object):
return help_text
- def to_yaml(self):
+ def to_yaml(self, expand=True):
files = {}
groups = {}
packages = {}
@@ -679,12 +701,12 @@ class GameData(object):
for filename, f in self.files.items():
if f.group_members is not None:
- groups[filename] = f.to_yaml()
+ groups[filename] = f.to_yaml(expand=expand)
else:
- files[filename] = f.to_yaml()
+ files[filename] = f.to_yaml(expand=expand)
for name, package in self.packages.items():
- packages[name] = package.to_yaml()
+ packages[name] = package.to_yaml(expand=expand)
if files:
ret['files'] = files
@@ -1023,19 +1045,19 @@ class GameData(object):
self.loaded_file_data = True
for package in self.packages.values():
- package.install = set(self._iter_expand_groups(package.install))
- package.optional = set(self._iter_expand_groups(package.optional))
+ package.install_files = set(self._iter_expand_groups(package.install))
+ package.optional_files = set(self._iter_expand_groups(package.optional))
for provider in package.install_contents_of:
- for filename in self._iter_expand_groups(self.files[provider].provides):
- if filename not in package.optional:
- package.install.add(filename)
+ for f in self._iter_expand_groups(self.files[provider].provides):
+ if f not in package.optional_files:
+ package.install_files.add(f)
for filename, f in self.files.items():
- f.provides = set(self._iter_expand_groups(f.provides))
+ f.provides_files = set(self._iter_expand_groups(f.provides))
- for provided in f.provides:
- self.providers.setdefault(provided, set()).add(filename)
+ for provided in f.provides_files:
+ self.providers.setdefault(provided.name, set()).add(filename)
if f.alternatives or f.group_members is not None:
continue
@@ -1070,11 +1092,9 @@ class GameData(object):
for package in self.packages.values():
for provider in package.install_contents_of:
assert provider in self.files, (package.name, provider)
- for filename in self.files[provider].provides:
- assert filename in self.files, (package.name, provider,
- filename)
- assert (filename in package.optional or
- filename in package.install), (package.name, filename)
+ for f in self.files[provider].provides_files:
+ assert (f in package.install_files or
+ f in package.optional_files), (package.name, f.name)
if package.rip_cd:
# we only support Ogg Vorbis for now
@@ -1082,11 +1102,7 @@ class GameData(object):
self.rip_cd_packages.add(package)
# there had better be something it wants to install
- assert package.install or package.rip_cd, package.name
- for installable in package.install:
- assert installable in self.files, installable
- for installable in package.optional:
- assert installable in self.files, installable
+ assert package.install_files or package.rip_cd, package.name
# check internal depedencies
for demo_for_item in package.demo_for:
@@ -1103,10 +1119,9 @@ class GameData(object):
for filename, wanted in self.files.items():
if wanted.unpack:
assert 'format' in wanted.unpack, filename
- assert wanted.provides, filename
- for f in wanted.provides:
- assert f in self.files, (filename, f)
- assert self.files[f].alternatives == [], (filename, f)
+ assert wanted.provides_files, filename
+ for f in wanted.provides_files:
+ assert f.alternatives == [], (filename, f.name)
if wanted.unpack['format'] == 'cat':
assert len(wanted.provides) == 1, filename
assert isinstance(wanted.unpack['other_parts'],
@@ -1147,7 +1162,8 @@ class GameData(object):
def _iter_expand_groups(self, grouped):
"""Given a set of strings that are either filenames or groups,
- yield the members of those groups, recursively.
+ yield the WantedFile instances for those names or the members of
+ those groups, recursively.
"""
for filename in grouped:
wanted = self.files.get(filename)
@@ -1156,7 +1172,7 @@ class GameData(object):
for x in self._iter_expand_groups(wanted.group_members):
yield x
else:
- yield filename
+ yield wanted
def construct_task(self, **kwargs):
self.load_file_data()
diff --git a/game_data_packager/build.py b/game_data_packager/build.py
index 0150045..78f1826 100644
--- a/game_data_packager/build.py
+++ b/game_data_packager/build.py
@@ -648,7 +648,7 @@ class PackagingTask(object):
if provider is None:
should_provide = set()
else:
- should_provide = set(provider.provides)
+ should_provide = set(provider.provides_files)
if stat.S_ISREG(st.st_mode):
self.consider_file(path, True)
@@ -667,7 +667,7 @@ class PackagingTask(object):
logger.warning('file "%s" does not exist or is not a file, ' +
'directory or CD block device', path)
- for missing in sorted(should_provide):
+ for missing in sorted(f.name for f in should_provide):
if missing not in self.found:
logger.error('%s should have provided %s but did not',
self.found[provider.name], missing)
@@ -680,16 +680,14 @@ class PackagingTask(object):
logger.debug('trying to fill any gaps for %s', package.name)
# this is redundant, it's only done to get the debug messages first
- for filename in package.install:
- if filename not in self.found:
- wanted = self.game.files[filename]
-
+ for wanted in package.install_files:
+ if wanted.name not in self.found:
for alt in wanted.alternatives:
if alt in self.found:
break
else:
logger.debug('gap needs to be filled for %s: %s',
- package.name, filename)
+ package.name, wanted.name)
result = FillResult.COMPLETE
@@ -697,25 +695,24 @@ class PackagingTask(object):
# to avoid extraneous downloads
unique_provider = list()
multi_provider = list()
- for filename in (package.install | package.optional):
- if len(self.game.providers.get(filename,[])) == 1:
- unique_provider.append(filename)
+ for wanted in (package.install_files | package.optional_files):
+ if len(self.game.providers.get(wanted.name,[])) == 1:
+ unique_provider.append(wanted)
else:
- multi_provider.append(filename)
+ multi_provider.append(wanted)
- for filename in unique_provider + multi_provider:
- if filename not in self.found:
- wanted = self.game.files[filename]
+ for wanted in unique_provider + multi_provider:
+ if wanted.name not in self.found:
# updates file_status as a side-effect
self.fill_gap(package, wanted, download=download,
recheck=recheck,
- log=(log and filename in package.install))
+ log=(log and wanted in package.install_files))
- logger.debug('%s: %s', filename, self.file_status[filename])
+ logger.debug('%s: %s', wanted.name, self.file_status[wanted.name])
- if filename in package.install:
+ if wanted in package.install_files:
# it is mandatory
- result &= self.file_status[filename]
+ result &= self.file_status[wanted.name]
self.package_status[package.name] = result
logger.debug('%s: %s', package.name, result)
@@ -727,7 +724,7 @@ class PackagingTask(object):
should_provide = set()
distinctive_dirs = False
else:
- try_to_unpack = provider.provides
+ try_to_unpack = set(f.name for f in provider.provides_files)
should_provide = set(try_to_unpack)
distinctive_dirs = provider.unpack.get('distinctive_dirs', True)
@@ -791,7 +788,7 @@ class PackagingTask(object):
try_to_unpack = self.game.files
should_provide = set()
else:
- try_to_unpack = provider.provides
+ try_to_unpack = set(f.name for f in provider.provides_files)
should_provide = set(try_to_unpack)
for entry in tar:
@@ -1079,7 +1076,8 @@ class PackagingTask(object):
with zipfile.ZipFile(found_name, 'r') as zf:
self.consider_zip(found_name, zf, provider)
elif fmt == 'lha':
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s',
to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
@@ -1091,7 +1089,8 @@ class PackagingTask(object):
cwd=tmpdir)
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt == 'id-shr-extract':
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s',
to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
@@ -1104,7 +1103,8 @@ class PackagingTask(object):
recursive_utime(tmpdir, os.stat(found_name).st_mtime)
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt == 'cabextract':
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s',
to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
@@ -1115,7 +1115,8 @@ class PackagingTask(object):
os.path.abspath(found_name)], cwd=tmpdir)
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt == 'unace-nonfree':
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s',
to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
@@ -1126,7 +1127,8 @@ class PackagingTask(object):
list(to_unpack), cwd=tmpdir)
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt == 'unrar-nonfree':
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s',
to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
@@ -1138,7 +1140,8 @@ class PackagingTask(object):
list(to_unpack), cwd=tmpdir)
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt == 'innoextract':
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s', to_unpack, found_name)
package.used_sources.add(provider.name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
@@ -1160,7 +1163,7 @@ class PackagingTask(object):
prefix += '/'
if '$provides' in to_unpack:
to_unpack.remove('$provides')
- to_unpack += provider.provides
+ to_unpack += [f.name for f in provider.provides_files]
for i in to_unpack:
cmdline.append('-I')
if prefix and i[0] != '/':
@@ -1170,7 +1173,8 @@ class PackagingTask(object):
check_call(cmdline)
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt == 'unzip' and which('unzip'):
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s',
to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
@@ -1184,7 +1188,8 @@ class PackagingTask(object):
# -C use case-insensitive matching
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt in ('7z', 'unzip'):
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s',
to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
@@ -1198,7 +1203,8 @@ class PackagingTask(object):
list(to_unpack), cwd=tmpdir)
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt in ('unar', 'unzip'):
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s', to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
provider_name + '.d')
@@ -1209,7 +1215,8 @@ class PackagingTask(object):
list(to_unpack), cwd=tmpdir)
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt == 'unshield':
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s', to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
provider_name + '.d')
@@ -1231,7 +1238,8 @@ class PackagingTask(object):
recursive_utime(tmpdir, os.stat(found_name).st_mtime)
self.consider_file_or_dir(tmpdir, provider=provider)
elif fmt == 'arj':
- to_unpack = provider.unpack.get('unpack', provider.provides)
+ to_unpack = provider.unpack.get('unpack',
+ [f.name for f in provider.provides_files])
logger.debug('Extracting %r from %s',
to_unpack, found_name)
tmpdir = os.path.join(self.get_workdir(), 'tmp',
@@ -1281,12 +1289,10 @@ class PackagingTask(object):
def check_complete(self, package, log=False):
# Got everything?
complete = True
- for filename in package.install:
- if filename in self.found:
+ for wanted in package.install_files:
+ if wanted.name in self.found:
continue
- wanted = self.game.files[filename]
-
for alt in wanted.alternatives:
if alt in self.found:
break
@@ -1331,12 +1337,12 @@ class PackagingTask(object):
'game-data-packager.\n' % package.name)
licenses = set()
- for f in package.install | package.optional:
- if self.file_status[f] is not FillResult.COMPLETE:
+ for f in (package.install_files | package.optional_files):
+ if self.file_status[f.name] is not FillResult.COMPLETE:
continue
- if not self.game.files[f].license:
+ if not f.license:
continue
- license_file = self.game.files[f].install_as
+ license_file = f.install_as
licenses.add("/usr/share/doc/%s/%s" % (package.name, license_file))
if os.path.splitext(license_file)[0].lower() == 'license':
lintian_license(destdir, package.name, license_file)
@@ -1364,17 +1370,17 @@ class PackagingTask(object):
count_usr = 0
exts = set()
count_doc = 0
- for f in package.install | package.optional:
- if self.file_status[f] is FillResult.IMPOSSIBLE:
+ for f in (package.install_files | package.optional_files):
+ if self.file_status[f.name] is FillResult.IMPOSSIBLE:
continue
- install_to = self.game.files[f].install_to
+ install_to = f.install_to
if install_to and install_to.startswith('$docdir'):
count_doc +=1
else:
count_usr +=1
# doesn't have to be a .wad, ROTT's EXTREME.RTL
# or any other one-datafile .deb would qualify too
- main_wad = self.game.files[f].install_as
+ main_wad = f.install_as
exts.add(os.path.splitext(main_wad.lower())[1])
if count_usr == 0 and count_doc == 1:
@@ -1480,12 +1486,11 @@ class PackagingTask(object):
self.fill_docs(package, destdir, docdir)
- for filename in (package.install | package.optional):
- wanted = self.game.files[filename]
+ for wanted in (package.install_files | package.optional_files):
install_as = wanted.install_as
- if filename in self.found:
- copy_from = self.found[filename]
+ if wanted.name in self.found:
+ copy_from = self.found[wanted.name]
else:
for alt in wanted.alternatives:
if alt in self.found:
@@ -1494,13 +1499,13 @@ class PackagingTask(object):
install_as = self.game.files[alt].install_as
break
else:
- if filename not in package.install:
+ if wanted not in package.install_files:
logger.debug('optional file %r is missing, ignoring',
- filename)
+ wanted.name)
continue
raise AssertionError('we already checked that %s exists' %
- (filename))
+ (wanted.name))
# cp it into place
with TemporaryUmask(0o22):
@@ -1899,8 +1904,11 @@ class PackagingTask(object):
self.consider_file_or_dir(arg)
def run_command_line(self, args):
- logger.debug('package description:\n%s',
- yaml.safe_dump(self.game.to_yaml()))
+ if logging.getLogger().isEnabledFor(logging.DEBUG):
+ logger.debug('package description:\n%s',
+ yaml.safe_dump(self.game.to_yaml(expand=False)))
+ logger.debug('package description after expansion:\n%s',
+ yaml.safe_dump(self.game.to_yaml(expand=True)))
self.verbose = getattr(args, 'verbose', False)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list