[game-data-packager] 06/14: Add support for named groups of files
Simon McVittie
smcv at debian.org
Mon Nov 2 00:49:18 UTC 2015
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit bdff2960e4c311e72edcfd9d459a80f658c0852a
Author: Simon McVittie <smcv at debian.org>
Date: Sun Nov 1 20:45:07 2015 +0000
Add support for named groups of files
---
Makefile | 3 +-
data/quake2.yaml | 54 ++++++---------
game_data_packager/__init__.py | 145 +++++++++++++++++++++++++++++++++++++++--
tools/yaml2json.py | 49 ++++++++++++++
4 files changed, 209 insertions(+), 42 deletions(-)
diff --git a/Makefile b/Makefile
index 54f25fb..f4c0043 100644
--- a/Makefile
+++ b/Makefile
@@ -37,7 +37,7 @@ out/vfs.zip: $(json)
rm -fr out/vfs
mkdir out/vfs
cp out/*.json out/*.files out/*.size_and_md5 out/*.cksums out/vfs/
- cp out/*.md5sums out/*.sha1sums out/*.sha256sums out/vfs/
+ cp out/*.md5sums out/*.sha1sums out/*.sha256sums out/*.groups out/vfs/
if [ -n "$(BUILD_DATE)" ]; then \
touch --date='$(BUILD_DATE)' out/vfs/*; \
fi
@@ -86,6 +86,7 @@ clean:
rm -f ./out/*.copyright
rm -f ./out/*.copyright.in
rm -f ./out/*.files
+ rm -f ./out/*.groups
rm -f ./out/*.md5sums
rm -f ./out/*.preinst.in
rm -f ./out/*.png
diff --git a/data/quake2.yaml b/data/quake2.yaml
index dd4e389..bc4400e 100644
--- a/data/quake2.yaml
+++ b/data/quake2.yaml
@@ -49,19 +49,7 @@ packages:
install:
- baseq2/pak0.pak
optional:
- # videos present in Steam and on smcv's Xplosiv-branded CD-ROM, but
- # apparently not strictly necessary (#776059)
- - baseq2/video/end.cin
- - baseq2/video/eou1_.cin
- - baseq2/video/eou2_.cin
- - baseq2/video/eou3_.cin
- - baseq2/video/eou4_.cin
- - baseq2/video/eou5_.cin
- - baseq2/video/eou6_.cin
- - baseq2/video/eou7_.cin
- - baseq2/video/eou8_.cin
- - baseq2/video/idlog.cin
- - baseq2/video/ntro.cin
+ - baseq2 videos
# semi-official bonus content
- baseq2/maps/base64.bsp
- baseq2/maps/city64.bsp
@@ -1562,17 +1550,7 @@ files:
- baseq2/players/zumlin/zumlin_i.pcx
- baseq2/players/zumlin/zumlinbase.pcx
- baseq2/players/zumlin/zumlinstory.txt
- - baseq2/video/end.cin
- - baseq2/video/eou1_.cin
- - baseq2/video/eou2_.cin
- - baseq2/video/eou3_.cin
- - baseq2/video/eou4_.cin
- - baseq2/video/eou5_.cin
- - baseq2/video/eou6_.cin
- - baseq2/video/eou7_.cin
- - baseq2/video/eou8_.cin
- - baseq2/video/idlog.cin
- - baseq2/video/ntro.cin
+ - baseq2 videos
- ctf/pak0.pak
- ctf/readme.txt
- ctf/server.cfg
@@ -1751,6 +1729,22 @@ files:
# + action, arena, capture, chaos, eraser, jail, kick, pb2 & rover:
# -> can not supported without source for a native game.so
+groups:
+ baseq2 videos: |
+ # videos present in Steam and on smcv's Xplosiv-branded CD-ROM, but
+ # apparently not strictly necessary (#776059)
+ 19311290 36fdaddd1c1b56ba10472466e4486ff8 baseq2/video/end.cin
+ 6038210 cbab517cddc03ec676d7153eeb13417b baseq2/video/eou1_.cin
+ 6586381 37201aa9c798982e7739dfdbad61c004 baseq2/video/eou2_.cin
+ 6708418 f2b7fac58d5aa24bcf76ec942253dfcc baseq2/video/eou3_.cin
+ 5464002 b0e360a5c4a55789e00d3f2223e23dc4 baseq2/video/eou4_.cin
+ 6848133 76651dc4f2d92ed278ee8e2cac7c7600 baseq2/video/eou5_.cin
+ 7099332 0b14fff288b8d260f812e830ce6bff7c baseq2/video/eou6_.cin
+ 7766671 a342a79fadfe9efa2ce549842168ec7c baseq2/video/eou7_.cin
+ 11875656 24a5178220322b3a4fb42f6b0757b821 baseq2/video/eou8_.cin
+ 3159828 0747670d94cc873f8ce522d7652143a1 baseq2/video/idlog.cin
+ 82836235 72846e547415856028006aaa4089c9c9 baseq2/video/ntro.cin
+
cksums: |
1206005048 239924 quake2-rogue-2.02.tar.xz
285620057 1421 quake2-rogue-2.02/CHANGELOG
@@ -1856,17 +1850,7 @@ cksums: |
size_and_md5: |
183997730 1ec55a724dc3109fd50dde71ab581d70 baseq2/pak0.pak?1dd586a
- 19311290 36fdaddd1c1b56ba10472466e4486ff8 baseq2/video/end.cin
- 6038210 cbab517cddc03ec676d7153eeb13417b baseq2/video/eou1_.cin
- 6586381 37201aa9c798982e7739dfdbad61c004 baseq2/video/eou2_.cin
- 6708418 f2b7fac58d5aa24bcf76ec942253dfcc baseq2/video/eou3_.cin
- 5464002 b0e360a5c4a55789e00d3f2223e23dc4 baseq2/video/eou4_.cin
- 6848133 76651dc4f2d92ed278ee8e2cac7c7600 baseq2/video/eou5_.cin
- 7099332 0b14fff288b8d260f812e830ce6bff7c baseq2/video/eou6_.cin
- 7766671 a342a79fadfe9efa2ce549842168ec7c baseq2/video/eou7_.cin
- 11875656 24a5178220322b3a4fb42f6b0757b821 baseq2/video/eou8_.cin
- 3159828 0747670d94cc873f8ce522d7652143a1 baseq2/video/idlog.cin
- 82836235 72846e547415856028006aaa4089c9c9 baseq2/video/ntro.cin
+
# Files from patch to install in game directory
1063 b2c3358d7be61f05651c88d9ef97d6aa baseq2/maps.lst
12992754 42663ea709b7cd3eb9b634b36cfecb1a baseq2/pak1.pak
diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index dd7a293..40c8695 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -19,6 +19,7 @@
import argparse
import glob
import importlib
+import io
import json
import logging
import os
@@ -54,6 +55,7 @@ class WantedFile(HashedFile):
def __init__(self, name):
super(WantedFile, self).__init__(name)
self.alternatives = []
+ self.group_members = set()
self.distinctive_name = True
self.distinctive_size = False
self.download = None
@@ -68,6 +70,22 @@ class WantedFile(HashedFile):
self.unpack = None
self.unsuitable = None
+ def apply_group_attributes(self, attributes):
+ for k, v in attributes.items():
+ if k == 'doc':
+ if v:
+ self.install_to = '$docdir'
+ continue
+
+ if k == 'license' and v:
+ self.install_to = '$docdir'
+ self.distinctive_name = False
+ self.license = v
+ continue
+
+ assert hasattr(self, k)
+ setattr(self, k, v)
+
@property
def look_for(self):
if self.alternatives:
@@ -107,6 +125,7 @@ class WantedFile(HashedFile):
'alternatives',
'distinctive_size',
'executable',
+ 'group_members',
'license',
'look_for',
'provides',
@@ -497,6 +516,42 @@ class GameData(object):
self.packages[binary] = package
self._populate_package(package, data)
+ if 'groups' in self.data:
+ groups = self.data['groups']
+ assert isinstance(groups, dict), self.shortname
+ for group_name, group_data in groups.items():
+ attrs = {}
+ if isinstance(group_data, dict):
+ members = group_data['group_members']
+ for k, v in group_data.items():
+ if k != 'group_members':
+ attrs[k] = v
+ elif isinstance(group_data, (str, list)):
+ members = group_data
+ else:
+ raise AssertionError('group %r should be dict, str or list' % group_name)
+
+ group = self._ensure_file(group_name)
+ group.apply_group_attributes(attrs)
+
+ if isinstance(members, str):
+ for line in members.splitlines():
+ f = self._add_hash(line.rstrip('\n'), 'size_and_md5')
+ if f is not None:
+ f.apply_group_attributes(attrs)
+ group.group_members.add(f.name)
+ elif isinstance(members, list):
+ for member_name in members:
+ f = self._ensure_file(member_name)
+ f.apply_group_attributes(attrs)
+ group.group_members.add(member_name)
+ else:
+ raise AssertionError('group %r members should be str or list' % group_name)
+
+ # an empty group is no use, and would break the assumption
+ # that we can use f.group_members to detect groups
+ assert group.group_members
+
if 'size_and_md5' in self.data:
for line in self.data['size_and_md5'].splitlines():
self._add_hash(line, 'size_and_md5')
@@ -606,6 +661,7 @@ class GameData(object):
def to_yaml(self):
files = {}
+ groups = {}
packages = {}
def sort_set_values(d):
@@ -615,7 +671,10 @@ class GameData(object):
ret[k] = sorted(v)
for filename, f in self.files.items():
- files[filename] = f.to_yaml()
+ if f.group_members:
+ groups[filename] = f.to_yaml()
+ else:
+ files[filename] = f.to_yaml()
for name, package in self.packages.items():
packages[name] = package.to_yaml()
@@ -631,6 +690,7 @@ class GameData(object):
'packages': packages,
'providers': sort_set_values(self.providers),
'files': files,
+ 'groups': groups,
}
def size(self, package):
@@ -843,12 +903,38 @@ class GameData(object):
f = self._ensure_file(filename)
- if size is not None:
+ if size is not None and size != '_':
f.size = int(size)
- if hexdigest is not None:
+ if hexdigest is not None and hexdigest != '_':
setattr(f, alg, hexdigest)
+ return f
+
+ def _populate_groups(self, stream):
+ current_group = None
+ attributes = {}
+
+ for line in stream:
+ stripped = line.strip()
+ if stripped == '' or stripped.startswith('#'):
+ continue
+
+ if stripped.startswith('['):
+ assert stripped.endswith(']'), repr(stripped)
+ current_group = self._ensure_file(stripped[1:-1])
+ attributes = {}
+ elif stripped.startswith('{'):
+ attributes = {}
+ attributes = json.loads(stripped)
+ assert current_group is not None
+ current_group.apply_group_attributes(attributes)
+ else:
+ f = self._add_hash(stripped, 'size_and_md5')
+ f.apply_group_attributes(attributes)
+ assert current_group is not None
+ current_group.group_members.add(f.name)
+
def load_file_data(self, use_vfs=USE_VFS):
if self.loaded_file_data:
return
@@ -869,6 +955,12 @@ class GameData(object):
data = json.loads(jsondata)
self._populate_files(data)
+ filename = '%s.groups' % self.shortname
+ if filename in files:
+ logger.debug('... %s/%s', zip, filename)
+ stream = io.TextIOWrapper(zf.open(filename), encoding='utf-8')
+ self._populate_groups(stream)
+
for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'):
filename = '%s.%s%s' % (self.shortname, alg,
'' if alg == 'size_and_md5' else 'sums')
@@ -884,6 +976,12 @@ class GameData(object):
data = json.load(open(filename, encoding='utf-8'))
self._populate_files(data)
+ filename = os.path.join(DATADIR, '%s.groups' % self.shortname)
+ if os.path.isfile(filename):
+ logger.debug('... %s', filename)
+ stream = open(filename, encoding='utf-8')
+ self._populate_groups(stream)
+
for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'):
filename = os.path.join(DATADIR, '%s.%s%s' %
(self.shortname, alg,
@@ -902,11 +1000,16 @@ class GameData(object):
if filename not in package.optional:
package.install.add(filename)
+ package.install = set(self._iter_expand_groups(package.install))
+ package.optional = set(self._iter_expand_groups(package.optional))
+
for filename, f in self.files.items():
+ f.provides = set(self._iter_expand_groups(f.provides))
+
for provided in f.provides:
self.providers.setdefault(provided, set()).add(filename)
- if f.alternatives:
+ if f.alternatives or f.group_members:
continue
if f.distinctive_size and f.size is not None:
@@ -982,8 +1085,12 @@ class GameData(object):
list), filename
if wanted.alternatives:
- for alt in wanted.alternatives:
- assert alt in self.files, alt
+ for alt_name in wanted.alternatives:
+ alt = self.files[alt_name]
+ # an alternative can't be a placeholder for alternatives
+ assert not alt.alternatives, alt_name
+ # an alternative can't be a placeholder for a group
+ assert not alt.group_members, alt_name
# if this is a placeholder for a bunch of alternatives, then
# it doesn't make sense for it to have a defined checksum
@@ -992,12 +1099,38 @@ class GameData(object):
assert wanted.sha1 is None, wanted.name
assert wanted.sha256 is None, wanted.name
assert wanted.size is None, wanted.name
+
+ # a placeholder for alternatives can't also be a placeholder
+ # for a group
+ assert not wanted.group_members, wanted.name
+ elif wanted.group_members:
+ for member_name in wanted.group_members:
+ assert member_name in self.files
+
+ assert wanted.md5 is None, wanted.name
+ assert wanted.sha1 is None, wanted.name
+ assert wanted.sha256 is None, wanted.name
+ assert wanted.size is None, wanted.name
+ assert not wanted.unpack, wanted.unpack
# FIXME: find out file size and add to yaml
else:
assert (wanted.size is not None or filename in
self.data.get('unknown_sizes', ())
), (self.shortname, wanted.name)
+ def _iter_expand_groups(self, grouped):
+ """Given a set of strings that are either filenames or groups,
+ yield the members of those groups, recursively.
+ """
+ for filename in grouped:
+ wanted = self.files.get(filename)
+ assert wanted is not None, filename
+ if wanted.group_members:
+ for x in self._iter_expand_groups(wanted.group_members):
+ yield x
+ else:
+ yield filename
+
def construct_task(self, **kwargs):
self.load_file_data()
return PackagingTask(self, **kwargs)
diff --git a/tools/yaml2json.py b/tools/yaml2json.py
index 8c0d1c0..a0efc88 100755
--- a/tools/yaml2json.py
+++ b/tools/yaml2json.py
@@ -40,6 +40,55 @@ def main(f, out):
elif os.path.isfile(offload):
os.remove(offload)
+ groups = data.pop('groups', None)
+ offload = os.path.splitext(out)[0] + '.groups'
+
+ if groups is not None:
+ with open(offload + '.tmp', 'w', encoding='utf-8') as writer:
+ assert isinstance(groups, dict)
+ for group_name, group_data in groups.items():
+ writer.write('[%s]\n' % group_name)
+
+ if isinstance(group_data, dict):
+ attrs = {}
+ members = group_data['group_members']
+ for k, v in group_data.items():
+ if k != 'group_members':
+ attrs[k] = v
+ if attrs:
+ json.dump(attrs, writer, sort_keys=True)
+ writer.write('\n')
+ elif isinstance(group_data, (str, list)):
+ members = group_data
+ else:
+ raise AssertionError('group %r should be dict, str or list' % group_name)
+
+ has_members = False
+
+ if isinstance(members, str):
+ for line in members.splitlines():
+ assert not line.startswith('[')
+ assert not line.startswith('{')
+ line = line.strip()
+ if line and not line.startswith('#'):
+ has_members = True
+ writer.write(' '.join(line.split()))
+ writer.write('\n')
+ elif isinstance(members, list):
+ for m in members:
+ has_members = True
+ writer.write('? ? %s\n' % m)
+ else:
+ raise AssertionError('group %r members should be str or list' % group_name)
+
+ # an empty group is no use, and would break the assumption
+ # that we can use f.group_members to detect groups
+ assert has_members
+
+ os.rename(offload + '.tmp', offload)
+ elif os.path.isfile(offload):
+ os.remove(offload)
+
for k in ('cksums', 'sha1sums', 'sha256sums', 'md5sums',
'size_and_md5'):
v = data.pop(k, None)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list