[game-data-packager] 06/09: Split out detailed information about files to be packaged
Simon McVittie
smcv at debian.org
Thu Oct 1 10:16:50 UTC 2015
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit 971f77ca7ca9258817d6df50d90146cb57d0908c
Author: Simon McVittie <smcv at debian.org>
Date: Thu Oct 1 09:48:25 2015 +0100
Split out detailed information about files to be packaged
It is now in separate JSON and *sums files, which are loaded lazily.
This speeds up "game-data-packager --help" considerably.
(Continuation of #779937)
---
Makefile | 5 +
debian/changelog | 3 +
game_data_packager/__init__.py | 256 +++++++++++++++++++++++------------------
game_data_packager/build.py | 1 +
tools/check_syntax.py | 1 +
tools/yaml2json.py | 25 +++-
6 files changed, 177 insertions(+), 114 deletions(-)
diff --git a/Makefile b/Makefile
index 20eacfd..79506aa 100644
--- a/Makefile
+++ b/Makefile
@@ -68,12 +68,17 @@ clean:
rm -f ./out/changelog.gz
rm -f ./out/copyright
rm -f ./out/game-data-packager
+ rm -f ./out/*.cksums
rm -f ./out/*.control.in
rm -f ./out/*.copyright
rm -f ./out/*.copyright.in
rm -f ./out/*.desktop.in
+ rm -f ./out/*.files
+ rm -f ./out/*.md5sums
rm -f ./out/*.preinst.in
rm -f ./out/*.png
+ rm -f ./out/*.sha1sums
+ rm -f ./out/*.sha256sums
rm -f ./out/*.svgz
rm -f ./out/*.svg
rm -f ./out/*.json
diff --git a/debian/changelog b/debian/changelog
index af9d5d7..3f2bc53 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -56,6 +56,9 @@ game-data-packager (43) UNRELEASED; urgency=medium
* Move steam and gog modes to their own modules
* Separate GameData (static information about the game) from
PackagingTask (the actual game-data-packaging)
+ * Move detailed information about files to be packaged into separate JSON
+ and *sums files, which are loaded lazily. This speeds up
+ "game-data-packager --help" considerably. (Continuation of #779937)
-- Simon McVittie <smcv at debian.org> Thu, 16 Jul 2015 09:59:23 +0200
diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index 1e5a57e..4b71cee 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -366,6 +366,9 @@ class GameData(object):
assert 'symlinks' not in self.data
assert 'install_files_from_cksums' not in self.data
+ # True if the lazy load of full file info has been done
+ self.loaded_file_data = False
+
# Map from WantedFile name to instance.
# { 'baseq3/pak1.pk3': WantedFile instance }
self.files = {}
@@ -408,124 +411,14 @@ class GameData(object):
self.packages[binary] = package
self._populate_package(package, data)
- if 'cksums' in self.data:
- for line in self.data['cksums'].splitlines():
- stripped = line.strip()
- if stripped == '' or stripped.startswith('#'):
- continue
-
- _, size, filename = line.split(None, 2)
- f = self._ensure_file(filename)
- f.size = int(size)
-
if 'size_and_md5' in self.data:
for line in self.data['size_and_md5'].splitlines():
- stripped = line.strip()
- if stripped == '' or stripped.startswith('#'):
- continue
-
- size, md5, filename = line.split(None, 2)
- f = self._ensure_file(filename)
- f.size = int(size)
- f.md5 = md5
+ self._add_hash(line, 'size_and_md5')
- for alg in ('md5', 'sha1', 'sha256'):
+ for alg in ('ck', 'md5', 'sha1', 'sha256'):
if alg + 'sums' in self.data:
for line in self.data[alg + 'sums'].splitlines():
- stripped = line.strip()
- if stripped == '' or stripped.startswith('#'):
- continue
-
- hexdigest, filename = MD5SUM_DIVIDER.split(line, 1)
- f = self._ensure_file(filename)
- setattr(f, alg, hexdigest)
-
- for filename, f in self.files.items():
- for provided in f.provides:
- self.providers.setdefault(provided, set()).add(filename)
-
- if f.alternatives:
- continue
-
- if f.distinctive_size and f.size is not None:
- self.known_sizes.setdefault(f.size, set()).add(filename)
-
- for lf in f.look_for:
- self.known_filenames.setdefault(lf, set()).add(filename)
-
- if f.md5 is not None:
- self.known_md5s.setdefault(f.md5, set()).add(filename)
-
- if f.sha1 is not None:
- self.known_sha1s.setdefault(f.sha1, set()).add(filename)
-
- if f.sha256 is not None:
- self.known_sha256s.setdefault(f.sha256, set()).add(filename)
-
- # consistency check
- for package in self.packages.values():
- for provider in package.install_contents_of:
- assert provider in self.files, (package.name, provider)
- for filename in self.files[provider].provides:
- assert filename in self.files, (package.name, provider,
- filename)
- if filename not in package.optional:
- package.install.add(filename)
-
- if package.rip_cd:
- # we only support Ogg Vorbis for now
- assert package.rip_cd['encoding'] == 'vorbis', package.name
- self.rip_cd_packages.add(package)
-
- # there had better be something it wants to install
- assert package.install or package.rip_cd, package.name
- for installable in package.install:
- assert installable in self.files, installable
- for installable in package.optional:
- assert installable in self.files, installable
-
- # check internal depedencies
- for demo_for_item in package.demo_for:
- assert demo_for_item in self.packages, demo_for_item
- assert (not package.expansion_for or
- package.expansion_for in self.packages), package.expansion_for
- assert (not package.better_version or
- package.better_version in self.packages), package.better_version
-
- # check for stale missing_langs
- if not package.demo_for:
- assert not set(package.langs).intersection(self.missing_langs)
-
- for filename, wanted in self.files.items():
- if wanted.unpack:
- assert 'format' in wanted.unpack, filename
- assert wanted.provides, filename
- if wanted.unpack['format'] == 'cat':
- assert len(wanted.provides) == 1, filename
- assert isinstance(wanted.unpack['other_parts'],
- list), filename
- if 'include' in wanted.unpack:
- assert isinstance(wanted.unpack['include'],
- list), filename
-
- if wanted.alternatives:
- for alt in wanted.alternatives:
- assert alt in self.files, alt
-
- # if this is a placeholder for a bunch of alternatives, then
- # it doesn't make sense for it to have a defined checksum
- # or size
- assert wanted.md5 is None, wanted.name
- assert wanted.sha1 is None, wanted.name
- assert wanted.sha256 is None, wanted.name
- assert wanted.size is None, wanted.name
- # FIXME: find out file size and add to yaml
- else:
- assert wanted.size is not None or filename in (
- 'hipnotic/pak0.pak_qdq_glquake_compat',
- 'resource.1_106_cd',
- 'vox0000.lab_unpatched',
- ), (self.shortname, wanted.name)
+ self._add_hash(line, alg)
# compute webshop URL's
gog_url = self.gog.get('url')
@@ -792,7 +685,144 @@ class GameData(object):
self.argument_parser = parser
return parser
+ def _add_hash(self, line, alg):
+ """Parse one line from md5sums-style data."""
+
+ stripped = line.strip()
+ if stripped == '' or stripped.startswith('#'):
+ return
+
+ if alg == 'ck':
+ _, size, filename = line.split(None, 2)
+ hexdigest = None
+ elif alg == 'size_and_md5':
+ size, hexdigest, filename = line.split(None, 2)
+ alg = 'md5'
+ else:
+ size = None
+ hexdigest, filename = MD5SUM_DIVIDER.split(line, 1)
+
+ f = self._ensure_file(filename)
+
+ if size is not None:
+ f.size = int(size)
+
+ if hexdigest is not None:
+ setattr(f, alg, hexdigest)
+
+ def load_file_data(self):
+ if self.loaded_file_data:
+ return
+
+ logger.debug('loading full data')
+
+ filename = os.path.join(DATADIR, '%s.files' % self.shortname)
+ if os.path.isfile(filename):
+ logger.debug('... %s', filename)
+ data = json.load(open(filename, encoding='utf-8'))
+ self._populate_files(data)
+
+ for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'):
+ filename = os.path.join(DATADIR, '%s.%s%s' %
+ (self.shortname, alg,
+ '' if alg == 'size_and_md5' else 'sums'))
+ if os.path.isfile(filename):
+ logger.debug('... %s', filename)
+ with open(filename) as f:
+ for line in f:
+ self._add_hash(line.rstrip('\n'), alg)
+
+ self.loaded_file_data = True
+
+ for filename, f in self.files.items():
+ for provided in f.provides:
+ self.providers.setdefault(provided, set()).add(filename)
+
+ if f.alternatives:
+ continue
+
+ if f.distinctive_size and f.size is not None:
+ self.known_sizes.setdefault(f.size, set()).add(filename)
+
+ for lf in f.look_for:
+ self.known_filenames.setdefault(lf, set()).add(filename)
+
+ if f.md5 is not None:
+ self.known_md5s.setdefault(f.md5, set()).add(filename)
+
+ if f.sha1 is not None:
+ self.known_sha1s.setdefault(f.sha1, set()).add(filename)
+
+ if f.sha256 is not None:
+ self.known_sha256s.setdefault(f.sha256, set()).add(filename)
+
+ # consistency check
+ for package in self.packages.values():
+ for provider in package.install_contents_of:
+ assert provider in self.files, (package.name, provider)
+ for filename in self.files[provider].provides:
+ assert filename in self.files, (package.name, provider,
+ filename)
+ if filename not in package.optional:
+ package.install.add(filename)
+
+ if package.rip_cd:
+ # we only support Ogg Vorbis for now
+ assert package.rip_cd['encoding'] == 'vorbis', package.name
+ self.rip_cd_packages.add(package)
+
+ # there had better be something it wants to install
+ assert package.install or package.rip_cd, package.name
+ for installable in package.install:
+ assert installable in self.files, installable
+ for installable in package.optional:
+ assert installable in self.files, installable
+
+ # check internal depedencies
+ for demo_for_item in package.demo_for:
+ assert demo_for_item in self.packages, demo_for_item
+ assert (not package.expansion_for or
+ package.expansion_for in self.packages), package.expansion_for
+ assert (not package.better_version or
+ package.better_version in self.packages), package.better_version
+
+ # check for stale missing_langs
+ if not package.demo_for:
+ assert not set(package.langs).intersection(self.missing_langs)
+
+ for filename, wanted in self.files.items():
+ if wanted.unpack:
+ assert 'format' in wanted.unpack, filename
+ assert wanted.provides, filename
+ if wanted.unpack['format'] == 'cat':
+ assert len(wanted.provides) == 1, filename
+ assert isinstance(wanted.unpack['other_parts'],
+ list), filename
+ if 'include' in wanted.unpack:
+ assert isinstance(wanted.unpack['include'],
+ list), filename
+
+ if wanted.alternatives:
+ for alt in wanted.alternatives:
+ assert alt in self.files, alt
+
+ # if this is a placeholder for a bunch of alternatives, then
+ # it doesn't make sense for it to have a defined checksum
+ # or size
+ assert wanted.md5 is None, wanted.name
+ assert wanted.sha1 is None, wanted.name
+ assert wanted.sha256 is None, wanted.name
+ assert wanted.size is None, wanted.name
+ # FIXME: find out file size and add to yaml
+ else:
+ assert wanted.size is not None or filename in (
+ 'hipnotic/pak0.pak_qdq_glquake_compat',
+ 'resource.1_106_cd',
+ 'vox0000.lab_unpatched',
+ ), (self.shortname, wanted.name)
+
def construct_task(self, **kwargs):
+ self.load_file_data()
return PackagingTask(self, **kwargs)
def construct_package(self, binary):
diff --git a/game_data_packager/build.py b/game_data_packager/build.py
index 9af3e2a..c9190e5 100644
--- a/game_data_packager/build.py
+++ b/game_data_packager/build.py
@@ -323,6 +323,7 @@ class PackagingTask(object):
def __init__(self, game):
# A GameData object.
self.game = game
+ self.game.load_file_data()
# A temporary directory.
self.__workdir = None
diff --git a/tools/check_syntax.py b/tools/check_syntax.py
index 7428f1e..397ae05 100755
--- a/tools/check_syntax.py
+++ b/tools/check_syntax.py
@@ -23,6 +23,7 @@ from game_data_packager.util import ascii_safe
if __name__ == '__main__':
for name, game in load_games().items():
+ game.load_file_data()
ascii_safe(game.longname, force=True).encode('ascii')
ascii_safe(game.help_text, force=True).encode('ascii')
if 'DEBUG' in os.environ:
diff --git a/tools/yaml2json.py b/tools/yaml2json.py
index 513b615..b3b6305 100755
--- a/tools/yaml2json.py
+++ b/tools/yaml2json.py
@@ -23,13 +23,36 @@ import yaml
def main(f, out):
data = yaml.load(open(f, encoding='utf-8'), Loader=yaml.CLoader)
- game = os.path.splitext(os.path.basename(f))[0]
+ game = f[5:].split('.')[0]
+
with open('data/wikipedia.csv', 'r', encoding='utf8') as csv:
for line in csv.readlines():
shortname, url = line.strip().split(';', 1)
if shortname == game:
data['wikipedia'] = url
break
+
+ v = data.pop('files', None)
+ if v is not None:
+ offload = os.path.splitext(out)[0] + '.files'
+ json.dump(v, open(offload + '.tmp', 'w', encoding='utf-8'), sort_keys=True)
+ os.rename(offload + '.tmp', offload)
+
+ for k in ('cksums', 'sha1sums', 'sha256sums', 'md5sums',
+ 'size_and_md5'):
+ v = data.pop(k, None)
+
+ if v is not None:
+ offload = os.path.splitext(out)[0] + '.' + k
+ with open(offload + '.tmp', 'w', encoding='utf-8') as writer:
+ for line in v.splitlines():
+ stripped = line.strip()
+ if stripped == '' or stripped.startswith('#'):
+ continue
+ writer.write(line)
+ writer.write('\n')
+ os.rename(offload + '.tmp', offload)
+
json.dump(data, open(out + '.tmp', 'w', encoding='utf-8'), sort_keys=True)
os.rename(out + '.tmp', out)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list