[game-data-packager] 01/03: Make it easier to mix "compiled" VFS bits with source YAML
Simon McVittie
smcv at debian.org
Sun Jul 23 13:42:43 UTC 2017
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit 86d0a9512d326bfbdbe4af6cd652642b513518c6
Author: Simon McVittie <smcv at debian.org>
Date: Mon Jul 17 00:39:03 2017 +0100
Make it easier to mix "compiled" VFS bits with source YAML
We seem to get occasional bug reports from people who have edited
the compiled/optimized versions of the data files. These are not
intended to be editable, and incorporating changes to them require
working out what change to the YAML was intended.
To make it easier for potential contributors to try out changes,
load YAML files from alongside vfs.zip if they exist.
---
game_data_packager/__init__.py | 100 ++++++++++++++++++++++++++++++++++++-----
game_data_packager/paths.py | 2 -
tools/check_equivalence.py | 8 ++--
3 files changed, 93 insertions(+), 17 deletions(-)
diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index 721914b..f802586 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -33,7 +33,7 @@ import yaml
from .build import (PackagingTask)
from .data import (FileGroup, Package, WantedFile)
from .packaging import (NoPackaging)
-from .paths import (DATADIR, USE_VFS)
+from .paths import (DATADIR)
from .util import ascii_safe
from .version import (GAME_PACKAGE_VERSION)
@@ -594,14 +594,78 @@ class GameData(object):
current_group.apply_group_attributes(f)
current_group.group_members.add(f.name)
- def load_file_data(self, use_vfs=USE_VFS,
- check=('GDP_UNINSTALLED' in os.environ)):
+ def load_file_data(self,
+ check=('GDP_UNINSTALLED' in os.environ),
+ datadir=DATADIR,
+ use_vfs=True):
if self.loaded_file_data:
return
logger.debug('loading full data')
- if use_vfs:
+ yamlfile = os.path.join(datadir, self.shortname + '.yaml')
+
+ if os.path.exists(yamlfile):
+ yamldata = open(yamlfile, encoding='utf-8').read()
+ data = yaml.load(yamldata, Loader=yaml.CSafeLoader)
+
+ for group_name, group_data in sorted(
+ data.get('groups', {}).items()):
+ group = self._ensure_group(group_name)
+
+ if isinstance(group_data, dict):
+ members = group_data['group_members']
+ for k, v in group_data.items():
+ if k != 'group_members':
+ setattr(group, k, v)
+ elif isinstance(group_data, (str, list)):
+ members = group_data
+ else:
+ raise AssertionError(
+ 'group %r should be dict, str or list' % group_name)
+
+ has_members = False
+
+ if isinstance(members, str):
+ for line in members.splitlines():
+ line = line.strip()
+ if line and not line.startswith('#'):
+ has_members = True
+ f = self._add_hash(line, 'size_and_md5')
+ # f can either be a WantedFile or a FileGroup here
+ group.apply_group_attributes(f)
+ group.group_members.add(f.name)
+ elif isinstance(members, list):
+ for m in members:
+ has_members = True
+ f = self._add_hash('? ? ' + m, 'size_and_md5')
+ # f can either be a WantedFile or a FileGroup here
+ group.apply_group_attributes(f)
+ group.group_members.add(f.name)
+ else:
+ raise AssertionError(
+ 'group %r members should be str or list' % group_name)
+
+ # an empty group is no use, and would break the assumption
+ # that we can use f.group_members to detect groups
+ assert has_members
+
+ for k in ('sha1sums', 'sha256sums', 'size_and_md5'):
+ v = data.get(k, None)
+
+ if k.endswith('sums'):
+ k = k[:-4]
+
+ if v is not None:
+ for line in v.splitlines():
+ stripped = line.strip()
+
+ if stripped == '' or stripped.startswith('#'):
+ continue
+
+ self._add_hash(stripped, k)
+
+ elif use_vfs:
if isinstance(use_vfs, str):
zip = use_vfs
else:
@@ -900,32 +964,42 @@ class GameData(object):
return
return gog.get('game') or gog.get('url')
-def load_games(game='*', use_vfs=USE_VFS, use_yaml=False):
+def load_games(game='*', datadir=DATADIR, use_vfs=True):
progress = (game == '*' and sys.stderr.isatty() and
not logger.isEnabledFor(logging.DEBUG))
games = {}
+ for yamlfile in glob.glob(os.path.join(datadir, game + '.yaml')):
+ if os.path.basename(yamlfile).startswith('launch-'):
+ continue
+
+ yamldata = open(yamlfile, encoding='utf-8').read()
+ load_game(progress, games, yamlfile, yamldata)
+
if use_vfs:
if isinstance(use_vfs, str):
zip = use_vfs
else:
- zip = os.path.join(DATADIR, 'vfs.zip')
+ zip = os.path.join(datadir, 'vfs.zip')
with zipfile.ZipFile(zip, 'r') as zf:
if game == '*':
for entry in zf.infolist():
- if entry.filename.split('.')[-1] == 'json':
+ if entry.filename.endswith('.json'):
+ if entry.filename[:-5] in games:
+ # shadowed by a YAML file
+ continue
+
jsonfile = '%s/%s' % (zip, entry.filename)
jsondata = zf.open(entry).read().decode('utf-8')
load_game(progress, games, jsonfile, jsondata)
+ elif game in games:
+ # shadowed by a YAML file
+ pass
else:
jsonfile = game + '.json'
jsondata = zf.open(jsonfile).read().decode('utf-8')
load_game(progress, games, '%s/%s' % (zip, jsonfile), jsondata)
- elif use_yaml:
- for yamlfile in glob.glob(os.path.join('data/', game + '.yaml')):
- yamldata = open(yamlfile, encoding='utf-8').read()
- load_game(progress, games, yamlfile, yamldata)
else:
vfs = os.path.join(DATADIR, 'vfs')
@@ -933,6 +1007,10 @@ def load_games(game='*', use_vfs=USE_VFS, use_yaml=False):
vfs = DATADIR
for jsonfile in glob.glob(os.path.join(vfs, game + '.json')):
+ if os.path.basename(jsonfile[:-5]) in games:
+ # shadowed by a YAML file
+ continue
+
jsondata = open(jsonfile, encoding='utf-8').read()
load_game(progress, games, jsonfile, jsondata)
diff --git a/game_data_packager/paths.py b/game_data_packager/paths.py
index 532cebd..838d565 100644
--- a/game_data_packager/paths.py
+++ b/game_data_packager/paths.py
@@ -7,7 +7,6 @@ if os.environ.get('GDP_UNINSTALLED'):
CONFIG = './etc/game-data-packager.conf'
DATADIR = './out'
ETCDIR = './etc'
- USE_VFS = bool(os.environ.get('GDP_USE_VFS'))
else:
CONFIG = '/etc/game-data-packager.conf'
if os.path.isdir('/usr/share/games/game-data-packager'):
@@ -15,4 +14,3 @@ else:
else:
DATADIR = '/usr/share/game-data-packager'
ETCDIR = '/etc/game-data-packager'
- USE_VFS = True
diff --git a/tools/check_equivalence.py b/tools/check_equivalence.py
index 171907b..33c0bd7 100755
--- a/tools/check_equivalence.py
+++ b/tools/check_equivalence.py
@@ -48,7 +48,7 @@ if __name__ == '__main__':
from_ref = None
t = time.process_time()
- from_vfs = load_games(games, use_vfs=True)
+ from_vfs = load_games(games)
dt = time.process_time() - t
print('# loaded game data from vfs.zip in %.3f seconds' % dt, flush=True)
@@ -58,7 +58,7 @@ if __name__ == '__main__':
print('# loaded game data from JSON in %.3f seconds' % dt, flush=True)
t = time.process_time()
- from_yaml = load_games(games, use_vfs=False, use_yaml=True)
+ from_yaml = load_games(games, datadir='data', use_vfs=False)
dt = time.process_time() - t
print('# loaded game data from YAML in %.3f seconds' % dt)
@@ -81,11 +81,11 @@ if __name__ == '__main__':
vfs_to_json = dump(game.to_data())
json_game = from_json[name]
- json_game.load_file_data(check=True)
+ json_game.load_file_data(check=True, use_vfs=False)
json_to_json = dump(json_game.to_data())
yaml_game = from_yaml[name]
- yaml_game.load_file_data(check=True)
+ yaml_game.load_file_data(check=True, datadir='data')
yaml_to_json = dump(yaml_game.to_data())
if yaml_to_json != vfs_to_json:
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list