[game-data-packager] 04/14: Add a test that various forms of the game data are equivalent
Simon McVittie
smcv at debian.org
Mon Nov 2 00:49:17 UTC 2015
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit 592d963f0cb5d0dcd30fcb52b472635981486e99
Author: Simon McVittie <smcv at debian.org>
Date: Sun Nov 1 15:09:35 2015 +0000
Add a test that various forms of the game data are equivalent
Also compare with ref.zip if it exists.
This provides a way for maintainers to determine whether a
"pure refactoring" change to the YAML has had any technical effect.
---
Makefile | 1 +
debian/changelog | 1 +
game_data_packager/__init__.py | 53 +++++++++++++------
tools/check_equivalence.py | 117 +++++++++++++++++++++++++++++++++++++++++
4 files changed, 155 insertions(+), 17 deletions(-)
diff --git a/Makefile b/Makefile
index 54adba7..54f25fb 100644
--- a/Makefile
+++ b/Makefile
@@ -106,6 +106,7 @@ clean:
check:
LC_ALL=C $(PYFLAKES3) game_data_packager/*.py game_data_packager/*/*.py runtime/*.py tools/*.py || :
LC_ALL=C GDP_UNINSTALLED=1 PYTHONPATH=. python3 tools/check_syntax.py
+ LC_ALL=C GDP_UNINSTALLED=1 PYTHONPATH=. python3 tools/check_equivalence.py
# Requires additional setup, so not part of "make check"
manual-check:
diff --git a/debian/changelog b/debian/changelog
index ff35637..f4305cd 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -17,6 +17,7 @@ game-data-packager (44) UNRELEASED; urgency=medium
* quake, quake2: make aliases consistently strings
* Always build vfs.zip, and optionally use it even when uninstalled
* to_yaml: turn sets into sorted lists, and output more fields
+ * Add a test that various forms of the game data are equivalent
-- Simon McVittie <smcv at debian.org> Sun, 01 Nov 2015 11:10:56 +0100
diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index 609c959..12f0c88 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -27,6 +27,8 @@ import re
import sys
import zipfile
+import yaml
+
from .build import (HashedFile,
PackagingTask)
from .config import read_config
@@ -848,19 +850,22 @@ class GameData(object):
if hexdigest is not None:
setattr(f, alg, hexdigest)
- def load_file_data(self):
+ def load_file_data(self, use_vfs=USE_VFS):
if self.loaded_file_data:
return
logger.debug('loading full data')
- if USE_VFS:
- zip = os.path.join(DATADIR, 'vfs.zip')
+ if use_vfs:
+ if isinstance(use_vfs, str):
+ zip = use_vfs
+ else:
+ zip = os.path.join(DATADIR, 'vfs.zip')
with zipfile.ZipFile(zip, 'r') as zf:
files = zf.namelist()
filename = '%s.files' % self.shortname
if filename in files:
- logger.debug('... vfs.zip/%s', filename)
+ logger.debug('... %s/%s', zip, filename)
jsondata = zf.open(filename).read().decode('utf-8')
data = json.loads(jsondata)
self._populate_files(data)
@@ -869,7 +874,7 @@ class GameData(object):
filename = '%s.%s%s' % (self.shortname, alg,
'' if alg == 'size_and_md5' else 'sums')
if filename in files:
- logger.debug('... vfs.zip/%s', filename)
+ logger.debug('... %s/%s', zip, filename)
rawdata = zf.open(filename).read().decode('utf-8')
for line in rawdata.splitlines():
self._add_hash(line.rstrip('\n'), alg)
@@ -1003,40 +1008,54 @@ class GameData(object):
return
return gog.get('game', gog['url'])
-def load_games(game='*'):
+def load_games(game='*', use_vfs=USE_VFS, use_yaml=False):
progress = (game == '*' and sys.stderr.isatty() and
not logging.getLogger().isEnabledFor(logging.DEBUG))
games = {}
- if USE_VFS:
- zip = os.path.join(DATADIR, 'vfs.zip')
+ if use_vfs:
+ if isinstance(use_vfs, str):
+ zip = use_vfs
+ else:
+ zip = os.path.join(DATADIR, 'vfs.zip')
+
with zipfile.ZipFile(zip, 'r') as zf:
if game == '*':
for entry in zf.infolist():
if entry.filename.split('.')[-1] == 'json':
- jsonfile = 'vfs.zip/' + entry.filename
+ jsonfile = '%s/%s' % (zip, entry.filename)
jsondata = zf.open(entry).read().decode('utf-8')
- load_json(progress, games, jsonfile, jsondata)
+ load_game(progress, games, jsonfile, jsondata)
else:
jsonfile = game + '.json'
jsondata = zf.open(jsonfile).read().decode('utf-8')
- load_json(progress, games, 'vfs.zip/' + jsonfile, jsondata)
+ load_game(progress, games, '%s/%s' % (zip, jsonfile), jsondata)
+ elif use_yaml:
+ for yamlfile in glob.glob(os.path.join('data/', game + '.yaml')):
+ yamldata = open(yamlfile, encoding='utf-8').read()
+ load_game(progress, games, yamlfile, yamldata)
else:
for jsonfile in glob.glob(os.path.join(DATADIR, game + '.json')):
jsondata = open(jsonfile, encoding='utf-8').read()
- load_json(progress, games, jsonfile, jsondata)
+ load_game(progress, games, jsonfile, jsondata)
+
+ if progress:
+ print('\r%s\r' % (' ' * len(games)), end='', flush=True, file=sys.stderr)
- print('\r%s\r' % (' ' * len(games)), end='', flush=True, file=sys.stderr)
return games
-def load_json(progress, games, jsonfile, jsondata):
+def load_game(progress, games, filename, content):
if progress:
print('.', end='', flush=True, file=sys.stderr)
try:
- g = os.path.basename(jsonfile)
+ g = os.path.basename(filename)
g = g[:len(g) - 5]
- data = json.loads(jsondata)
+ if filename.endswith('.yaml'):
+ data = yaml.load(content, Loader=yaml.CLoader)
+ else:
+ data = json.loads(content)
+
plugin = data.get('plugin', g)
plugin = plugin.replace('-', '_')
@@ -1051,7 +1070,7 @@ def load_json(progress, games, jsonfile, jsondata):
games[g] = game_data_constructor(g, data)
except:
- print('Error loading %s:\n' % jsonfile)
+ print('Error loading %s:\n' % filename)
raise
def run_command_line():
diff --git a/tools/check_equivalence.py b/tools/check_equivalence.py
new file mode 100755
index 0000000..2679bab
--- /dev/null
+++ b/tools/check_equivalence.py
@@ -0,0 +1,117 @@
+#!/usr/bin/python3
+# encoding=utf-8
+#
+# Copyright © 2015 Simon McVittie <smcv at debian.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# You can find the GPL license text on a Debian system under
+# /usr/share/common-licenses/GPL-2.
+
+import difflib
+import json
+import os
+import sys
+import time
+
+from game_data_packager import load_games
+from game_data_packager.util import ascii_safe
+
+def dump(serialized):
+ return json.dumps(serialized, sort_keys=True, indent=2)
+
+if __name__ == '__main__':
+ games = '*'
+
+ if len(sys.argv) > 1:
+ assert len(sys.argv) == 2
+ games = sys.argv[1]
+
+ if os.path.exists('ref.zip'):
+ t = time.process_time()
+ # usage:
+ # make
+ # cp out/vfs.zip ref.zip
+ # make
+ # make check
+ from_ref = load_games(games, use_vfs='ref.zip')
+ dt = time.process_time() - t
+ print('# loaded game data from ref.zip in %.3f seconds' % dt)
+ else:
+ from_ref = None
+
+ t = time.process_time()
+ from_vfs = load_games(games, use_vfs=True)
+ dt = time.process_time() - t
+ print('# loaded game data from vfs.zip in %.3f seconds' % dt)
+
+ t = time.process_time()
+ from_json = load_games(games, use_vfs=False)
+ dt = time.process_time() - t
+ print('# loaded game data from JSON in %.3f seconds' % dt)
+
+ t = time.process_time()
+ from_yaml = load_games(games, use_vfs=False, use_yaml=True)
+ dt = time.process_time() - t
+ print('# loaded game data from YAML in %.3f seconds' % dt)
+
+ assert set(from_vfs.keys()) == set(from_json.keys())
+ assert set(from_vfs.keys()) == set(from_yaml.keys())
+
+ if from_ref is not None:
+ assert set(from_vfs.keys()) == set(from_ref.keys())
+
+ fail = False
+
+ for (name, game) in sorted(from_vfs.items()):
+ print('# %s -----------------------------------------' % name)
+
+ game.load_file_data()
+ ascii_safe(game.longname, force=True).encode('ascii')
+ ascii_safe(game.help_text, force=True).encode('ascii')
+ vfs_to_json = dump(game.to_yaml())
+
+ json_game = from_json[name]
+ json_game.load_file_data()
+ json_to_json = dump(json_game.to_yaml())
+
+ yaml_game = from_yaml[name]
+ yaml_game.load_file_data()
+ yaml_to_json = dump(yaml_game.to_yaml())
+
+ if yaml_to_json != vfs_to_json:
+ sys.stdout.writelines(difflib.unified_diff(
+ yaml_to_json.splitlines(True),
+ vfs_to_json.splitlines(True),
+ '%s loaded from YAML' % name,
+ '%s loaded from vfs.zip' % name, n=50))
+ fail = True
+
+ if json_to_json != vfs_to_json:
+ sys.stdout.writelines(difflib.unified_diff(
+ json_to_json.splitlines(True),
+ vfs_to_json.splitlines(True),
+ '%s loaded from JSON' % name,
+ '%s loaded from vfs.zip' % name, n=50))
+ fail = True
+
+ if from_ref is not None:
+ ref_game = from_ref[name]
+ ref_game.load_file_data(use_vfs='ref.zip')
+ ref_to_json = dump(ref_game.to_yaml())
+
+ if ref_to_json != vfs_to_json:
+ sys.stdout.writelines(difflib.unified_diff(
+ ref_to_json.splitlines(True),
+ vfs_to_json.splitlines(True),
+ '%s loaded from ref.zip' % name,
+ '%s loaded from vfs.zip' % name, n=50))
+
+ raise SystemExit(fail)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list