[game-data-packager] 02/02: store all game information in a .zip
Alexandre Detiste
detiste-guest at moszumanska.debian.org
Sun Oct 18 08:55:55 UTC 2015
This is an automated email from the git hooks/post-receive script.
detiste-guest pushed a commit to branch master
in repository game-data-packager.
commit 2254ab8a114054e9bcb4b938cfadfb7e26c9a93b
Author: Alexandre Detiste <alexandre.detiste at gmail.com>
Date: Sun Oct 18 10:55:22 2015 +0200
store all game information in a .zip
---
Makefile | 8 +++-
debian/game-data-packager.install | 8 +---
debian/rules | 4 ++
game_data_packager/__init__.py | 80 +++++++++++++++++++++++++++++----------
game_data_packager/paths.py | 2 +
5 files changed, 74 insertions(+), 28 deletions(-)
diff --git a/Makefile b/Makefile
index ee64566..531a3f0 100644
--- a/Makefile
+++ b/Makefile
@@ -31,6 +31,11 @@ out/%: data/%
out/%.json: data/%.yaml
python3 tools/yaml2json.py $< $@
+out/vfs.zip:
+ rm -f out/vfs.zip
+ find out -regex '.*\.\(json\|files\|size_and_md5\|cksums\|md5sums\|sha1sums\|sha256sums\)' \
+ | LC_ALL=C sort | TZ=UTC zip out/vfs.zip -X -j -q -@
+
out/bash_completion: $(in_yaml)
python3 tools/bash_completion.py > ./out/bash_completion
chmod 0644 ./out/bash_completion
@@ -82,6 +87,7 @@ clean:
rm -f ./out/*.svgz
rm -f ./out/*.svg
rm -f ./out/*.json
+ rm -f ./out/vfs.zip
rm -f ./out/index.html
rm -rf game_data_packager/__pycache__
rm -rf game_data_packager/games/__pycache__
@@ -105,4 +111,4 @@ html: $(DIRS) $(json)
LC_ALL=C GDP_UNINSTALLED=1 PYTHONPATH=. python3 -m tools.babel
rsync out/index.html alioth.debian.org:/var/lib/gforge/chroot/home/groups/pkg-games/htdocs/game-data/ -e ssh -v
-.PHONY: default clean check manual-check html
+.PHONY: default clean check manual-check html out/vfs.zip
diff --git a/debian/game-data-packager.install b/debian/game-data-packager.install
index 9874a25..763195d 100644
--- a/debian/game-data-packager.install
+++ b/debian/game-data-packager.install
@@ -3,20 +3,14 @@ data/bash-completion/game-data-packager usr/share/bash-completion/completions
etc/game-data-packager.conf etc
etc/*-mirrors etc/game-data-packager
game_data_packager usr/share/games/game-data-packager
-out/*.cksums usr/share/games/game-data-packager
out/*.control.in usr/share/games/game-data-packager
out/*.copyright usr/share/games/game-data-packager
out/*.copyright.in usr/share/games/game-data-packager
-out/*.files usr/share/games/game-data-packager
-out/*.md5sums usr/share/games/game-data-packager
out/*.png usr/share/games/game-data-packager
-out/*.sha1sums usr/share/games/game-data-packager
-out/*.sha256sums usr/share/games/game-data-packager
-out/*.size_and_md5 usr/share/games/game-data-packager
out/*.svgz usr/share/games/game-data-packager
out/*.preinst.in usr/share/games/game-data-packager
out/bash_completion usr/share/games/game-data-packager
out/changelog.gz usr/share/games/game-data-packager
out/copyright usr/share/games/game-data-packager
-out/*.json usr/share/games/game-data-packager
+out/vfs.zip usr/share/games/game-data-packager
runtime/doom2-masterlevels.desktop usr/share/applications
diff --git a/debian/rules b/debian/rules
index b0a76ad..18fdeda 100755
--- a/debian/rules
+++ b/debian/rules
@@ -5,12 +5,16 @@
export DH_OPTIONS
include /usr/share/dpkg/pkg-info.mk
+BUILD_DATE := $(shell dpkg-parsechangelog --show-field=Date)
%:
dh $@ --parallel --with python3,bash-completion
override_dh_auto_build:
dh_auto_build --parallel VERSION=$(DEB_VERSION)
+ find out -regex '.*\.\(json\|files\|size_and_md5\|cksums\|md5sums\|sha1sums\|sha256sums\)' \
+ -exec touch --date='$(BUILD_DATE)' {} \;
+ make out/vfs.zip
override_dh_auto_clean:
dh_auto_clean --parallel VERSION=$(DEB_VERSION)
diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index 47518ce..148eb6a 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -25,12 +25,13 @@ import os
import random
import re
import sys
+import zipfile
from .build import (HashedFile,
PackagingTask)
from .config import read_config
from .gog import run_gog_meta_mode
-from .paths import DATADIR
+from .paths import (DATADIR,USE_VFS)
from .util import ascii_safe
from .steam import run_steam_meta_mode
from .version import GAME_PACKAGE_VERSION
@@ -792,21 +793,41 @@ class GameData(object):
logger.debug('loading full data')
- filename = os.path.join(DATADIR, '%s.files' % self.shortname)
- if os.path.isfile(filename):
- logger.debug('... %s', filename)
- data = json.load(open(filename, encoding='utf-8'))
- self._populate_files(data)
-
- for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'):
- filename = os.path.join(DATADIR, '%s.%s%s' %
- (self.shortname, alg,
- '' if alg == 'size_and_md5' else 'sums'))
+ if USE_VFS:
+ zip = os.path.join(DATADIR, 'vfs.zip')
+ with zipfile.ZipFile(zip, 'r') as zf:
+ files = zf.namelist()
+ filename = '%s.files' % self.shortname
+ if filename in files:
+ logger.debug('... vfs.zip/%s', filename)
+ jsondata = zf.open(filename).read().decode('utf-8')
+ data = json.loads(jsondata)
+ self._populate_files(data)
+
+ for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'):
+ filename = '%s.%s%s' % (self.shortname, alg,
+ '' if alg == 'size_and_md5' else 'sums')
+ if filename in files:
+ logger.debug('... vfs.zip/%s', filename)
+ rawdata = zf.open(filename).read().decode('utf-8')
+ for line in rawdata.splitlines():
+ self._add_hash(line.rstrip('\n'), alg)
+ else:
+ filename = os.path.join(DATADIR, '%s.files' % self.shortname)
if os.path.isfile(filename):
logger.debug('... %s', filename)
- with open(filename) as f:
- for line in f:
- self._add_hash(line.rstrip('\n'), alg)
+ data = json.load(open(filename, encoding='utf-8'))
+ self._populate_files(data)
+
+ for alg in ('ck', 'md5', 'sha1', 'sha256', 'size_and_md5'):
+ filename = os.path.join(DATADIR, '%s.%s%s' %
+ (self.shortname, alg,
+ '' if alg == 'size_and_md5' else 'sums'))
+ if os.path.isfile(filename):
+ logger.debug('... %s', filename)
+ with open(filename) as f:
+ for line in f:
+ self._add_hash(line.rstrip('\n'), alg)
self.loaded_file_data = True
@@ -911,16 +932,38 @@ class GameData(object):
return gog.get('game', gog['url'])
def load_games(game='*'):
+ progress = game == '*' and sys.stderr.isatty()
games = {}
- for jsonfile in glob.glob(os.path.join(DATADIR, game + '.json')):
- if game == '*' and sys.stderr.isatty():
+ if USE_VFS:
+ zip = os.path.join(DATADIR, 'vfs.zip')
+ with zipfile.ZipFile(zip, 'r') as zf:
+ if game == '*':
+ for entry in zf.infolist():
+ if entry.filename.split('.')[-1] == 'json':
+ jsonfile = 'vfs.zip/' + entry.filename
+ jsondata = zf.open(entry).read().decode('utf-8')
+ load_json(progress, games, jsonfile, jsondata)
+ else:
+ jsonfile = game + '.json'
+ jsondata = zf.open(jsonfile).read().decode('utf-8')
+ load_json(progress, games, 'vfs.zip/' + jsonfile, jsondata)
+ else:
+ for jsonfile in glob.glob(os.path.join(DATADIR, game + '.json')):
+ jsondata = open(jsonfile, encoding='utf-8').read()
+ load_json(progress, games, jsonfile, jsondata)
+
+ print('\r%s\r' % (' ' * len(games)), end='', flush=True, file=sys.stderr)
+ return games
+
+def load_json(progress, games, jsonfile, jsondata):
+ if progress:
print('.', end='', flush=True, file=sys.stderr)
try:
g = os.path.basename(jsonfile)
g = g[:len(g) - 5]
- data = json.load(open(jsonfile, encoding='utf-8'))
+ data = json.loads(jsondata)
plugin = data.get('plugin', g)
try:
@@ -937,9 +980,6 @@ def load_games(game='*'):
print('Error loading %s:\n' % jsonfile)
raise
- print('\r%s\r' % (' ' * len(games)), end='', flush=True, file=sys.stderr)
- return games
-
def run_command_line():
logger.debug('Arguments: %r', sys.argv)
diff --git a/game_data_packager/paths.py b/game_data_packager/paths.py
index 0cba549..417ca31 100644
--- a/game_data_packager/paths.py
+++ b/game_data_packager/paths.py
@@ -6,7 +6,9 @@ if os.environ.get('GDP_UNINSTALLED'):
CONFIG = './etc/game-data-packager.conf'
DATADIR = './out'
ETCDIR = './etc'
+ USE_VFS = False
else:
CONFIG = '/etc/game-data-packager.conf'
DATADIR = '/usr/share/games/game-data-packager'
ETCDIR = '/etc/game-data-packager'
+ USE_VFS = True
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list