[game-data-packager] 35/51: make_template: Use GameData.files instead of reinventing it
Simon McVittie
smcv at debian.org
Fri Dec 29 01:23:37 UTC 2017
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit 997c3721bf236aff40ede4b03d136145e61e40c2
Author: Simon McVittie <smcv at debian.org>
Date: Thu Dec 28 17:24:57 2017 +0000
make_template: Use GameData.files instead of reinventing it
Signed-off-by: Simon McVittie <smcv at debian.org>
---
game_data_packager/make_template.py | 208 +++++++++++++++++++++---------------
1 file changed, 120 insertions(+), 88 deletions(-)
diff --git a/game_data_packager/make_template.py b/game_data_packager/make_template.py
index a836174..1eedb23 100644
--- a/game_data_packager/make_template.py
+++ b/game_data_packager/make_template.py
@@ -35,7 +35,7 @@ except ImportError:
from distutils.version import LooseVersion as Version
ON_DEBIAN = False
-from .data import (FileGroup, HashedFile)
+from .data import (FileGroup, HashedFile, WantedFile)
from .game import (GameData)
from .gog import GOG
from .steam import parse_acf
@@ -158,11 +158,6 @@ class Template:
self.licenses, self.archives, self.unwanted):
self.groups[group.name] = group
- self.file_data = {}
- self.size = {}
- self.md5 = {}
- self.sha1 = {}
- self.sha256 = {}
self.has_dosbox = False
def is_scummvm(self,path):
@@ -172,30 +167,32 @@ class Template:
return True
return False
- def add_archive(self, name, lower=False, reader=None):
- out_name = os.path.basename(name)
+ def add_archive(self, path, lower=False, reader=None):
+ out_name = os.path.basename(path)
if lower:
out_name = out_name.lower()
- self.add_file(name, out_name=out_name, group=self.archives)
+ result = self.add_file(path, out_name=out_name, group=self.archives)
- unpacker = automatic_unpacker(name)
+ unpacker = automatic_unpacker(path)
if unpacker is not None:
with unpacker:
- self.add_unpacker(name, out_name, unpacker)
+ self.add_unpacker(path, result, unpacker)
- def add_unpacker(self, name, out_name, unpacker):
- file_data = self.file_data.setdefault(out_name, {})
- file_data['provides'] = ['contents of %s' % out_name]
- unpack = file_data.setdefault('unpack', {})
- unpack['format'] = unpacker.format
+ return result
+
+ def add_unpacker(self, path, result, unpacker):
+ result.provides = ['contents of %s' % result.name]
+
+ if result.unpack is None:
+ result.unpack = dict(format=unpacker.format)
if isinstance(unpacker, TarUnpacker):
- unpack['skip'] = unpacker.skip
+ result.unpack['skip'] = unpacker.skip
- group = FileGroup('contents of %s' % out_name)
+ group = FileGroup('contents of %s' % result.name)
self.groups[group.name] = group
for entry in unpacker:
@@ -206,13 +203,15 @@ class Template:
group=group)
except NotImplementedError as e:
logger.warning("Can't decompress %s from %s: %s" %
- (entry.name, out_name, e))
+ (entry.name, result.name, e))
def add_file(self, path, out_name=None, group=None, opened=None, size=None,
lang=None, parent_unpacker=None):
if out_name is None:
out_name = path
+ match_path = '/' + path.lower()
+
if group is not None:
# we already know what sort of file it is, probably an archive
pass
@@ -226,44 +225,83 @@ class Template:
'gog.ico', 'gfw_high.ico'):
group = self.unwanted
- if size is None:
- size = os.path.getsize(path)
-
if opened is None:
is_plain_file = True
opened = open(path, 'rb')
else:
is_plain_file = False
- hf = HashedFile.from_file(path, opened)
-
- # avoid that look_for: overlaps in cases where
- # that leeds to looping during package build.
- #
- # e.g.:
- # md5 English = $A
- # md5 German = $B
- # md5 French = $B
- for existing in self.size.keys():
- if (size == self.size[existing] and
- hf.md5 == self.md5[existing] and
- existing.startswith(out_name)):
- out_name = existing
- break
+ hf = HashedFile.from_file(path, opened, size=size)
+ result = None
+ existing = None
+
+ # TODO: This resembles PackagingTask.consider_file()
+ # and PackagingTask.use_file()
+ for look_for, candidates in self.game.known_filenames.items():
+ if match_path.endswith('/' + look_for):
+ for c in candidates:
+ existing = self.game.files[c]
+
+ if hf.matches(existing):
+ # TODO: Show provenance of archive members somehow
+ logger.info('Found %s at %s', existing.name, out_name)
+
+ if existing.size is None:
+ existing.size = hf.size
+
+ if existing.md5 is None:
+ existing.md5 = hf.md5
+
+ if existing.sha1 is None:
+ existing.sha1 = hf.sha1
+
+ if existing.sha256 is None:
+ existing.sha256 = hf.sha256
+
+ result = existing
+ break
+ else:
+ existing = None
+
+ if result is not None:
+ break
else:
- if out_name in self.size:
- if (size == self.size[out_name] and
- hf.md5 == self.md5[out_name]):
- pass
- elif lang:
+ # We are creating a new WantedFile
+ assert existing is None
+
+ if out_name not in self.game.files:
+ pass
+ elif lang and out_name + '?' + lang not in self.game.files:
out_name += ('?' + lang)
- else:
- out_name += ('?' + hf.md5[:6])
- self.size[out_name] = size
- self.md5[out_name] = hf.md5
- self.sha1[out_name] = hf.sha1
- self.sha256[out_name] = hf.sha256
+ elif out_name + '?' + hf.md5[:6] not in self.game.files:
+ out_name += ('?' + hf.md5[:6])
+
+ else:
+ i = 0
+
+ while out_name + '?' + str(i) in self.game.files:
+ i += 1
+
+ out_name += ('?' + str(i))
+
+ assert out_name not in self.game.files
+
+ result = WantedFile(out_name)
+ result.size = hf.size
+ result.md5 = hf.md5
+ result.sha1 = hf.sha1
+ result.sha256 = hf.sha256
+
+ self.game.files[result.name] = result
+
+ for lf in result.look_for:
+ self.game.known_filenames.setdefault(lf, set()).add(result.name)
+
+ self.game.known_md5s.setdefault(result.md5, set()).add(result.name)
+ self.game.known_sha1s.setdefault(result.sha1, set()).add(result.name)
+ self.game.known_sha256s.setdefault(result.sha256, set()).add(
+ result.name)
unpacker = None
@@ -293,7 +331,9 @@ class Template:
if unpacker is not None:
with unpacker:
- self.add_unpacker(path, out_name, unpacker)
+ self.add_unpacker(path, result, unpacker)
+
+ return result
def add_one_dir(self, destdir, lower=False, game=None, lang=None):
if destdir.startswith('/usr/local') or destdir.startswith('/opt/'):
@@ -420,8 +460,14 @@ class Template:
self.add_one_dir(os.path.join(tmp, 'app'), game=game, lang=guess_lang(exe), lower=lower)
rm_rf(tmp)
- self.add_archive(exe, lower)
- self.file_data[os.path.basename(exe)] = dict(unpack=dict(format='innoextract'),provides=['file1','file2'])
+ result = self.add_archive(exe, lower)
+
+ result.unpack = dict(format='innoextract')
+ result.provides = [
+ # FIXME: Fill in what it really provides
+ 'file1',
+ 'file2',
+ ]
def add_one_deb(self,deb,lower):
if not ON_DEBIAN or not which('dpkg-deb'):
@@ -513,22 +559,20 @@ class Template:
target = entry.get_symbolic_link_target()
if entry.is_regular_file:
- hf = HashedFile.from_file(deb + '//data.tar.*//' + name,
- unpacker.open(entry), size=entry.size)
name_l = name.lower()
basename_l = os.path.basename(name_l)
+ install_absolute = False
if os.path.splitext(name_l)[1] in ('.exe', '.bat'):
- self.unwanted.group_members.add(name_l)
+ group = self.unwanted
elif 'support/gog' in name_l or os.path.basename(name_l) in (
'start.sh', 'uninstall.sh'):
- self.unwanted.group_members.add(name_l)
+ group = self.unwanted
elif name.startswith('opt/') and is_license(name):
name = basename_l
- self.licenses.group_members.add(name)
+ group = self.licenses
elif name.startswith('opt/') and is_doc(name):
- name = basename_l
- self.documentation.group_members.add(name)
+ group = self.documentation
elif (install_to is not None and
name.startswith(install_to + '/')):
name = name[len(install_to) + 1:]
@@ -536,15 +580,21 @@ class Template:
name = name.lower()
if self.gog and name.startswith('data/'):
name = name[len('data/'):]
- self.required.group_members.add(name)
+ group = self.required
else:
- self.optional.group_members.add(name)
- self.file_data[name] = dict(install_to='.')
-
- self.size[name] = hf.size
- self.md5[name] = hf.md5
- self.sha1[name] = hf.sha1
- self.sha256[name] = hf.sha256
+ group = self.optional
+ install_absolute = True
+
+ result = self.add_file(
+ deb + '//data.tar.*//' + name,
+ out_name=name,
+ group=group,
+ opened=unpacker.open(entry),
+ size=entry.size,
+ parent_unpacker=unpacker)
+
+ if install_absolute:
+ result.install_to = '.'
elif entry.is_directory:
pass
elif target is not None:
@@ -589,10 +639,6 @@ class Template:
yaml.safe_dump(self.data, stream=sys.stdout,
default_flow_style=False)
- if self.file_data:
- yaml.safe_dump({ 'files': self.file_data }, stream=sys.stdout,
- default_flow_style=False)
-
print('\ngroups:')
for group in self.groups.values():
@@ -609,26 +655,12 @@ class Template:
print(' group_members: |')
for f in sorted(group.group_members):
+ wanted = self.game.files[f]
+
if f in self.unwanted.group_members:
- print(' .%-9s %s %s' % (self.size[f], self.md5[f], f))
+ print(' .%-9s %s %s' % (wanted.size, wanted.md5, f))
else:
- print(' %-9s %s %s' % (self.size[f], self.md5[f], f))
-
- print('\nsha1sums: |')
-
- for f in sorted(self.sha1.keys()):
- if f in self.unwanted.group_members:
- print(' .%s %s' % (self.sha1[f], f))
- else:
- print(' %s %s' % (self.sha1[f], f))
-
- print('\nsha256sums: |')
-
- for f in sorted(self.sha256.keys()):
- if f in self.unwanted.group_members:
- print(' .%s %s' % (self.sha256[f], f))
- else:
- print(' %s %s' % (self.sha256[f], f))
+ print(' %-9s %s %s' % (wanted.size, wanted.md5, f))
print('\n...')
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list