[game-data-packager] 03/06: Redo logic for files with alternatives to avoid unnecessary warnings (Closes: #775152)
Simon McVittie
smcv at debian.org
Mon Jan 12 10:49:24 UTC 2015
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit e079ec622104fb20c8bb146db9dbef4eaf6e9b46
Author: Simon McVittie <smcv at debian.org>
Date: Mon Jan 12 10:10:08 2015 +0000
Redo logic for files with alternatives to avoid unnecessary warnings (Closes: #775152)
---
data/doom.yaml | 10 +-
data/doom2.yaml | 14 +-
data/final-doom.yaml | 6 -
data/heretic.yaml | 4 -
data/hexen.yaml | 3 -
data/rtcw.yaml | 3 -
debian/changelog | 2 +
lib/game_data_packager/__init__.py | 272 ++++++++++++++++++++++++---------
lib/game_data_packager/check_syntax.py | 8 +-
9 files changed, 217 insertions(+), 105 deletions(-)
diff --git a/data/doom.yaml b/data/doom.yaml
index afd45d3..88c584c 100644
--- a/data/doom.yaml
+++ b/data/doom.yaml
@@ -34,11 +34,13 @@ files:
alternatives:
- doom.wad_1.9ud
- doom.wad_1.9
- # presumably some of the other alternatives would be OK too
+ - doom.wad_xbox
+ - doom.wad_psn
+ - doom.wad_bfg
+ - doom1.wad_pocketpc
# Best available full version: The Ultimate Doom
doom.wad_1.9ud:
- distinctive_name: false
size: 12408292
sha1: 9b07b02ab3c275a6a7570c3f73cc20d63a0e3833
md5: c4fe9fd920207691a9f493668e0a2083
@@ -47,28 +49,24 @@ files:
# Other 1.9 variants
doom.wad_1.9:
- distinctive_name: false
size: 11159840
sha1: 7742089b4468a736cadb659a7deca3320fe6dcbd
md5: 1cd63c5ddff1bf8ce844237f580e9cf3
look_for:
- doom.wad
doom.wad_xbox:
- distinctive_name: false
size: 12538385
md5: 0c8758f102ccafe26a3040bee8ba5021
sha1: 1d1d4f69fe14fa255228d8243470678b1b4efdc5
look_for:
- doom.wad
doom.wad_psn:
- distinctive_name: false
size: 12474561
md5: e4f120eab6fb410a5b6e11c947832357
sha1: 117015379c529573510be08cf59810aa10bb934e
look_for:
- doom.wad
doom.wad_bfg:
- distinctive_name: false
size: 12487824
md5: fb35c4a5a9fd49ec29ab6e900572c524
sha1: e5ec79505530e151ff0e6f517f3ce1fd65969c46
diff --git a/data/doom2.yaml b/data/doom2.yaml
index cb05841..df1266c 100644
--- a/data/doom2.yaml
+++ b/data/doom2.yaml
@@ -25,16 +25,19 @@ files:
doom2.wad:
alternatives:
- doom2.wad_1.9
- # FIXME: do older wads work?
+ - doom2.wad_bfg
+ - doom2.wad_xbox360_bfg
+ - doom2.wad_xbla
+ - doom2.wad_xbox_roe
+ - doom2.wad_psn
+ - doom2.wad_tapwave_zodiac
doom2.wad_1.9:
size: 14604584
look_for: [doom2.wad]
- distinctive_name: false
doom2.wad_bfg:
size: 14691821
look_for: [doom2.wad]
- distinctive_name: false
doom2.wad_1.666g:
size: 14824716
look_for: [doom2.wad]
@@ -58,23 +61,18 @@ files:
doom2.wad_xbox360_bfg:
size: 14677988
look_for: [doom2.wad]
- distinctive_name: false
doom2.wad_xbla:
size: 14685034
look_for: [doom2.wad]
- distinctive_name: false
doom2.wad_xbox_roe:
size: 14683458
look_for: [doom2.wad]
- distinctive_name: false
doom2.wad_psn:
size: 14599800
look_for: [doom2.wad]
- distinctive_name: false
doom2.wad_tapwave_zodiac:
size: 14639397
look_for: [doom2.wad]
- distinctive_name: false
md5sums: |
25e1459ca71d321525f84628f45ca8cd doom2.wad_1.9
diff --git a/data/final-doom.yaml b/data/final-doom.yaml
index 707f36e..4dd648c 100644
--- a/data/final-doom.yaml
+++ b/data/final-doom.yaml
@@ -36,7 +36,6 @@ files:
- tnt.wad_psn
tnt.wad_id_anthology:
- distinctive_name: false
size: 18654796
sha1: 4a65c8b960225505187c36040b41a40b152f8f3e
md5: 1d39e405bf6ee3df69a8d2646c8d5c49
@@ -44,7 +43,6 @@ files:
- tnt.wad
tnt.wad_1.9:
- distinctive_name: false
size: 18195736
sha1: 9fbc66aedef7fe3bae0986cdb9323d2b8db4c9d3
md5: 4e158d9953c79ccf97bd0663244cc6b6
@@ -52,7 +50,6 @@ files:
- tnt.wad
tnt.wad_psn:
- distinctive_name: false
size: 18222568
sha1: 139e26d801a64b404b8d898defca10227a61867b
md5: be626c12b7c9d94b1dfb9c327566b4ff
@@ -66,7 +63,6 @@ files:
- plutonia.wad_psn
plutonia.wad_id_anthology:
- distinctive_name: false
size: 18240172
sha1: f131cbe1946d7fddb3caec4aa258c83399c21e60
md5: 3493be7e1e2588bc9c8b31eab2587a04
@@ -74,7 +70,6 @@ files:
- plutonia.wad
plutonia.wad_1.9:
- distinctive_name: false
size: 17420824
sha1: 90361e2a538d2388506657252ae41aceeb1ba360
md5: 75c8cf89566741fa9d22447604053bd7
@@ -82,7 +77,6 @@ files:
- plutonia.wad
plutonia.wad_psn:
- distinctive_name: false
size: 17417800
sha1: 327f8c41ebd4138354e9fca63cebbbd1b9489749
md5: b77ca6a809c4fae086162dad8e7a1335
diff --git a/data/heretic.yaml b/data/heretic.yaml
index 15f6cfa..e6e188d 100644
--- a/data/heretic.yaml
+++ b/data/heretic.yaml
@@ -50,13 +50,10 @@ files:
# format: zip
heretic.wad:
- distinctive_name: false
alternatives:
- heretic.wad_1.3
- # FIXME: do older wads work?
heretic.wad_1.3:
- distinctive_name: false
size: 14189976
look_for: [heretic.wad]
@@ -71,7 +68,6 @@ files:
look_for: [heretic.wad]
heretic1.wad_1.2:
- distinctive_name: false
size: 5120920
look_for: [heretic1.wad]
diff --git a/data/hexen.yaml b/data/hexen.yaml
index 92f0db7..5e65294 100644
--- a/data/hexen.yaml
+++ b/data/hexen.yaml
@@ -71,17 +71,14 @@ files:
look_for: [hexen.wad]
hexdd.wad_1.1:
- distinctive_name: false
size: 4440584
look_for: [hexdd.wad]
hexdd.wad_1.0:
- distinctive_name: false
size: 4429700
look_for: [hexdd.wad]
hexen.wad_1.1:
- distinctive_name: false
size: 20083672
look_for: [hexen.wad]
diff --git a/data/rtcw.yaml b/data/rtcw.yaml
index ddb2333..a1b4177 100644
--- a/data/rtcw.yaml
+++ b/data/rtcw.yaml
@@ -48,7 +48,6 @@ packages:
install_files:
main/scripts/translation.cfg_141_unix:
- distinctive_name: false
look_for:
- main/scripts/translation.cfg
install_as: main/scripts/translation.cfg
@@ -77,13 +76,11 @@ files:
- main/sp_pak1.pk3_fr
main/sp_pak1.pk3_en:
- distinctive_name: false
look_for:
- main/sp_pak1.pk3
size: 293887431
main/sp_pak1.pk3_fr:
- distinctive_name: false
look_for:
- main/sp_pak1.pk3
size: 256811934
diff --git a/debian/changelog b/debian/changelog
index c7da387..089b740 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -34,6 +34,8 @@ game-data-packager (39) UNRELEASED; urgency=medium
* Add support for Hexen: Deathkings of the Dark Citadel,
loosely based on patches by Johey Shmit (partially addresses #737137)
* Switch Doom packages' icons to .png, GNOME Shell doesn't like .xpm
+ * Redo logic for files with alternatives to avoid unnecessary warnings
+ (Closes: #775152)
-- Simon McVittie <smcv at debian.org> Mon, 05 Jan 2015 19:38:04 +0000
diff --git a/lib/game_data_packager/__init__.py b/lib/game_data_packager/__init__.py
index 50ff6f9..3bacd9a 100644
--- a/lib/game_data_packager/__init__.py
+++ b/lib/game_data_packager/__init__.py
@@ -112,6 +112,12 @@ class HashedFile(object):
self.sha256 = sha256.hexdigest()
return self
+ @property
+ def have_hashes(self):
+ return ((self.md5 is not None) or
+ (self.sha1 is not None) or
+ (self.sha256 is not None))
+
def matches(self, other):
matched = False
@@ -330,19 +336,36 @@ class GameData(object):
assert 'install_files_from_cksums' not in self.yaml
# Map from WantedFile name to instance.
- # { 'baseq3/pak1.pk3' => WantedFile instance }
+ # { 'baseq3/pak1.pk3': WantedFile instance }
self.files = {}
# Map from WantedFile name to a set of names of WantedFile instances
# from which the file named in the key can be extracted or generated.
- # { 'baseq3/pak1.pk3' => set(['linuxq3apoint-1.32b-3.x86.run']) }
+ # { 'baseq3/pak1.pk3': set(['linuxq3apoint-1.32b-3.x86.run']) }
self.providers = {}
# Map from WantedFile name to the absolute or relative path of
# a matching file on disk.
- # { 'baseq3/pak1.pk3' => '/usr/share/games/quake3/baseq3/pak1.pk3' }
+ # { 'baseq3/pak1.pk3': '/usr/share/games/quake3/baseq3/pak1.pk3' }
self.found = {}
+ # Map from WantedFile look_for name to a set of names of WantedFile
+ # instances which might be it
+ # { 'doom2.wad': set(['doom2.wad_1.9', 'doom2.wad_bfg', ...]) }
+ self.known_filenames = {}
+
+ # Map from WantedFile size to a set of names of WantedFile
+ # instances which might be it
+ # { 14604584: set(['doom2.wad_1.9']) }
+ self.known_sizes = {}
+
+ # Maps from md5, sha1, sha256 to the name of a unique
+ # WantedFile instance
+ # { '25e1459...': 'doom2.wad_1.9' }
+ self.known_md5s = {}
+ self.known_sha1s = {}
+ self.known_sha256s = {}
+
# Failed downloads
self.download_failed = set()
@@ -381,6 +404,34 @@ class GameData(object):
for provided in f.provides:
self.providers.setdefault(provided, set()).add(filename)
+ if f.alternatives:
+ continue
+
+ if f.distinctive_size and f.size is not None:
+ self.known_sizes.setdefault(f.size, set()).add(filename)
+
+ if f.distinctive_name:
+ for lf in f.look_for:
+ self.known_filenames.setdefault(lf, set()).add(filename)
+
+ if f.md5 is not None:
+ if self.known_md5s.get(f.md5):
+ logger.warning('md5 %s matches %s and also %s' %
+ (f.md5, self.known_md5s[f.md5], filename))
+ self.known_md5s[f.md5] = filename
+
+ if f.sha1 is not None:
+ if self.known_sha1s.get(f.sha1):
+ logger.warning('sha1 %s matches %s and also %s' %
+ (f.sha1, self.known_sha1s[f.sha1], filename))
+ self.known_sha1s[f.sha1] = filename
+
+ if f.sha256 is not None:
+ if self.known_sha256s.get(f.sha256):
+ logger.warning('sha256 %s matches %s and also %s' %
+ (f.sha256, self.known_sha256s[f.sha256], filename))
+ self.known_sha256s[f.sha256] = filename
+
if 'compress_deb' in self.yaml:
self.compress_deb = self.yaml['compress_deb']
@@ -438,6 +489,8 @@ class GameData(object):
files = {}
providers = {}
packages = {}
+ known_filenames = {}
+ known_sizes = {}
for filename, f in self.files.items():
files[filename] = f.to_yaml()
@@ -445,11 +498,22 @@ class GameData(object):
for provided, by in self.providers.items():
providers[provided] = list(by)
+ for size, known in self.known_sizes.items():
+ known_sizes[size] = list(known)
+
+ for filename, known in self.known_filenames.items():
+ known_filenames[filename] = list(known)
+
for name, package in self.packages.items():
packages[name] = package.to_yaml()
return {
'help_text': self.help_text,
+ 'known_filenames': known_filenames,
+ 'known_md5s': self.known_md5s,
+ 'known_sha1s': self.known_sha1s,
+ 'known_sha256s': self.known_sha256s,
+ 'known_sizes': known_sizes,
'packages': packages,
'providers': providers,
'files': files,
@@ -539,11 +603,11 @@ class GameData(object):
return self.files[name]
- def use_file(self, wanted, path, hashes=None):
+ def use_file(self, wanted, path, hashes=None, log=True):
logger.debug('found possible %s at %s', wanted.name, path)
size = os.stat(path).st_size
if wanted.size is not None and wanted.size != size:
- if wanted.distinctive_name:
+ if log:
logger.warning('found possible %s\n' +
'but its size does not match:\n' +
' file: %s\n' +
@@ -562,7 +626,8 @@ class GameData(object):
progress=(size > QUITE_LARGE))
if not wanted.skip_hash_matching and not hashes.matches(wanted):
- logger.warning('found possible %s\n' +
+ if log:
+ logger.warning('found possible %s\n' +
'but its checksums do not match:\n' +
' file: %s\n' +
' expected:\n' +
@@ -587,45 +652,100 @@ class GameData(object):
self.found[wanted.name] = path
return True
+ def _ensure_hashes(self, hashes, path, size):
+ if hashes is not None:
+ return hashes
+
+ if size > QUITE_LARGE:
+ logger.info('identifying %s', path)
+ return HashedFile.from_file(path, open(path, 'rb'), size=size,
+ progress=(size > QUITE_LARGE))
+
def consider_file(self, path, really_should_match_something):
if not os.path.exists(path):
# dangling symlink
return
+ tried = set()
+
match_path = '/' + path.lower()
size = os.stat(path).st_size
+ # if a file (as opposed to a directory) is specified on the
+ # command-line, try harder to match it to something
if really_should_match_something:
- if size > QUITE_LARGE:
- logger.info('identifying %s', path)
- hashes = HashedFile.from_file(path, open(path, 'rb'), size=size,
- progress=(size > QUITE_LARGE))
+ hashes = self._ensure_hashes(None, path, size)
else:
hashes = None
- for wanted in self.files.values():
- if wanted.alternatives:
- continue
-
- for lf in wanted.look_for:
- if match_path.endswith('/' + lf):
- self.use_file(wanted, path, hashes)
- if wanted.distinctive_name:
+ for look_for, candidates in self.known_filenames.items():
+ if match_path.endswith('/' + look_for):
+ hashes = self._ensure_hashes(hashes, path, size)
+ for wanted_name in candidates:
+ if wanted_name in tried:
+ continue
+ tried.add(wanted_name)
+ if self.use_file(self.files[wanted_name], path, hashes,
+ log=(len(candidates) == 1)):
+ return
+ else:
+ if len(candidates) > 1:
+ self._log_not_any_of(path, size, hashes,
+ 'possible "%s"' % look_for,
+ [self.files[c] for c in candidates])
+
+ if size in self.known_sizes:
+ hashes = self._ensure_hashes(hashes, path, size)
+ candidates = self.known_sizes[size]
+ for wanted_name in candidates:
+ if wanted_name in tried:
+ continue
+ tried.add(wanted_name)
+ if self.use_file(self.files[wanted_name], path, hashes,
+ log=(len(candidates) == 1)):
+ return
+ else:
+ if len(candidates) > 1:
+ self._log_not_any_of(path, size, hashes,
+ 'file of size %d' % size,
+ [self.files[c] for c in candidates])
+
+ if hashes is not None:
+ for wanted_name in (self.known_md5s.get(hashes.md5),
+ self.known_sha1s.get(hashes.sha1),
+ self.known_sha256s.get(hashes.sha256)):
+ if wanted_name is not None and wanted_name not in tried:
+ tried.add(wanted_name)
+ if self.use_file(self.files[wanted_name], path, hashes):
return
- if wanted.distinctive_size:
- if wanted.size == size:
- logger.debug('... matched by distinctive size %d', size)
- self.use_file(wanted, path, hashes)
+ if really_should_match_something:
+ logger.warning('file "%s" does not match any known file', path)
+
+ def _log_not_any_of(self, path, size, hashes, why, candidates):
+ message = ('found %s but it is not one of the expected ' +
+ 'versions:\n' +
+ ' file: %s\n' +
+ ' size: %d bytes\n' +
+ ' md5: %s\n' +
+ ' sha1: %s\n' +
+ ' sha256: %s\n' +
+ 'expected one of:\n')
+ args = (why, path, size, hashes.md5, hashes.sha1, hashes.sha256)
+
+ for candidate in candidates:
+ message = message + (' %s:\n' +
+ ' size: ' + (
+ '%s' if candidate.size is None else '%d bytes') +
+ '\n' +
+ ' md5: %s\n' +
+ ' sha1: %s\n' +
+ ' sha256: %s\n')
+ args = args + (candidate.name, candidate.size, candidate.md5,
+ candidate.sha1, candidate.sha256)
+
+ logger.warning(message, *args)
- if hashes is not None:
- if not wanted.skip_hash_matching and hashes.matches(wanted):
- logger.debug('... matched hashes of %s', wanted.name)
- self.use_file(wanted, path, hashes)
- return
- else:
- if really_should_match_something:
- logger.warning('file "%s" does not match any known file', path)
def consider_file_or_dir(self, path):
if os.path.isfile(path):
@@ -659,21 +779,13 @@ class GameData(object):
for filename in package.install:
if filename not in self.found:
wanted = self.files[filename]
- alt_possible = False
for alt in wanted.alternatives:
- logger.debug('trying alternative: %s', alt)
if alt in self.found:
- alt_possible = True
break
- elif self.fill_gap(self.files[alt], download=download,
- log=log):
- alt_possible = True
-
- if alt_possible:
- pass
- elif not self.fill_gap(wanted, download=download, log=log):
- possible = False
+ else:
+ if not self.fill_gap(wanted, download=download, log=log):
+ possible = False
return possible
@@ -829,6 +941,38 @@ class GameData(object):
logger.debug('could not find %s, trying to derive it...', wanted.name)
+ if wanted.alternatives:
+ for alt in wanted.alternatives:
+ if alt in self.found:
+ return True
+ elif self.fill_gap(self.files[alt], download=download,
+ log=False):
+ return True
+
+ if log:
+ logger.error('could not find a suitable version of %s:',
+ wanted.name)
+
+ for alt in wanted.alternatives:
+ alt = self.files[alt]
+ logger.error('%s:\n' +
+ ' expected:\n' +
+ ' size: ' + (
+ '%s' if alt.size is None else '%d bytes') +
+ '\n' +
+ ' md5: %s\n' +
+ ' sha1: %s\n' +
+ ' sha256: %s',
+ alt.name,
+ alt.size,
+ alt.md5,
+ alt.sha1,
+ alt.sha256)
+
+ return False
+
+ # no alternatives: try getting the file itself
+
possible = False
if wanted.download:
@@ -932,39 +1076,19 @@ class GameData(object):
if not possible:
if log:
- if wanted.alternatives:
- logger.error('could not find any version of %s:',
- wanted.name)
-
- for alt in wanted.alternatives:
- alt = self.files[alt]
- logger.error('%s:\n' +
- ' expected:\n' +
- ' size: ' + (
- '%s' if alt.size is None else '%d bytes') +
- '\n' +
- ' md5: %s\n' +
- ' sha1: %s\n' +
- ' sha256: %s',
- alt.name,
- alt.size,
- alt.md5,
- alt.sha1,
- alt.sha256)
- else:
- logger.error('could not find %s:\n' +
- ' expected:\n' +
- ' size: ' + (
- '%s' if wanted.size is None else '%d bytes') +
- '\n' +
- ' md5: %s\n' +
- ' sha1: %s\n' +
- ' sha256: %s',
- wanted.name,
- wanted.size,
- wanted.md5,
- wanted.sha1,
- wanted.sha256)
+ logger.error('could not find %s:\n' +
+ ' expected:\n' +
+ ' size: ' + (
+ '%s' if wanted.size is None else '%d bytes') +
+ '\n' +
+ ' md5: %s\n' +
+ ' sha1: %s\n' +
+ ' sha256: %s',
+ wanted.name,
+ wanted.size,
+ wanted.md5,
+ wanted.sha1,
+ wanted.sha256)
return False
diff --git a/lib/game_data_packager/check_syntax.py b/lib/game_data_packager/check_syntax.py
index 6c1f86f..969826e 100644
--- a/lib/game_data_packager/check_syntax.py
+++ b/lib/game_data_packager/check_syntax.py
@@ -15,7 +15,13 @@
# You can find the GPL license text on a Debian system under
# /usr/share/common-licenses/GPL-2.
+import os
+import yaml
+
from . import load_yaml_games
if __name__ == '__main__':
- load_yaml_games()
+ for name, game in load_yaml_games().items():
+ if 'DEBUG' in os.environ:
+ print('# %s -----------------------------------------' % name)
+ print(yaml.safe_dump(game.to_yaml()))
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list