[game-data-packager] 04/06: Distinguish between files and groups

Simon McVittie smcv at debian.org
Mon Jan 25 10:26:29 UTC 2016


This is an automated email from the git hooks/post-receive script.

smcv pushed a commit to branch master
in repository game-data-packager.

commit 2b5a38e03747045c1bb7844c3a522821213cb841
Author: Simon McVittie <smcv at debian.org>
Date:   Mon Jan 25 10:19:54 2016 +0000

    Distinguish between files and groups
    
    We no longer need to (ab)use WantedFile to represent a group, and can
    use a smaller, more targeted object which does not support nonsensical
    operations like setting the md5sum of a group.
---
 game_data_packager/__init__.py      | 178 ++++++++++++++++++++++++------------
 game_data_packager/build.py         |   3 +-
 game_data_packager/data.py          |  34 ++++++-
 game_data_packager/make_template.py |  23 ++---
 4 files changed, 159 insertions(+), 79 deletions(-)

diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index 56b3262..792625a 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -31,7 +31,7 @@ import zipfile
 import yaml
 
 from .build import (PackagingTask)
-from .data import (PackageRelation, WantedFile)
+from .data import (FileGroup, PackageRelation, WantedFile)
 from .paths import (DATADIR, USE_VFS)
 from .util import ascii_safe
 from .version import (GAME_PACKAGE_VERSION)
@@ -414,6 +414,9 @@ class GameData(object):
         # { 'baseq3/pak1.pk3': WantedFile instance }
         self.files = {}
 
+        # Map from FileGroup name to instance.
+        self.groups = {}
+
         # Map from WantedFile name to a set of names of WantedFile instances
         # from which the file named in the key can be extracted or generated.
         # { 'baseq3/pak1.pk3': set(['linuxq3apoint-1.32b-3.x86.run']) }
@@ -453,34 +456,50 @@ class GameData(object):
         if 'groups' in self.data:
             groups = self.data['groups']
             assert isinstance(groups, dict), self.shortname
+
+            # Before doing anything else, we do one pass through the list
+            # of groups to record that each one is a group, so that when we
+            # encounter an entry that is not known to be a group in a
+            # group's members, it is definitely a file.
+            for group_name in groups:
+                self._ensure_group(group_name)
+
             for group_name, group_data in groups.items():
+                group = self.groups[group_name]
                 attrs = {}
+
                 if isinstance(group_data, dict):
                     members = group_data['group_members']
                     for k, v in group_data.items():
                         if k != 'group_members':
+                            assert hasattr(group, k), k
+                            setattr(group, k, v)
                             attrs[k] = v
                 elif isinstance(group_data, (str, list)):
                     members = group_data
                 else:
                     raise AssertionError('group %r should be dict, str or list' % group_name)
 
-                group = self._ensure_file(group_name)
-                if group.group_members is None:
-                    group.group_members = set()
-                group.apply_group_attributes(attrs)
-
                 if isinstance(members, str):
                     for line in members.splitlines():
                         f = self._add_hash(line.rstrip('\n'), 'size_and_md5')
                         if f is not None:
-                            f.apply_group_attributes(attrs)
+                            # f may be a WantedFile or a FileGroup,
+                            # this works for either
                             group.group_members.add(f.name)
+                            group.apply_group_attributes(f)
+
                 elif isinstance(members, list):
                     for member_name in members:
-                        f = self._ensure_file(member_name)
-                        f.apply_group_attributes(attrs)
-                        group.group_members.add(member_name)
+                        f = self.groups.get(member_name)
+
+                        if f is None:
+                            f = self._ensure_file(member_name)
+
+                        # f may be a WantedFile or a FileGroup,
+                        # this works for either
+                        group.group_members.add(f.name)
+                        group.apply_group_attributes(f)
                 else:
                     raise AssertionError('group %r members should be str or list' % group_name)
 
@@ -605,10 +624,10 @@ class GameData(object):
             return ret
 
         for filename, f in self.files.items():
-            if f.group_members is not None:
-                groups[filename] = f.to_data(expand=expand)
-            else:
-                files[filename] = f.to_data(expand=expand)
+            files[filename] = f.to_data(expand=expand)
+
+        for name, g in self.groups.items():
+            groups[name] = g.to_data(expand=expand)
 
         for name, package in self.packages.items():
             packages[name] = package.to_data(expand=expand)
@@ -647,8 +666,7 @@ class GameData(object):
     def size(self, package):
         size_min = 0
         size_max = 0
-        for filename in package._install:
-           file = self.files[filename]
+        for file in package.install_files:
            if file.alternatives:
                # 'or 0' is a workaround for the files without known size
                size_min += min(set(self.files[a].size or 0 for a in file.alternatives))
@@ -656,8 +674,7 @@ class GameData(object):
            elif file.size:
                size_min += file.size
                size_max += file.size
-        for filename in package._optional:
-           file = self.files[filename]
+        for file in package.optional_files:
            if file.alternatives:
                size_max += max(set(self.files[a].size for a in file.alternatives))
            elif file.size:
@@ -831,8 +848,20 @@ class GameData(object):
                 if k in data:
                     setattr(f, k, data[k])
 
+    def _ensure_group(self, name):
+        assert name not in self.files, (self.shortname, name)
+
+        if name not in self.groups:
+            logger.debug('Adding group: %s', name)
+            self.groups[name] = FileGroup(name)
+
+        return self.groups[name]
+
     def _ensure_file(self, name):
+        assert name not in self.groups, (self.shortname, name)
+
         if name not in self.files:
+            logger.debug('Adding file: %s', name)
             self.files[name] = WantedFile(name)
 
         return self.files[name]
@@ -889,6 +918,15 @@ class GameData(object):
             size = None
             hexdigest, filename = MD5SUM_DIVIDER.split(line, 1)
 
+        if filename in self.groups:
+            assert size in (None, '_'), \
+                    "%s group %s should not have size" % (
+                            self.shortname, filename)
+            assert hexdigest in (None, '_'), \
+                    "%s group %s should not have hexdigest" % (
+                            self.shortname, filename)
+            return self.groups[filename]
+
         f = self._ensure_file(filename)
 
         if size is not None and size != '_':
@@ -903,6 +941,22 @@ class GameData(object):
         current_group = None
         attributes = {}
 
+        # Before doing anything else, we do one pass through the list
+        # of groups to record that each one is a group, so that when we
+        # encounter an entry that is not known to be a group in a
+        # group's members, it is definitely a file.
+        stream.seek(0)
+
+        for line in stream:
+            stripped = line.strip()
+
+            if stripped.startswith('['):
+                assert stripped.endswith(']'), repr(stripped)
+                self._ensure_group(stripped[1:-1])
+
+        # Now go back and re-read them, with their members this time.
+        stream.seek(0)
+
         for line in stream:
             stripped = line.strip()
             if stripped == '' or stripped.startswith('#'):
@@ -910,19 +964,20 @@ class GameData(object):
 
             if stripped.startswith('['):
                 assert stripped.endswith(']'), repr(stripped)
-                current_group = self._ensure_file(stripped[1:-1])
-                if current_group.group_members is None:
-                    current_group.group_members = set()
+                current_group = self._ensure_group(stripped[1:-1])
                 attributes = {}
             elif stripped.startswith('{'):
-                attributes = {}
-                attributes = json.loads(stripped)
                 assert current_group is not None
-                current_group.apply_group_attributes(attributes)
+                attributes = json.loads(stripped)
+
+                for k, v in attributes.items():
+                    assert hasattr(current_group, k), k
+                    setattr(current_group, k, v)
             else:
                 f = self._add_hash(stripped, 'size_and_md5')
-                f.apply_group_attributes(attributes)
+                # f can either be a WantedFile or a FileGroup here
                 assert current_group is not None
+                current_group.apply_group_attributes(f)
                 current_group.group_members.add(f.name)
 
     def load_file_data(self, use_vfs=USE_VFS):
@@ -936,14 +991,9 @@ class GameData(object):
                 zip = use_vfs
             else:
                 zip = os.path.join(DATADIR, 'vfs.zip')
+
             with zipfile.ZipFile(zip, 'r') as zf:
                 files = zf.namelist()
-                filename = '%s.files' % self.shortname
-                if filename in files:
-                    logger.debug('... %s/%s', zip, filename)
-                    jsondata = zf.open(filename).read().decode('utf-8')
-                    data = json.loads(jsondata)
-                    self._populate_files(data)
 
                 filename = '%s.groups' % self.shortname
                 if filename in files:
@@ -951,6 +1001,13 @@ class GameData(object):
                     stream = io.TextIOWrapper(zf.open(filename), encoding='utf-8')
                     self._populate_groups(stream)
 
+                filename = '%s.files' % self.shortname
+                if filename in files:
+                    logger.debug('... %s/%s', zip, filename)
+                    jsondata = zf.open(filename).read().decode('utf-8')
+                    data = json.loads(jsondata)
+                    self._populate_files(data)
+
                 for alg in ('sha1', 'sha256', 'size_and_md5'):
                     filename = '%s.%s%s' % (self.shortname, alg,
                             '' if alg == 'size_and_md5' else 'sums')
@@ -965,6 +1022,11 @@ class GameData(object):
             if not os.path.isdir(vfs):
                 vfs = DATADIR
 
+            filename = os.path.join(vfs, '%s.groups' % self.shortname)
+            if os.path.isfile(filename):
+                logger.debug('... %s', filename)
+                stream = open(filename, encoding='utf-8')
+                self._populate_groups(stream)
 
             filename = os.path.join(vfs, '%s.files' % self.shortname)
             if os.path.isfile(filename):
@@ -972,12 +1034,6 @@ class GameData(object):
                 data = json.load(open(filename, encoding='utf-8'))
                 self._populate_files(data)
 
-            filename = os.path.join(vfs, '%s.groups' % self.shortname)
-            if os.path.isfile(filename):
-                logger.debug('... %s', filename)
-                stream = open(filename, encoding='utf-8')
-                self._populate_groups(stream)
-
             for alg in ('sha1', 'sha256', 'size_and_md5'):
                 filename = os.path.join(vfs, '%s.%s%s' %
                         (self.shortname, alg,
@@ -994,11 +1050,23 @@ class GameData(object):
             d = self.data['packages'][package.name]
 
             for filename in d.get('doc', ()):
-                f = self._ensure_file(filename)
+                f = self.groups.get(filename)
+
+                if f is None:
+                    f = self._ensure_file(filename)
+
+                # WantedFile and FileGroup both have this
+                assert hasattr(f, 'doc')
                 f.doc = True
 
             for filename in d.get('license', ()):
-                f = self._ensure_file(filename)
+                f = self.groups.get(filename)
+
+                if f is None:
+                    f = self._ensure_file(filename)
+
+                # WantedFile and FileGroup both have this
+                assert hasattr(f, 'license')
                 f.license = True
 
             package.install_files = set(self._iter_expand_groups(package.install))
@@ -1009,12 +1077,10 @@ class GameData(object):
             f.provides_files = set(self._iter_expand_groups(f.provides))
 
         for filename, f in self.files.items():
-            f.provides_files = set(self._iter_expand_groups(f.provides))
-
             for provided in f.provides_files:
                 self.providers.setdefault(provided.name, set()).add(filename)
 
-            if f.alternatives or f.group_members is not None:
+            if f.alternatives:
                 continue
 
             if f.distinctive_size and f.size is not None:
@@ -1123,8 +1189,6 @@ class GameData(object):
                     alt = self.files[alt_name]
                     # an alternative can't be a placeholder for alternatives
                     assert not alt.alternatives, alt_name
-                    # an alternative can't be a placeholder for a group
-                    assert alt.group_members is None, alt_name
 
                 # if this is a placeholder for a bunch of alternatives, then
                 # it doesn't make sense for it to have a defined checksum
@@ -1133,36 +1197,28 @@ class GameData(object):
                 assert wanted.sha1 is None, wanted.name
                 assert wanted.sha256 is None, wanted.name
                 assert wanted.size is None, wanted.name
-
-                # a placeholder for alternatives can't also be a placeholder
-                # for a group
-                assert wanted.group_members is None, wanted.name
-            elif wanted.group_members is not None:
-                for member_name in wanted.group_members:
-                    assert member_name in self.files
-
-                assert wanted.md5 is None, wanted.name
-                assert wanted.sha1 is None, wanted.name
-                assert wanted.sha256 is None, wanted.name
-                assert wanted.size is None, wanted.name
-                assert not wanted.unpack, wanted.unpack
             else:
                 assert (wanted.size is not None or filename in
                         self.data.get('unknown_sizes', ())
                         ), (self.shortname, wanted.name)
 
+        for name, group in self.groups.items():
+            for member_name in group.group_members:
+                assert member_name in self.files or member_name in self.groups
+
     def _iter_expand_groups(self, grouped):
         """Given a set of strings that are either filenames or groups,
         yield the WantedFile instances for those names or the members of
         those groups, recursively.
         """
         for filename in grouped:
-            wanted = self._ensure_file(filename)
-            if wanted.group_members is not None:
-                for x in self._iter_expand_groups(wanted.group_members):
+            group = self.groups.get(filename)
+
+            if group is not None:
+                for x in self._iter_expand_groups(group.group_members):
                     yield x
             else:
-                yield wanted
+                yield self._ensure_file(filename)
 
     def construct_task(self, **kwargs):
         self.load_file_data()
diff --git a/game_data_packager/build.py b/game_data_packager/build.py
index 71caf4b..f778b07 100644
--- a/game_data_packager/build.py
+++ b/game_data_packager/build.py
@@ -2530,9 +2530,10 @@ class PackagingTask(object):
         # optional license file is present
         if package.component == 'local':
             return
-        for f in package.optional:
+        for f in package.optional_files:
              if not self.game.files[f].license:
                  continue
+
              if self.file_status[f] is not FillResult.COMPLETE:
                  package.component = 'local'
                  return
diff --git a/game_data_packager/data.py b/game_data_packager/data.py
index 7fb2ee3..f63cd06 100644
--- a/game_data_packager/data.py
+++ b/game_data_packager/data.py
@@ -165,7 +165,6 @@ class WantedFile(HashedFile):
         super(WantedFile, self).__init__(name)
         self.alternatives = []
         self.doc = False
-        self.group_members = None
         self._distinctive_name = None
         self.distinctive_size = False
         self.download = None
@@ -264,7 +263,6 @@ class WantedFile(HashedFile):
 
         for k in (
                 'download',
-                'group_members',
                 'install_as',
                 'size',
                 'unsuitable',
@@ -359,3 +357,35 @@ class PackageRelation:
 
     def __repr__(self):
         return 'PackageRelation(' + repr(self.to_data()) + ')'
+
+class FileGroup:
+    __APPLY_TO_ALL = ('doc', 'executable', 'install_to', 'license')
+
+    def __init__(self, name):
+        self.name = name
+        self.group_members = set()
+
+        # Attributes to apply to every member of this group.
+        for attr in self.__APPLY_TO_ALL:
+            setattr(self, attr, None)
+
+    def apply_group_attributes(self, other):
+        assert isinstance(other, WantedFile) or isinstance(other, FileGroup)
+
+        for attr in self.__APPLY_TO_ALL:
+            assert hasattr(other, attr)
+            value = getattr(self, attr)
+
+            if value is not None:
+                setattr(other, attr, value)
+
+    def to_data(self, expand=True):
+        ret = {}
+
+        for attr in self.__APPLY_TO_ALL:
+            value = getattr(self, attr)
+
+            if value is not None:
+                ret[attr] = value
+
+        return ret
diff --git a/game_data_packager/make_template.py b/game_data_packager/make_template.py
index c92b6e8..622d8f8 100644
--- a/game_data_packager/make_template.py
+++ b/game_data_packager/make_template.py
@@ -35,7 +35,7 @@ except ImportError:
     from distutils.version import LooseVersion as Version
     ON_DEBIAN = False
 
-from . import (HashedFile, WantedFile)
+from .data import (FileGroup, HashedFile)
 from .gog import GOG
 from .steam import parse_acf
 from .unpack import TarUnpacker
@@ -147,18 +147,12 @@ class GameData(object):
         self.data = dict()
 
         self.groups = {}
-        self.required = WantedFile('probably required')
-        self.required.group_members = set()
-        self.optional = WantedFile('probably optional')
-        self.optional.group_members = set()
-        self.documentation = WantedFile('probably documentation')
-        self.documentation.group_members = set()
-        self.licenses = WantedFile('probably licenses')
-        self.licenses.group_members = set()
-        self.unwanted = WantedFile('probably unwanted')
-        self.unwanted.group_members = set()
-        self.archives = WantedFile('archives')
-        self.archives.group_members = set()
+        self.required = FileGroup('probably required')
+        self.optional = FileGroup('probably optional')
+        self.documentation = FileGroup('probably documentation')
+        self.licenses = FileGroup('probably licenses')
+        self.unwanted = FileGroup('probably unwanted')
+        self.archives = FileGroup('archives')
 
         for group in (self.required, self.optional, self.documentation,
                 self.licenses, self.archives, self.unwanted):
@@ -201,8 +195,7 @@ class GameData(object):
         if isinstance(unpacker, TarUnpacker):
             unpack['skip'] = unpacker.skip
 
-        group = WantedFile('contents of %s' % out_name)
-        group.group_members = set()
+        group = FileGroup('contents of %s' % out_name)
         self.groups[group.name] = group
 
         for entry in unpacker:

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git



More information about the Pkg-games-commits mailing list