[game-data-packager] 01/02: Avoid two-pass processing of groups files

[Simon McVittie]

This is an automated email from the git hooks/post-receive script.

smcv pushed a commit to branch master
in repository game-data-packager.

commit d219ef08935214d6a0b956d325b7243adf31e8c5
Author: Simon McVittie <smcv at debian.org>
Date:   Mon Feb 22 23:38:07 2016 +0000

    Avoid two-pass processing of groups files
    
    Since commit 2b5a38e0 this has been crashing with io.UnsupportedOperation
    "underlying stream is not seekable" for some people and in some
    situations, but not consistently. Duplicating the list of group names
    at the beginning so that we can process the file linearly is not very
    expensive, and should be mitigated by the compression anyway.
---
 Makefile                       |  2 +-
 game_data_packager/__init__.py | 26 ++++++++++----------------
 tools/compile_yaml.py          |  6 ++++++
 3 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/Makefile b/Makefile
index 55d1001..a1bd516 100644
--- a/Makefile
+++ b/Makefile
@@ -39,7 +39,7 @@ out/%: data/%
 	@mkdir -p out
 	if [ -L $< ]; then cp -a $< $@ ; else install -m644 $< $@ ; fi
 
-out/vfs/%.json: data/%.yaml
+out/vfs/%.json: data/%.yaml tools/compile_yaml.py
 	@mkdir -p out/vfs
 	$(PYTHON) tools/compile_yaml.py $< $@
 
diff --git a/game_data_packager/__init__.py b/game_data_packager/__init__.py
index 7f1fab6..d110df9 100644
--- a/game_data_packager/__init__.py
+++ b/game_data_packager/__init__.py
@@ -945,31 +945,24 @@ class GameData(object):
         current_group = None
         attributes = {}
 
-        # Before doing anything else, we do one pass through the list
-        # of groups to record that each one is a group, so that when we
-        # encounter an entry that is not known to be a group in a
-        # group's members, it is definitely a file.
-        stream.seek(0)
-
         for line in stream:
             stripped = line.strip()
 
-            if stripped.startswith('['):
-                assert stripped.endswith(']'), repr(stripped)
-                self._ensure_group(stripped[1:-1])
-
-        # Now go back and re-read them, with their members this time.
-        stream.seek(0)
-
-        for line in stream:
-            stripped = line.strip()
             if stripped == '' or stripped.startswith('#'):
                 continue
 
-            if stripped.startswith('['):
+            # The group data starts with a list of groups. This is necessary
+            # so we can know whether a group member, encountered later on in
+            # the data, is a group or a file.
+            if stripped.startswith('*'):
+                assert current_group is None
+                self._ensure_group(stripped[1:])
+            # After that, [Group] opens a section for each group
+            elif stripped.startswith('['):
                 assert stripped.endswith(']'), repr(stripped)
                 current_group = self._ensure_group(stripped[1:-1])
                 attributes = {}
+            # JSON metadata is on a line with {}
             elif stripped.startswith('{'):
                 assert current_group is not None
                 attributes = json.loads(stripped)
@@ -977,6 +970,7 @@ class GameData(object):
                 for k, v in attributes.items():
                     assert hasattr(current_group, k), k
                     setattr(current_group, k, v)
+            # Every other line is a member, either a file or a group
             else:
                 f = self._add_hash(stripped, 'size_and_md5')
                 # f can either be a WantedFile or a FileGroup here
diff --git a/tools/compile_yaml.py b/tools/compile_yaml.py
index fd2056a..698fa7e 100755
--- a/tools/compile_yaml.py
+++ b/tools/compile_yaml.py
@@ -46,6 +46,12 @@ def main(f, out):
     if groups is not None:
         with open(offload + '.tmp', 'w', encoding='utf-8') as writer:
             assert isinstance(groups, dict)
+            # The group data starts with a list of groups. This is necessary
+            # so we can know whether a group member, encountered later on in
+            # the data, is a group or a file.
+            for group_name in sorted(groups.keys()):
+                writer.write('*%s\n' % group_name)
+
             for group_name, group_data in sorted(groups.items()):
                 writer.write('[%s]\n' % group_name)
 

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git