[game-data-packager] 02/03: build: Break out file downloading to its own module
Simon McVittie
smcv at debian.org
Tue Nov 28 09:56:54 UTC 2017
This is an automated email from the git hooks/post-receive script.
smcv pushed a commit to branch master
in repository game-data-packager.
commit 557a065945abab7fa7fda779a0fca4ee4c337e62
Author: Simon McVittie <smcv at debian.org>
Date: Tue Nov 28 08:41:23 2017 +0000
build: Break out file downloading to its own module
This makes it a little easier to test.
Signed-off-by: Simon McVittie <smcv at debian.org>
---
game_data_packager/build.py | 146 +++++--------------------------
game_data_packager/download.py | 194 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 218 insertions(+), 122 deletions(-)
diff --git a/game_data_packager/build.py b/game_data_packager/build.py
index 7080175..f855674 100644
--- a/game_data_packager/build.py
+++ b/game_data_packager/build.py
@@ -20,12 +20,10 @@ from collections import defaultdict
from enum import Enum
import logging
import os
-import random
import shutil
import stat
import subprocess
import tempfile
-import urllib.request
import zipfile
import yaml
@@ -37,13 +35,13 @@ except ImportError:
BACKPORT_SUFFIX = ''
from .data import (HashedFile)
+from .download import (Downloader, OutOfSpace)
from .gog import GOG
from .packaging import (get_native_packaging_system)
-from .paths import (DATADIR, ETCDIR)
+from .paths import (DATADIR)
from .unpack import (TarUnpacker, ZipUnpacker)
from .unpack.umod import (Umod)
-from .util import (AGENT,
- TemporaryUmask,
+from .util import (TemporaryUmask,
check_call,
check_output,
copy_with_substitutions,
@@ -114,60 +112,6 @@ class DownloadNotAllowed(Exception):
class CDRipFailed(Exception):
pass
-def choose_mirror(wanted):
- mirrors = []
- mirror = os.environ.get('GDP_MIRROR')
- if mirror:
- if mirror.startswith('/'):
- mirror = 'file://' + mirror
- elif mirror.split(':')[0] not in ('http', 'https', 'ftp', 'file'):
- mirror = 'http://' + mirror
- if not mirror.endswith('/'):
- mirror = mirror + '/'
-
- if type(wanted.download) is str:
- if not mirror:
- return [wanted.download]
- url_basename = os.path.basename(wanted.download)
- if '?' not in url_basename:
- mirrors.append(mirror + url_basename)
- wanted.name = wanted.name.replace(' ','%20')
- if wanted.name != url_basename and '?' not in wanted.name:
- mirrors.append(mirror + wanted.name)
- mirrors.append(wanted.download)
- return mirrors
-
- for mirror_list, details in wanted.download.items():
- try:
- f = open(os.path.join(ETCDIR, mirror_list), encoding='utf-8')
- for line in f:
- url = line.strip()
- if not url:
- continue
- if url.startswith('#'):
- continue
- if details.get('path', '.') != '.':
- if not url.endswith('/'):
- url = url + '/'
- url = url + details['path']
- if not url.endswith('/'):
- url = url + '/'
- url = url + details.get('name', wanted.name)
- mirrors.append(url)
- except:
- logger.warning('Could not open mirror list "%s"', mirror_list,
- exc_info=True)
- random.shuffle(mirrors)
- if mirror:
- if mirrors and '?' not in mirrors[0]:
- mirrors.insert(0, mirror + os.path.basename(mirrors[0]))
- elif '?' not in wanted.name:
- mirrors.insert(0, mirror + wanted.name)
- if not mirrors:
- logger.error('Could not select a mirror for "%s"', wanted.name)
- return []
- return mirrors
-
def iter_fat_mounts(folder):
with open('/proc/mounts', 'r', encoding='utf8') as mounts:
for line in mounts.readlines():
@@ -249,6 +193,7 @@ class PackagingTask(object):
# Factory for a progress report (or None).
self.progress_factory = lambda info=None: None
+ self.downloader = None
self.game.load_file_data()
def __del__(self):
@@ -749,71 +694,28 @@ class PackagingTask(object):
self.file_status[wanted.name] = FillResult.DOWNLOAD_NEEDED
if download:
- logger.debug('trying to download %s...', wanted.name)
-
- tmpdir = self.save_downloads or os.path.dirname(self.get_workdir())
- statvfs = os.statvfs(tmpdir)
- if wanted.size > statvfs.f_frsize * statvfs.f_bavail:
- logger.error("Out of space on %s, can't download %s.",
- tmpdir, wanted.name)
- self.download_failed |= set(choose_mirror(wanted))
- return FillResult.IMPOSSIBLE
-
- urls = choose_mirror(wanted)
- for url in urls:
- if url in self.download_failed:
- logger.debug('... no, it already failed')
- continue
-
- logger.debug('... %s', url)
-
- tmp = None
- try:
- rf = urllib.request.urlopen(urllib.request.Request(
- url,headers={'User-Agent': AGENT}))
- if rf is None:
- continue
+ if self.downloader is None:
+ self.downloader = Downloader(
+ progress_factory=self.progress_factory)
- try:
- size = int(rf.info().get('Content-Length'))
- except:
- size = None
- if size and size != wanted.size:
- logger.warning("File doesn't have expected size"
- " (%s vs %s), skipping %s",
- size, wanted.size, url)
- self.download_failed.add(url)
- continue
+ if self.save_downloads is not None:
+ dest = self.save_downloads
+ else:
+ dest = self.get_workdir()
- if self.save_downloads is not None:
- tmp = os.path.join(self.save_downloads,
- wanted.name)
- else:
- tmp = os.path.join(self.get_workdir(),
- 'tmp', wanted.name)
- mkdir_p(os.path.dirname(tmp))
-
- wf = open(tmp, 'wb')
- logger.info('downloading %s', url)
- hf = HashedFile.from_file(url, rf, wf,
- size=wanted.size,
- progress=self.progress_factory())
- wf.close()
-
- if self.use_file(wanted.name, (wanted,), tmp, hf):
- assert self.found[wanted.name] == tmp
- assert (self.file_status[wanted.name] ==
- FillResult.COMPLETE)
- return FillResult.COMPLETE
- else:
- # file corrupted or something
- os.remove(tmp)
- except Exception as e:
- logger.warning('Failed to download "%s": %s', url,
- e)
- self.download_failed.add(url)
- if tmp is not None:
- os.remove(tmp)
+ try:
+ path, hasher = self.downloader.download(wanted, dest)
+ except OutOfSpace:
+ return FillResult.IMPOSSIBLE
+ else:
+ if self.use_file(wanted.name, (wanted,), path, hasher):
+ assert self.found[wanted.name] == path
+ assert (self.file_status[wanted.name] ==
+ FillResult.COMPLETE)
+ return FillResult.COMPLETE
+ else:
+ # file corrupted or something
+ os.remove(path)
providers = list(self.game.providers.get(wanted.name, ()))
diff --git a/game_data_packager/download.py b/game_data_packager/download.py
new file mode 100644
index 0000000..d9a9df5
--- /dev/null
+++ b/game_data_packager/download.py
@@ -0,0 +1,194 @@
+#!/usr/bin/python3
+# encoding=utf-8
+#
+# Copyright © 2014-2017 Simon McVittie <smcv at debian.org>
+# Copyright © 2015-2016 Alexandre Detiste <alexandre at detiste.be>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# You can find the GPL license text on a Debian system under
+# /usr/share/common-licenses/GPL-2.
+
+import logging
+import os
+import random
+import urllib.request
+
+from .data import (HashedFile)
+from .paths import (ETCDIR)
+from .util import (AGENT, mkdir_p)
+
+logging.basicConfig()
+logger = logging.getLogger(__name__)
+
+
+class OutOfSpace(Exception):
+ pass
+
+
+class Downloader:
+
+ def __init__(self, progress_factory=None):
+ self.download_failed = set()
+
+ if progress_factory is None:
+ self.progress_factory = lambda info=None: None
+ else:
+ self.progress_factory = progress_factory
+
+ @staticmethod
+ def choose_mirror(wanted):
+ mirrors = []
+ mirror = os.environ.get('GDP_MIRROR')
+ if mirror:
+ if mirror.startswith('/'):
+ mirror = 'file://' + mirror
+ elif mirror.split(':')[0] not in ('http', 'https', 'ftp', 'file'):
+ mirror = 'http://' + mirror
+ if not mirror.endswith('/'):
+ mirror = mirror + '/'
+
+ if type(wanted.download) is str:
+ if not mirror:
+ return [wanted.download]
+ url_basename = os.path.basename(wanted.download)
+ if '?' not in url_basename:
+ mirrors.append(mirror + url_basename)
+ wanted.name = wanted.name.replace(' ','%20')
+ if wanted.name != url_basename and '?' not in wanted.name:
+ mirrors.append(mirror + wanted.name)
+ mirrors.append(wanted.download)
+ return mirrors
+
+ for mirror_list, details in wanted.download.items():
+ try:
+ f = open(os.path.join(ETCDIR, mirror_list), encoding='utf-8')
+ for line in f:
+ url = line.strip()
+ if not url:
+ continue
+ if url.startswith('#'):
+ continue
+ if details.get('path', '.') != '.':
+ if not url.endswith('/'):
+ url = url + '/'
+ url = url + details['path']
+ if not url.endswith('/'):
+ url = url + '/'
+ url = url + details.get('name', wanted.name)
+ mirrors.append(url)
+ except:
+ logger.warning('Could not open mirror list "%s"', mirror_list,
+ exc_info=True)
+ random.shuffle(mirrors)
+ if mirror:
+ if mirrors and '?' not in mirrors[0]:
+ mirrors.insert(0, mirror + os.path.basename(mirrors[0]))
+ elif '?' not in wanted.name:
+ mirrors.insert(0, mirror + wanted.name)
+ if not mirrors:
+ logger.error('Could not select a mirror for "%s"', wanted.name)
+ return []
+ return mirrors
+
+ def download(self, wanted, dest):
+ logger.debug('trying to download %s...', wanted.name)
+ statvfs = os.statvfs(dest)
+ if wanted.size > statvfs.f_frsize * statvfs.f_bavail:
+ logger.error("Out of space on %s, can't download %s.",
+ dest, wanted.name)
+ self.download_failed |= set(self.choose_mirror(wanted))
+ raise OutOfSpace
+
+ urls = self.choose_mirror(wanted)
+ for url in urls:
+ if url in self.download_failed:
+ logger.debug('... no, it already failed')
+ continue
+
+ logger.debug('... %s', url)
+
+ tmp = None
+ try:
+ rf = urllib.request.urlopen(urllib.request.Request(
+ url,headers={'User-Agent': AGENT}))
+ if rf is None:
+ continue
+
+ try:
+ size = int(rf.info().get('Content-Length'))
+ except:
+ size = None
+ if size and size != wanted.size:
+ logger.warning("File doesn't have expected size"
+ " (%s vs %s), skipping %s",
+ size, wanted.size, url)
+ self.download_failed.add(url)
+ continue
+
+ tmp = os.path.join(dest, wanted.name)
+ mkdir_p(os.path.dirname(tmp))
+
+ wf = open(tmp, 'wb')
+ logger.info('downloading %s', url)
+ hf = HashedFile.from_file(url, rf, wf,
+ size=wanted.size,
+ progress=self.progress_factory())
+ wf.close()
+
+ return tmp, hf
+ except Exception as e:
+ logger.warning('Failed to download "%s": %s', url,
+ e)
+ self.download_failed.add(url)
+ if tmp is not None:
+ os.remove(tmp)
+ else:
+ return None, None
+
+if __name__ == '__main__':
+ # Usage:
+ # GDP_UNINSTALLED=1 \
+ # PYTHONPATH=$(pwd) \
+ # python3 -m game_data_packager.download \
+ # unreal skaarj_logo.jpg .
+
+ import sys
+
+ from . import (load_games)
+
+ game = sys.argv[1]
+ filename = sys.argv[2]
+ dest = sys.argv[3]
+
+ games = load_games(game=game)
+ game = games[game]
+ game.load_file_data()
+ wanted = game.files[filename]
+
+ path, hasher = Downloader().download(wanted, dest)
+
+ if path is None:
+ logger.error('Unable to download "%s"', filename)
+ else:
+ logger.info('Downloaded "%s" to "%s"', filename, path)
+
+ if hasher.size != wanted.size:
+ logger.info('size: %s, expected %s', hasher.size, wanted.size)
+
+ if hasher.md5 != wanted.md5:
+ logger.info('md5: %s, expected %s', hasher.md5, wanted.md5)
+
+ if hasher.sha1 != wanted.sha1:
+ logger.info('sha1: %s, expected %s', hasher.sha1, wanted.sha1)
+
+ if hasher.sha256 != wanted.sha256:
+ logger.info(
+ 'sha256: %s, expected %s', hasher.sha256, wanted.sha256)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git
More information about the Pkg-games-commits
mailing list