[game-data-packager] 04/04: mirror: download files with g-d-p code, not wget

[Simon McVittie]

This is an automated email from the git hooks/post-receive script.

smcv pushed a commit to branch master
in repository game-data-packager.

commit 0e8e8f1621839e80c254728873251e09655185cd
Author: Simon McVittie <smcv at debian.org>
Date:   Tue Feb 2 22:25:32 2016 +0000

    mirror: download files with g-d-p code, not wget
    
    This avoids sites that block wget by its User-Agent, and also avoids
    going through the whole file block-by-block for a second time to
    hash it.
---
 tools/mirror.py | 24 ++++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

diff --git a/tools/mirror.py b/tools/mirror.py
index 3e1b607..590ed82 100755
--- a/tools/mirror.py
+++ b/tools/mirror.py
@@ -29,11 +29,13 @@ KEEP_FREE_SPACE = 250 * 1024 * 1024
 import argparse
 import os
 import subprocess
+import urllib
 
 from game_data_packager import (load_games)
 from game_data_packager.build import (choose_mirror)
 from game_data_packager.command_line import (TerminalProgress)
 from game_data_packager.data import (HashedFile)
+from game_data_packager.util import (AGENT)
 
 archives = []
 
@@ -85,17 +87,27 @@ for a in archives:
        if a['size'] > freespace - KEEP_FREE_SPACE:
            print('out of space, can not download %s' % a['name'])
            continue
-       subprocess.check_call(['wget', a['download'],
-                              '-O', a['name']],
-                              cwd=args.destination)
+
+       rf = urllib.request.urlopen(urllib.request.Request(a['download'],
+               headers={'User-Agent': AGENT}))
+       if rf is None:
+           continue
+
+       wf = open(archive, 'wb')
+       hf = HashedFile.from_file(a['download'], rf, wf, size=a['size'],
+               progress=TerminalProgress(info='downloading %s' %
+                   a['download']))
+       wf.close()
+       rf.close()
+   else:
+       print('checking %s ...' % archive)
+       hf = HashedFile.from_file(archive, open(archive, 'rb'),
+            size=a['size'], progress=TerminalProgress())
 
    if os.path.getsize(archive) == 0:
        exit("%s is empty !!!" % archive)
    if os.path.getsize(archive) != a['size']:
        exit("%s has the wrong size !!!" % archive)
-   print('checking %s ...' % archive)
-   hf = HashedFile.from_file(archive, open(archive, 'rb'),
-        size=a['size'], progress=TerminalProgress())
    if a['md5'] and a['md5'] != hf.md5:
        exit("md5 doesn't match for %s !!!" % archive)
    if a['sha1'] and a['sha1'] != hf.sha1:

-- 
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-games/game-data-packager.git