[Simple-cdd-devel] Bug#861198: Shutting down public FTP services

[Vagrant Cascadian]

On 2017-04-28, Vagrant Cascadian wrote:
> On 2017-04-28, Enrico Zini wrote:
>> Would you like me to try and provide a version which uses urlretrieve?
>
> All these ideas sound good to me, so please take a shot at it!

Actually, being able to verify the checksums instead of relying on
timestamps got me excited enough to try this myself...

The http_proxy environment variable isn't set from profiles/*.conf, but
works if you expressly call it like so:

  http_proxy=http://127.0.0.1:8000 ./build-simple-cdd

So we may need to set that with os.environ or something?


diff --git a/simple_cdd/tools/mirror_wget.py b/simple_cdd/tools/mirror_wget.py
index b05c813..8bec6d0 100644
--- a/simple_cdd/tools/mirror_wget.py
+++ b/simple_cdd/tools/mirror_wget.py
@@ -3,6 +3,7 @@ from simple_cdd.utils import run_command, Checksums
 from simple_cdd.gnupg import Gnupg
 from .base import Tool
 from urllib.parse import urlparse, urljoin
+from urllib import request
 import os
 import re
 import logging
@@ -35,13 +36,25 @@ class ToolMirrorWget(Tool):
             baseurl = env.get("wget_debian_mirror")
             path_depth = urlparse(baseurl).path.strip("/").count("/") + 1
 
-            def _download(url, output):
+            def _download(url, output, checksums=None, relname=None):
+                if checksums:
+                    if os.path.exists(output):
+                        try:
+                            checksums.verify_file(output, relname)
+                            log.debug("skipping download: %s checksum matched", output)
+                            return
+                        except:
+                            log.debug("re-downloading: %s checksum invalid", output)
+                            pass
                 if not os.path.isdir(os.path.dirname(output)):
                     os.makedirs(os.path.dirname(output))
-                args = ["wget", "--output-document="+output, "--timestamping", url]
-                retval = run_command("wget {}".format(url), args, logfd=logfd, env=wget_env)
-                if retval != 0:
-                    raise Fail("wget exited with code %s, see %s for full output log", retval, logfilename)
+                log.debug("downloading: %s", output)
+                request.urlretrieve(url, filename=output)
+                if checksums:
+                    try:
+                        checksums.verify_file(output, relname)
+                    except:
+                        raise Fail("Checksum invalid: %s", output)
 
             # Build the environment for running reprepro
             wget_env = {}
@@ -82,9 +95,7 @@ class ToolMirrorWget(Tool):
                         })
 
                 for x in ef_files:
-                    _download(x["url"], x["absname"])
-                    extrafile_sums.verify_file(x["absname"], x["relname"])
-
+                    _download(x["url"], x["absname"], checksums=extrafile_sums, relname=x["relname"])
 
             checksum_files = env.get("checksum_files")
 
@@ -92,11 +103,6 @@ class ToolMirrorWget(Tool):
             files = []
             files.extend(checksum_files)
 
-            for x in files:
-                p = os.path.join(env.get("MIRROR"), x)
-                d = os.path.join(env.get("wget_debian_mirror"), x)
-                _download(d, p)
-
             if checksum_files:
                 # Get the release file and verify that it is valid
                 release_file = os.path.join(env.get("simple_cdd_temp"), env.format("{DI_CODENAME}_Release"))
@@ -125,8 +131,9 @@ class ToolMirrorWget(Tool):
                     prefix_len = 7 + len(env.get("DI_CODENAME")) # dists/{DI_CODENAME}/
                     relname = file[prefix_len:]
                     absname = os.path.join(env.get("MIRROR"), file)
+                    url = os.path.join(env.get("wget_debian_mirror"), file)
                     # Validate the file
-                    sums.verify_file(absname, relname)
+                    _download(url, absname, checksums=sums, relname=relname)
 
                     # Get the list of extra files to download: those whose
                     # pathname matches di_match
@@ -148,7 +155,4 @@ class ToolMirrorWget(Tool):
                     file_sums.parse_checksums_file(absname, hashtype)
                     for f in extra_files:
                         # Download the extra files
-                        _download(f["url"], f["absname"])
-                        file_sums.verify_file(f["absname"], f["relname"])
-
-
+                        _download(f["url"], f["absname"], checksums=file_sums, relname=f["relname"])
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 832 bytes
Desc: not available
URL: <http://lists.alioth.debian.org/pipermail/simple-cdd-devel/attachments/20170428/78fb8a64/attachment.sig>