[Simple-cdd-devel] Bug#861198: simple-cdd: downloading docs/tools/README without ftp

[Vagrant Cascadian]

Control: tags 861198 +patch

On 2017-04-25, Vagrant Cascadian wrote:
> On 2017-04-25, Vagrant Cascadian wrote:
>>> Is there a different way to get a copy of the files in doc/ ? 
>>
>> At a quick glance, there is:
>>
>>   http://ftp.us.debian.org/debian/extrafiles
>>
>> Which has the added benefit of being signed, lists the files we want,
>> with checksums.

The only worry I have with it is if "extrafiles" disappears or changes
format. Right now it appears to be an inline signed gpg list of
checksums.

At any rate, it's the best I've got to go on, and here's a working (if
not beautiful) and tested patch!


diff --git a/simple_cdd/gnupg.py b/simple_cdd/gnupg.py
index 2a930db..b5fc245 100644
--- a/simple_cdd/gnupg.py
+++ b/simple_cdd/gnupg.py
@@ -29,6 +29,23 @@ class Gnupg:
             self.import_keyring(keyring_file)
 
 
+    def extract_inline_contents(self, pathname, sigpathname):
+        args = ["gpg", "--no-default-keyring"]
+        for k in self.env.get("keyring"):
+            args.extend(("--keyring", k))
+        args.extend(["--decrypt"])
+        args.extend([sigpathname])
+        proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        stdout, stderr = proc.communicate()
+        if stdout:
+            lines = stdout.splitlines(keepends=True)
+            x = open(pathname, 'w')
+            for line in lines:
+                x.write(line.decode('utf-8'))
+            x.close()
+        else:
+            return False
+
     def verify_detached_sig(self, pathname, sigpathname):
         args = ["gpg", "--no-default-keyring"]
         for k in self.env.get("keyring"):
@@ -38,6 +55,15 @@ class Gnupg:
         if retval != 0:
             raise Fail("Signature verification failed on %s", pathname)
 
+    def verify_inline_sig(self, pathname):
+        args = ["gpg", "--no-default-keyring"]
+        for k in self.env.get("keyring"):
+            args.extend(("--keyring", k))
+        args.extend(("--verify", pathname))
+        retval = run_command("verify gpg signature", args)
+        if retval != 0:
+            raise Fail("Signature verification failed on %s", pathname)
+
     def import_keyring(self, keyring_file):
         """
         Import a keyring into our keyring file
diff --git a/simple_cdd/tools/mirror_wget.py b/simple_cdd/tools/mirror_wget.py
index d822936..5ccbbca 100644
--- a/simple_cdd/tools/mirror_wget.py
+++ b/simple_cdd/tools/mirror_wget.py
@@ -35,34 +35,66 @@ class ToolMirrorWget(Tool):
             baseurl = env.get("wget_debian_mirror")
             path_depth = urlparse(baseurl).path.strip("/").count("/") + 1
 
-            def _wget_many(urls):
-                args = ["wget", "--continue", "--timestamping", "--no-verbose",
-                        "--no-parent", "--no-host-directories", "--recursive", "--cut-dirs={}".format(path_depth),
-                        "--directory-prefix=" + env.get("MIRROR")]
-                args.extend(urls)
-                retval = run_command("wget {} files".format(len(urls)), args, logfd=logfd, env=wget_env)
-                if retval != 0:
-                    raise Fail("wget exited with code %s, see %s for full output log", retval, logfilename)
-
             def _wget_one(url, output):
-                args = ["wget", "-O", output, url]
+                if not os.path.isdir(os.path.dirname(output)):
+                    os.makedirs(os.path.dirname(output))
+                args = ["wget", "--output-document="+output, "--timestamping", url]
                 retval = run_command("wget {}".format(url), args, logfd=logfd, env=wget_env)
                 if retval != 0:
                     raise Fail("wget exited with code %s, see %s for full output log", retval, logfilename)
 
+            # Build the environment for running reprepro
+            wget_env = {}
+            for name, val, changed in self.env.export_iter():
+                wget_env[name] = str(val)
+
+            if env.get("mirror_files"):
+                # Download the checksums present in the archive "extrafiles" and verify
+                extrafiles_file_inlinesig = os.path.join(env.get("MIRROR"), "extrafiles")
+                extrafiles_file= os.path.join(env.get("simple_cdd_temp"), "extrafiles.unsigned")
+                download_extrafiles_file = os.path.join(env.get("wget_debian_mirror"), "extrafiles")
+                _wget_one(download_extrafiles_file, extrafiles_file_inlinesig)
+                self.gnupg.verify_inline_sig(extrafiles_file_inlinesig)
+                self.gnupg.extract_inline_contents(extrafiles_file, extrafiles_file_inlinesig)
+
+                # import checksums
+                extrafile_sums = Checksums()
+                extrafile_sums.parse_checksums_file(extrafiles_file, 'SHA256')
+
+                ef=open(extrafiles_file, 'r')
+                efile=ef.readlines()
+                ef.close()
+                match_mirror_files=str()
+                for m in env.get("mirror_files"):
+                    match_mirror_files=match_mirror_files + m + '|'
+                match_mirror_files=match_mirror_files.rstrip('|')
+                match_mirror_files='^(' + match_mirror_files + ')'
+                ef_match = re.compile(match_mirror_files)
+                ef_files = []
+                for line in efile:
+                    hashsum, relname=line.split()
+                    if ef_match.search(relname):
+                        ef_files.append({
+                            "absname": os.path.join(env.get("MIRROR"), relname),
+                            "relname": relname,
+                            "url": os.path.join(env.get("wget_debian_mirror"), relname),
+                        })
+
+                for x in ef_files:
+                    _wget_one(x["url"], x["absname"])
+                    extrafile_sums.verify_file(x["absname"], x["relname"])
+
+
             checksum_files = env.get("checksum_files")
 
             # Download files needed to build debian-installer image
             files = []
-            files.extend(env.get("mirror_files"))
             files.extend(checksum_files)
 
-            # Build the environment for running reprepro
-            wget_env = {}
-            for name, val, changed in self.env.export_iter():
-                wget_env[name] = str(val)
-
-            _wget_many([urljoin(baseurl, x) for x in files])
+            for x in files:
+                p = os.path.join(env.get("MIRROR"), x)
+                d = os.path.join(env.get("wget_debian_mirror"), x)
+                _wget_one(d, p)
 
             if checksum_files:
                 # Get the release file and verify that it is valid
@@ -110,13 +142,12 @@ class ToolMirrorWget(Tool):
                                 "url": os.path.join(env.get("wget_debian_mirror"), dirname, relname),
                             })
 
-                    # Download the extra files
-                    _wget_many([x["url"] for x in extra_files])
-
                     # Check downloaded files against their corresponding checksums.
                     file_sums = Checksums()
                     file_sums.parse_checksums_file(absname, hashtype)
                     for f in extra_files:
+                        # Download the extra files
+                        _wget_one(f["url"], f["absname"])
                         file_sums.verify_file(f["absname"], f["relname"])
 
 

As an added bonus, this makes it feasible to set "deb.debian.org" as the
default mirror, and even use https by default.


live well,
  vagrant
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 832 bytes
Desc: not available
URL: <http://lists.alioth.debian.org/pipermail/simple-cdd-devel/attachments/20170426/ae3b59c2/attachment.sig>