[Simple-cdd-devel] Bug#861198: simple-cdd: downloading docs/tools/README without ftp
Vagrant Cascadian
vagrant at debian.org
Wed Apr 26 03:57:28 UTC 2017
Control: severity 861198 important
On 2017-04-25, Vagrant Cascadian wrote:
>> Is there a different way to get a copy of the files in doc/ ?
>
> At a quick glance, there is:
>
> http://ftp.us.debian.org/debian/extrafiles
>
> Which has the added benefit of being signed, lists the files we want,
> with checksums.
>
> The code used to download debian-installer images based on the signed
> Release files in the distribution is similar to what's needed to parse
> extrafiles... e.g. download extrafiles, verify the signature, look for
> the files we want, download the individual files... so that code could
> be re-used, refactored, rewritten, etc. to support this.
Ok, I drafted up quick-and-dirty, cut-and-pasty, rough, untested,
conceptual code to handle downloading the individual files by matching
what's in the "extrafiles" file, much like how the debian-installer
files check against the Release files, to download the *SUMS files, to
then get a list of files to install.
Initially, it just switches to only using wget to fetch individual
files, which should be fairly easy to then switch to httplib or some
other native python library...
Having worked on this so infrequently... all simple-cdd code could
really use some refactoring once the proof-of-concept is done... *sigh*
live well,
vagrant
diff --git a/simple_cdd/gnupg.py b/simple_cdd/gnupg.py
index 2a930db..ed17fe6 100644
--- a/simple_cdd/gnupg.py
+++ b/simple_cdd/gnupg.py
@@ -29,6 +29,18 @@ class Gnupg:
self.import_keyring(keyring_file)
+ def extract_inline_contents(self, pathname, sigpathname):
+ args = ["gpg", "--no-default-keyring"]
+ for k in self.env.get("keyring"):
+ args.extend(("--keyring", k))
+ args.extend("--decrypt")
+ args.extend(sigpathname)
+ contents = subprocess.Popen([args], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ x = open(pathname, 'rx')
+ for line in contents:
+ x.write(line)
+ x.close()
+
def verify_detached_sig(self, pathname, sigpathname):
args = ["gpg", "--no-default-keyring"]
for k in self.env.get("keyring"):
@@ -38,6 +50,15 @@ class Gnupg:
if retval != 0:
raise Fail("Signature verification failed on %s", pathname)
+ def verify_inline_sig(self, pathname):
+ args = ["gpg", "--no-default-keyring"]
+ for k in self.env.get("keyring"):
+ args.extend(("--keyring", k))
+ args.extend(("--verify", pathname))
+ retval = run_command("verify gpg signature", args)
+ if retval != 0:
+ raise Fail("Signature verification failed on %s", pathname)
+
def import_keyring(self, keyring_file):
"""
Import a keyring into our keyring file
diff --git a/simple_cdd/tools/mirror_wget.py b/simple_cdd/tools/mirror_wget.py
index d822936..7248270 100644
--- a/simple_cdd/tools/mirror_wget.py
+++ b/simple_cdd/tools/mirror_wget.py
@@ -35,26 +35,51 @@ class ToolMirrorWget(Tool):
baseurl = env.get("wget_debian_mirror")
path_depth = urlparse(baseurl).path.strip("/").count("/") + 1
- def _wget_many(urls):
- args = ["wget", "--continue", "--timestamping", "--no-verbose",
- "--no-parent", "--no-host-directories", "--recursive", "--cut-dirs={}".format(path_depth),
- "--directory-prefix=" + env.get("MIRROR")]
- args.extend(urls)
- retval = run_command("wget {} files".format(len(urls)), args, logfd=logfd, env=wget_env)
- if retval != 0:
- raise Fail("wget exited with code %s, see %s for full output log", retval, logfilename)
-
def _wget_one(url, output):
+ os.makedirs(os.path.dirname(output))
args = ["wget", "-O", output, url]
retval = run_command("wget {}".format(url), args, logfd=logfd, env=wget_env)
if retval != 0:
raise Fail("wget exited with code %s, see %s for full output log", retval, logfilename)
+ if env.get("mirror_files"):
+ # Download the checksums present in the archive "extrafiles" and verify
+ extrafiles_file = os.path.join(env.get("simple_cdd_temp"), "extrafiles")
+ extrafiles_file_inlinesig = extrafiles_file + ".inlinesig"
+ download_extrafiles_file = os.path.join(env.get("wget_debian_mirror"), "extrafiles")
+ _wget_one(download_extrafiles_file, extrafiles_file_inlinesig)
+ self.gnupg.verify_inline_sig(extrafiles_file_inlinesig)
+ self.gnupg.extract_inline_contents(extrafiles_file, extrafiles_file_inlinesig)
+
+ # import checksums
+ extrafile_sums = Checksums()
+ extrafile_sums.parse_checksums_file(extrafiles_file, 'SHA256')
+
+ ef=open(extrafiles_file, 'r')
+ efile=ef.readlines()
+ ef.close()
+ ef_match = re.compile(env.get("mirror_files"))
+ ef_files = []
+ for line in efile:
+ hashsum, relname=line.split()
+ if not ef_match.search(relname): continue
+ if relname.startswith(x):
+ if filename == x or x.endswith('/'):
+ ef_files.append({
+ "absname": os.path.join(env.get("MIRROR"), relname),
+ "relname": relname,
+ "url": os.path.join(env.get("wget_debian_mirror"), relname),
+ })
+
+ for x in ef_files:
+ _wget_one(x["url"], x["absname"])
+ extrafile_sums.verify_file(x["absname"], x["relname"])
+
+
checksum_files = env.get("checksum_files")
# Download files needed to build debian-installer image
files = []
- files.extend(env.get("mirror_files"))
files.extend(checksum_files)
# Build the environment for running reprepro
@@ -62,7 +87,10 @@ class ToolMirrorWget(Tool):
for name, val, changed in self.env.export_iter():
wget_env[name] = str(val)
- _wget_many([urljoin(baseurl, x) for x in files])
+ for x in files:
+ p = os.path.join(env.get("simple_cdd_temp"), x)
+ d = os.path.join(env.get("wget_debian_mirror"), x)
+ _wget_one(d, p)
if checksum_files:
# Get the release file and verify that it is valid
@@ -110,13 +138,12 @@ class ToolMirrorWget(Tool):
"url": os.path.join(env.get("wget_debian_mirror"), dirname, relname),
})
- # Download the extra files
- _wget_many([x["url"] for x in extra_files])
-
# Check downloaded files against their corresponding checksums.
file_sums = Checksums()
file_sums.parse_checksums_file(absname, hashtype)
for f in extra_files:
+ # Download the extra files
+ _wget_one([f["url"], f["absname"])
file_sums.verify_file(f["absname"], f["relname"])
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 832 bytes
Desc: not available
URL: <http://lists.alioth.debian.org/pipermail/simple-cdd-devel/attachments/20170425/566ae826/attachment.sig>
More information about the Simple-cdd-devel
mailing list