[pkg-eucalyptus-commits] [SCM] managing cloud instances for Eucalyptus branch, master, updated. 3.0.0-alpha3-257-g1da8e3a
Garrett Holmstrom
gholms at fedoraproject.org
Sun Jun 16 02:31:28 UTC 2013
The following commit has been merged in the master branch:
commit 27cc310f39bbd624c1f17b75b7bd3261e09f7ede
Author: Garrett Holmstrom <gholms at fedoraproject.org>
Date: Tue May 21 17:27:03 2013 -0700
Port euca-unbundle to requestbuilder
Fixes TOOLS-196
diff --git a/bin/euca-unbundle b/bin/euca-unbundle
index 3f17ac8..7042655 100755
--- a/bin/euca-unbundle
+++ b/bin/euca-unbundle
@@ -1,42 +1,6 @@
-#!/usr/bin/python
-# -*- coding: utf-8 -*-
-
-# Software License Agreement (BSD License)
-#
-# Copyright (c) 2009-2011, Eucalyptus Systems, Inc.
-# All rights reserved.
-#
-# Redistribution and use of this software in source and binary forms, with or
-# without modification, are permitted provided that the following conditions
-# are met:
-#
-# Redistributions of source code must retain the above
-# copyright notice, this list of conditions and the
-# following disclaimer.
-#
-# Redistributions in binary form must reproduce the above
-# copyright notice, this list of conditions and the
-# following disclaimer in the documentation and/or other
-# materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-#
-# Author: Neil Soman neil at eucalyptus.com
-# Mitch Garnaat mgarnaat at eucalyptus.com
+#!/usr/bin/python -tt
import euca2ools.commands.bundle.unbundle
if __name__ == '__main__':
- cmd = euca2ools.commands.bundle.unbundle.Unbundle()
- cmd.main_cli()
-
+ euca2ools.commands.bundle.unbundle.Unbundle.run()
diff --git a/euca2ools/commands/bundle/bundle.py b/euca2ools/commands/bundle/bundle.py
index 7f0249a..30404ba 100644
--- a/euca2ools/commands/bundle/bundle.py
+++ b/euca2ools/commands/bundle/bundle.py
@@ -28,12 +28,15 @@
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
+import binascii
import hashlib
import multiprocessing
import itertools
import logging
+import lxml.objectify
import os.path
import random
+import shutil
import subprocess
import sys
import tarfile
@@ -50,8 +53,8 @@ class Bundle(object):
def __init__(self):
self.digest = None
- self.digest_algorithm = 'SHA1'
- self.enc_algorithm = 'AES-128-CBC'
+ self.digest_algorithm = None
+ self.enc_algorithm = None
self.enc_key = None # a hex string
self.enc_iv = None # a hex string
self.image_filename = None
@@ -73,11 +76,44 @@ class Bundle(object):
progressbar=progressbar)
return new_bundle
+ @classmethod
+ def create_from_manifest(cls, manifest_filename, partdir=None,
+ privkey_filename=None):
+ if partdir is None:
+ partdir = os.path.dirname(manifest_filename)
+ new_bundle = cls()
+ with open(manifest_filename) as manifest_file:
+ manifest = lxml.objectify.parse(manifest_file).getroot()
+ new_bundle.digest = manifest.image.digest.text
+ new_bundle.digest_algorithm = manifest.image.digest.get('algorithm')
+ new_bundle.enc_algorithm = manifest.image.user_encrypted_key.get(
+ 'algorithm')
+ new_bundle.enc_key = _try_to_decrypt_keys(
+ (manifest.image.user_encrypted_key.text,
+ manifest.image.ec2_encrypted_key.text), privkey_filename)
+ new_bundle.enc_iv = _try_to_decrypt_keys(
+ (manifest.image.user_encrypted_iv.text,
+ manifest.image.ec2_encrypted_iv.text), privkey_filename)
+ new_bundle.image_size = int(manifest.image.size.text)
+ new_bundle.parts = []
+ for part in manifest.image.parts.part:
+ part_dict = {}
+ part_filename = os.path.join(partdir, part.filename.text)
+ if not os.path.isfile(part_filename):
+ raise ValueError("no such part: '{0}'".format(part_filename))
+ part_dict['path'] = part_filename
+ part_dict['digest'] = part.digest.text
+ part_dict['size'] = os.path.getsize(part_filename)
+ new_bundle.parts.append(part_dict)
+ return new_bundle
+
def __create_from_image(self, image_filename, part_prefix, part_size=None,
progressbar=None):
if part_size is None:
part_size = self.DEFAULT_PART_SIZE
with self._lock:
+ self.digest_algorithm = 'SHA1'
+ self.enc_algorithm = 'AES-128-CBC'
self.image_filename = image_filename
self.image_size = os.path.getsize(image_filename)
if self.image_size > self.EC2_IMAGE_SIZE_LIMIT:
@@ -161,6 +197,89 @@ class Bundle(object):
# That's the last part
return
+ def extract_image(self, destdir, progressbar=None):
+ assert self.digest_algorithm == 'SHA1'
+ assert self.enc_algorithm == 'AES-128-CBC'
+
+ # pipe for getting the digest from sha1sum
+ digest_pipe_out, digest_pipe_in = multiprocessing.Pipe(duplex=False)
+ # pipe for getting the extracted image's file name from tar
+ imgname_pipe_out, imgname_pipe_in = multiprocessing.Pipe(duplex=False)
+ # pipe for gzip -> sha1sum
+ gzip_out_pipe_out, gzip_out_pipe_in = os.pipe()
+ # pipe for sha1sum -> tar
+ sha_out_pipe_out, sha_out_pipe_in = os.pipe()
+
+ # part reader -> openssl
+ openssl = subprocess.Popen(['openssl', 'enc', '-d', '-aes-128-cbc',
+ '-K', self.enc_key, '-iv', self.enc_iv],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE, bufsize=-1)
+
+ # openssl -> gzip
+ try:
+ subprocess.Popen(['pigz', '-c', '-d'], stdin=openssl.stdout,
+ stdout=gzip_out_pipe_in, close_fds=True,
+ bufsize=-1)
+ except OSError:
+ subprocess.Popen(['gzip', '-c', '-d'], stdin=openssl.stdout,
+ stdout=gzip_out_pipe_in, close_fds=True,
+ bufsize=-1)
+ openssl.stdout.close()
+ os.close(gzip_out_pipe_in)
+
+ # gzip -> sha1sum
+ pid = os.fork()
+ if pid == 0:
+ openssl.stdin.close()
+ digest_pipe_out.close()
+ os.close(sha_out_pipe_out)
+ _calc_digest_and_exit(gzip_out_pipe_out, sha_out_pipe_in,
+ digest_pipe_in)
+ digest_pipe_in.close()
+ os.close(gzip_out_pipe_out)
+ os.close(sha_out_pipe_in)
+
+ tar_thread = threading.Thread(
+ target=_extract_image_from_tarball,
+ args=(os.fdopen(sha_out_pipe_out), destdir, imgname_pipe_in))
+ tar_thread.start()
+
+ # Drive everything by feeding each part in turn to openssl
+ bytes_read = 0 # progressbar must be based on bundled_size, not size
+ progressbar.start()
+ try:
+ for part in self.parts:
+ with open(part['path']) as part_file:
+ while True:
+ chunk = part_file.read(8192)
+ if chunk:
+ openssl.stdin.write(chunk)
+ bytes_read += len(chunk)
+ progressbar.update(bytes_read)
+ else:
+ break
+ finally:
+ openssl.stdin.close()
+ progressbar.finish()
+ tar_thread.join()
+
+ written_digest = digest_pipe_out.recv()
+ digest_pipe_out.close()
+ with self._lock:
+ self.image_filename = imgname_pipe_out.recv()
+ imgname_pipe_out.close()
+ if written_digest != self.digest:
+ raise ValueError('extracted image appears to be corrupt '
+ '(expected digest: {0}, actual: {1})'
+ .format(self.digest, written_digest))
+ if os.path.getsize(self.image_filename) != self.image_size:
+ raise ValueError('extracted image appears to be corrupt '
+ '(expected size: {0}, actual: {1})'
+ .format(self.image_size,
+ os.path.getsize(self.image_filename)))
+ return self.image_filename
+
### BUNDLE CREATION ###
def _write_tarball(infile, outfile, progressbar=None):
@@ -218,3 +337,47 @@ def _write_single_part(infile, part_fname, part_size):
break
return {'path': part_fname, 'digest': part_digest.hexdigest(),
'size': part.tell()}
+
+
+### BUNDLE EXTRACTION ###
+def _get_bundle_keys_from_manifest(manifest, privkey_filename):
+ enc_key = _try_to_decrypt_keys((manifest.image.user_encrypted_key.text,
+ manifest.image.ec2_encrypted_key.text),
+ privkey_filename)
+ enc_iv = _try_to_decrypt_keys((manifest.image.user_encrypted_iv.text,
+ manifest.image.ec2_encrypted_iv.text),
+ privkey_filename)
+ return enc_key, enc_iv
+
+
+def _try_to_decrypt_keys(hex_encrypted_keys, privkey_filename):
+ for key in hex_encrypted_keys:
+ popen = subprocess.Popen(['openssl', 'rsautl', '-decrypt', '-pkcs',
+ '-inkey', privkey_filename],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ (decrypted_key, __) = popen.communicate(binascii.unhexlify(key))
+ try:
+ # Make sure it might actually be an encryption key.
+ # This isn't perfect, but it's still better than nothing.
+ int(decrypted_key, 16)
+ return decrypted_key
+ except ValueError:
+ pass
+ raise ValueError("Failed to decrypt the manifest's encryption info. "
+ "Ensure the key supplied matches the one used for "
+ "bundling.")
+
+
+def _extract_image_from_tarball(infile, destdir, imgname_pipe):
+ tarball = tarfile.open(mode='r|', fileobj=infile)
+ try:
+ tarinfo = tarball.next()
+ assert tarinfo is not None
+ outfile_name = os.path.join(destdir, tarinfo.name)
+ imgname_pipe.send(outfile_name)
+ imgname_pipe.close()
+ tarred_image = tarball.extractfile(tarinfo)
+ with open(outfile_name, 'w') as outfile:
+ shutil.copyfileobj(tarred_image, outfile)
+ finally:
+ tarball.close()
diff --git a/euca2ools/commands/bundle/unbundle.py b/euca2ools/commands/bundle/unbundle.py
index ad3535e..2e27b68 100644
--- a/euca2ools/commands/bundle/unbundle.py
+++ b/euca2ools/commands/bundle/unbundle.py
@@ -1,6 +1,6 @@
# Software License Agreement (BSD License)
#
-# Copyright (c) 2009-2011, Eucalyptus Systems, Inc.
+# Copyright (c) 2009-2013, Eucalyptus Systems, Inc.
# All rights reserved.
#
# Redistribution and use of this software in source and binary forms, with or
@@ -27,66 +27,70 @@
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
-#
-# Author: Neil Soman neil at eucalyptus.com
-# Mitch Garnaat mgarnaat at eucalyptus.com
-import os
-import sys
-from boto.roboto.param import Param
-import euca2ools.commands.eucacommand
-import euca2ools.bundler
-from euca2ools.exceptions import NotFoundError, CommandFailed
+from euca2ools.commands import Euca2ools
+from euca2ools.commands.bundle.bundle import Bundle
+import os.path
+from requestbuilder import Arg
+from requestbuilder.command import BaseCommand
+from requestbuilder.exceptions import ArgumentError
+from requestbuilder.mixins import FileTransferProgressBarMixin
+from requestbuilder.util import set_userregion
-class Unbundle(euca2ools.commands.eucacommand.EucaCommand):
- Description = 'Unbundles a previously uploaded bundle.'
- Options = [Param(name='manifest_path',
- short_name='m', long_name='manifest',
- optional=False, ptype='file',
- doc='Path to the manifest file.'),
- Param(name='private_key_path',
- short_name='k', long_name='privatekey',
- optional=True, ptype='file',
- doc='Path to private key used to encrypt bundle.'),
- Param(name='destination_dir',
- short_name='d', long_name='destination',
- optional=True, ptype='dir', default='.',
- doc="""Directory to store the image to.
- Defaults to the current directory."""),
- Param(name='source_dir',
- short_name='s', long_name='source',
- optional=True, ptype='dir',
- doc="""Source directory for the bundled image parts.
- Defaults to manifest directory.""")]
+class Unbundle(BaseCommand, FileTransferProgressBarMixin):
+ DESCRIPTION = ('Recreate an image from its bundled parts\n\nThe key used '
+ 'to unbundle the image must match the certificate that was '
+ 'used to bundle it.')
+ SUITE = Euca2ools
+ ARGS = [Arg('-m', '--manifest', metavar='FILE', required=True,
+ help="the bundle's manifest file (required)"),
+ Arg('-k', '--privatekey', metavar='FILE',
+ help='''file containing the private key to decrypt the bundle
+ with. This must match the certificate used when bundling the
+ image.'''),
+ Arg('-d', '--destination', metavar='DIR', default='.',
+ help='''where to place the unbundled image (default: current
+ directory)'''),
+ Arg('-s', '--source', metavar='DIR', default='.',
+ help='''directory containing the bundled image parts (default:
+ current directory)'''),
+ Arg('--region', dest='userregion', metavar='USER at REGION',
+ help='''use encryption keys specified for a user and/or region
+ in configuration files''')]
- def main(self):
- if not self.source_dir:
- self.source_dir = self.get_file_path(self.manifest_path)
- if not self.private_key_path:
- self.private_key_path = self.get_environ('EC2_PRIVATE_KEY')
- if not os.path.isfile(self.private_key_path):
- msg = 'Private Key not found: %s' % self.private_key_path
- self.display_error_and_exit(msg)
+ def configure(self):
+ BaseCommand.configure(self)
+ set_userregion(self.config, self.args.get('userregion'))
+ set_userregion(self.config, os.getenv('EUCA_REGION'))
- bundler = euca2ools.bundler.Bundler(self)
- (parts, encrypted_key, encrypted_iv) = \
- bundler.parse_manifest(self.manifest_path)
- image = bundler.assemble_parts(self.source_dir, self.destination_dir,
- self.manifest_path, parts)
- print 'Decrypting image'
- decrypted_image = bundler.decrypt_image(image, encrypted_key,
- encrypted_iv, self.private_key_path)
- os.remove(image)
- print 'Uncompressing image'
- try:
- bundler.untarzip_image(self.destination_dir, decrypted_image)
- except NotFoundError:
- sys.exit(1)
- except CommandFailed:
- sys.exit(1)
- os.remove(decrypted_image)
+ if not self.args.get('privatekey'):
+ config_privatekey = self.config.get_user_option('private-key')
+ if self.args.get('userregion'):
+ self.args['privatekey'] = config_privatekey
+ elif 'EC2_PRIVATE_KEY' in os.environ:
+ self.args['privatekey'] = os.getenv('EC2_PRIVATE_KEY')
+ elif config_privatekey:
+ self.args['privatekey'] = config_privatekey
+ else:
+ raise ArgumentError(
+ 'missing private key; please supply one with -k')
+ self.args['privatekey'] = os.path.expanduser(os.path.expandvars(
+ self.args['privatekey']))
+ if not os.path.exists(self.args['privatekey']):
+ raise ArgumentError("private key file '{0}' does not exist"
+ .format(self.args['privatekey']))
+ if not os.path.isfile(self.args['privatekey']):
+ raise ArgumentError("private key file '{0}' is not a file"
+ .format(self.args['privatekey']))
- def main_cli(self):
- self.main()
+ def main(self):
+ bundle = Bundle.create_from_manifest(
+ self.args['manifest'], partdir=self.args['source'],
+ privkey_filename=self.args['privatekey'])
+ pbar = self.get_progressbar(maxval=bundle.bundled_size)
+ return bundle.extract_image(self.args['destination'],
+ progressbar=pbar)
+ def print_result(self, result):
+ print 'Wrote', result
--
managing cloud instances for Eucalyptus
More information about the pkg-eucalyptus-commits
mailing list