[pkg-eucalyptus-commits] [SCM] managing cloud instances for Eucalyptus branch, master, updated. 3.0.0-alpha3-257-g1da8e3a
Matt Spaulding
mspaulding06 at gmail.com
Sun Jun 16 02:31:20 UTC 2013
The following commit has been merged in the master branch:
commit b4cdc87edfd8b79ee8ca917293bffaf68c023c85
Author: Matt Spaulding <mspaulding06 at gmail.com>
Date: Wed May 8 09:52:37 2013 -0700
Ported euca-download-bundle to requestbuilder
* Moved common code into a helper module
Fixes TOOLS-194 TOOLS-195 TOOLS-67 TOOLS-81
diff --git a/bin/euca-download-bundle b/bin/euca-download-bundle
index af45469..a79ed25 100755
--- a/bin/euca-download-bundle
+++ b/bin/euca-download-bundle
@@ -37,6 +37,5 @@
import euca2ools.commands.bundle.downloadbundle
if __name__ == '__main__':
- cmd = euca2ools.commands.bundle.downloadbundle.DownloadBundle()
- cmd.main_cli()
+ euca2ools.commands.bundle.downloadbundle.DownloadBundle.run()
diff --git a/euca2ools/commands/bundle/deletebundle.py b/euca2ools/commands/bundle/deletebundle.py
index c3995da..57271be 100644
--- a/euca2ools/commands/bundle/deletebundle.py
+++ b/euca2ools/commands/bundle/deletebundle.py
@@ -31,37 +31,29 @@
# Author: Neil Soman neil at eucalyptus.com
# Mitch Garnaat mgarnaat at eucalyptus.com
+from euca2ools.commands.bundle.helpers import download_files
+from euca2ools.commands.bundle.helpers import get_manifest_keys
+from euca2ools.commands.bundle.helpers import get_manifest_parts
from euca2ools.commands.walrus import WalrusRequest
from euca2ools.commands.walrus.checkbucket import CheckBucket
from euca2ools.commands.walrus.deletebucket import DeleteBucket
from euca2ools.commands.walrus.deleteobject import DeleteObject
-from euca2ools.commands.walrus.listbucket import ListBucket
from euca2ools.commands.walrus.getobject import GetObject
+from euca2ools.commands.walrus.listbucket import ListBucket
from euca2ools.exceptions import AWSError
from requestbuilder import Arg, MutuallyExclusiveArgList
+from requestbuilder.exceptions import ArgumentError
import argparse
import os
import shutil
import sys
import tempfile
-import textwrap
import time
from xml.dom import minidom
class DeleteBundle(WalrusRequest):
- DESCRIPTION = textwrap.dedent('''\
- Delete a previously-uploaded bundle.
-
- Use --manifest to delete a specific bundle based on the contents of
- a locally-available manifest file.
-
- Use --prefix to delete a specific bundle based on the contents of a
- manifest file in the bucket.
-
- (Deprecated) When neither --manifest nor --prefix is supplied, all
- bundles in the given bucket are deleted.''')
-
+ DESCRIPTION = 'Delete a previously-uploaded bundle.'
ARGS = [Arg('-b', '--bucket', dest='bucket', metavar='BUCKET',
required=True, help='Name of the bucket to delete from.'),
MutuallyExclusiveArgList(True,
@@ -74,44 +66,12 @@ class DeleteBundle(WalrusRequest):
Arg('--delete-all-bundles', dest='delete_all',
action='store_true', help=argparse.SUPPRESS)),
Arg('--clear', dest='clear', action='store_true',
- help='Delete the entire bucket if possible')]
-
- def _get_part_paths(self, manifest):
- paths = []
- bucket = self.args.get('bucket')
- try:
- dom = minidom.parse(manifest)
- elem = dom.getElementsByTagName('manifest')[0]
- for tag in elem.getElementsByTagName('filename'):
- for node in tag.childNodes:
- if node.nodeType == node.TEXT_NODE:
- paths.append(os.path.join(bucket, node.data))
- except:
- print >> sys.stderr, 'problem parsing: %s' % manifest
- return paths
-
- def _get_manifest_keys(self):
- manifests = []
- response = ListBucket(paths=[self.args.get('bucket')],
- service=self.service, config=self.config).main()
- for item in response.get('Contents'):
- key = item.get('Key')
- if key.endswith('.manifest.xml'):
- manifests.append(key)
- return manifests
-
- def _download_manifests(self, manifest_keys, directory):
- bucket = self.args.get('bucket')
- paths = [os.path.join(bucket, key) for key in manifest_keys]
- try:
- GetObject(paths=paths, opath=directory).main()
- return True
- except AWSError as err:
- return False
+ help='Delete the entire bucket if possible')]
def _delete_manifest_parts(self, manifest_keys, directory):
+ bucket = self.args.get('bucket')
for key in manifest_keys:
- paths = self._get_part_paths(os.path.join(directory, key))
+ paths = get_manifest_parts(os.path.join(directory, key), bucket)
DeleteObject(paths=paths, service=self.service,
config=self.config).main()
@@ -122,16 +82,38 @@ class DeleteBundle(WalrusRequest):
def _delete_by_local_manifest(self):
manifest_path = self.args.get('manifest_path')
- manifest_keys = [self.get_relative_filename(manifest_path)]
- directory = self.get_file_path(manifest_path)
+ if not os.path.isfile(manifest_path):
+ raise ArgumentError("manifest file '%s' does not exist." % manifest_path)
+ manifest_keys = [os.path.basename(manifest_path)]
+ directory = os.path.dirname(manifest_path) or '.'
+ # When we use a local manifest file, we should still check if there is
+ # a matching manifest in the bucket. If it's there then we delete it.
+ # It's okay if this fails. It might not be there.
+ try:
+ self._delete_manifest_keys(manifest_keys)
+ except AWSError as err:
+ if err[2] == 'NoSuchEntity':
+ pass
+ else:
+ raise
self._delete_manifest_parts(manifest_keys, directory)
def _delete_by_prefix(self):
+ bucket = self.args.get('bucket')
directory = tempfile.mkdtemp()
try:
manifest_keys = ['%s.manifest.xml' % self.args.get('prefix')]
- if self._download_manifests(manifest_keys, directory):
- self._delete_manifest_parts(manifest_keys, directory)
+ try:
+ download_files(bucket, manifest_keys, directory,
+ service=self.service, config=self.config)
+ except AWSError as err:
+ if err[2] == 'NoSuchEntity':
+ raise ArgumentError("manifest file '%s' does not exist in "
+ "bucket '%s'."
+ % (manifest_keys[0], bucket))
+ else:
+ raise
+ self._delete_manifest_parts(manifest_keys, directory)
self._delete_manifest_keys(manifest_keys)
finally:
shutil.rmtree(directory)
@@ -155,9 +137,11 @@ If this is not what you want, press Ctrl+C in the next 10 seconds""" % bucket
directory = tempfile.mkdtemp()
try:
- manifest_keys = self._get_manifest_keys()
- if self._download_manifests(manifest_keys, directory):
- self._delete_manifest_parts(manifest_keys, directory)
+ manifest_keys = get_manifest_keys(bucket, service=self.service,
+ config=self.config)
+ download_files(bucket, manifest_keys, directory,
+ service=self.service, config=self.config)
+ self._delete_manifest_parts(manifest_keys, directory)
self._delete_manifest_keys(manifest_keys)
finally:
shutil.rmtree(directory)
diff --git a/euca2ools/commands/bundle/downloadbundle.py b/euca2ools/commands/bundle/downloadbundle.py
index 8eea990..7b518d2 100644
--- a/euca2ools/commands/bundle/downloadbundle.py
+++ b/euca2ools/commands/bundle/downloadbundle.py
@@ -1,6 +1,6 @@
# Software License Agreement (BSD License)
#
-# Copyright (c) 20092011, Eucalyptus Systems, Inc.
+# Copyright (c) 2009-2013, Eucalyptus Systems, Inc.
# All rights reserved.
#
# Redistribution and use of this software in source and binary forms, with or
@@ -31,96 +31,81 @@
# Author: Neil Soman neil at eucalyptus.com
# Mitch Garnaat mgarnaat at eucalyptus.com
-import euca2ools.commands.eucacommand
-from boto.roboto.param import Param
-import sys
+from euca2ools.commands.bundle.helpers import download_files
+from euca2ools.commands.bundle.helpers import get_manifest_keys
+from euca2ools.commands.bundle.helpers import get_manifest_parts
+from euca2ools.commands.walrus import WalrusRequest
+from euca2ools.commands.walrus.checkbucket import CheckBucket
+from euca2ools.exceptions import AWSError
import os
-from xml.dom import minidom
-from boto.exception import S3ResponseError, S3CreateError
-from boto.s3.key import Key
+from requestbuilder import Arg, MutuallyExclusiveArgList
+from requestbuilder.exceptions import ArgumentError
+import shutil
+import sys
+import tempfile
-class DownloadBundle(euca2ools.commands.eucacommand.EucaCommand):
- Description = 'Downloads a bundled image from a bucket.'
- Options = [Param(name='bucket', short_name='b', long_name='bucket',
- optional=False, ptype='string',
- doc='Name of the bucket to upload to.'),
- Param(name='manifest_path',
- short_name='m', long_name='manifest',
- optional=True, ptype='string',
- doc='Path to the manifest file for bundled image.'),
- Param(name='prefix', short_name='p', long_name='prefix',
- optional=True, ptype='string',
- doc='Prefix used to identify the image in the bucket'),
- Param(name='directory',
- short_name='d', long_name='directory',
- optional=True, ptype='dir', default='/tmp',
- doc='The directory to download the parts to.')]
+class DownloadBundle(WalrusRequest):
+ DESCRIPTION = 'Downloads a bundled image from a bucket.'
+ ARGS = [Arg('-b', '--bucket', metavar='BUCKET', required=True,
+ help='Name of the bucket to upload to.'),
+ MutuallyExclusiveArgList(
+ Arg('-m', '--manifest', dest='manifest_path', metavar='FILE',
+ help='Path to local manifest file for bundled image.'),
+ Arg('-p', '--prefix', metavar='PREFIX',
+ help='Prefix used to identify the image in the bucket')),
+ Arg('-d', '--directory', metavar='DIRECTORY',
+ help='The directory to download the parts to.')]
- def ensure_bucket(self, bucket):
- bucket_instance = None
- s3conn = self.make_connection_cli('s3')
- try:
- bucket_instance = s3conn.get_bucket(bucket)
- except S3ResponseError, s3error:
- print >> sys.stderr, 'Unable to get bucket %s' % bucket
- sys.exit()
- return bucket_instance
+ def _download_parts(self, manifests, directory):
+ bucket = self.args.get('bucket')
+ for manifest in manifests:
+ parts = get_manifest_parts(os.path.join(directory, manifest))
+ download_files(bucket, parts, directory, service=self.service,
+ config=self.config,
+ show_progress=self.args.get('show_progress', True))
- def get_parts(self, manifest_filename):
- parts = []
- dom = minidom.parse(manifest_filename)
- manifest_elem = dom.getElementsByTagName('manifest')[0]
- parts_list = manifest_elem.getElementsByTagName('filename')
- for part_elem in parts_list:
- nodes = part_elem.childNodes
- for node in nodes:
- if node.nodeType == node.TEXT_NODE:
- parts.append(node.data)
- return parts
+ def _download_by_local_manifest(self, directory):
+ manifest_path = self.args.get('manifest_path')
+ if not os.path.isfile(manifest_path):
+ raise ArgumentError("manifest file '%s' does not exist." % manifest_path)
+ manifest_key = os.path.basename(manifest_path)
+ if not os.path.exists(os.path.join(directory, manifest_key)):
+ shutil.copyfile(manifest_path, directory)
+ self._download_parts(manifest_key, directory)
- def get_manifests(self, bucket, image_prefix=None):
- manifests = []
- keys = bucket.get_all_keys()
- for k in keys:
- if k.name:
- if k.name.find('manifest') >= 0:
- if image_prefix:
- if k.name.startswith(image_prefix):
- manifests.append(k.name)
- else:
- manifests.append(k.name)
- return manifests
+ def _download_by_prefix(self, directory):
+ bucket = self.args.get('bucket')
+ prefix = self.args.get('prefix')
+ manifest_keys = get_manifest_keys(bucket, prefix, service=self.service,
+ config=self.config)
+ try:
+ download_files(bucket, manifest_keys, directory,
+ service=self.service, config=self.config,
+ show_progress=self.args.get('show_progress', True))
+ except AWSError as err:
+ if err[2] != 'NoSuchEntity':
+ raise
+ raise ArgumentError("cannot find manifest file(s) %s in "
+ "bucket '%s'."
+ % (",".join(manifest_keys), bucket))
+ self._download_parts(manifest_keys, directory)
- def download_manifests(self, bucket, manifests, directory):
- if len(manifests) > 0:
- if not os.path.exists(directory):
- os.makedirs(directory)
- for manifest in manifests:
- k = Key(bucket)
- k.key = manifest
- print 'Downloading', manifest
- manifest_file = open(os.path.join(directory, manifest), 'wb')
- k.get_contents_to_file(manifest_file)
- manifest_file.close()
+ def main(self):
+ bucket = self.args.get('bucket')
+ CheckBucket(bucket=bucket, service=self.service,
+ config=self.config).main()
- def download_parts(self, bucket, manifests, directory):
- for manifest in manifests:
- manifest_filename = os.path.join(directory, manifest)
- parts = self.get_parts(manifest_filename)
- for part in parts:
- k = Key(bucket)
- k.key = part
- print 'Downloading', part
- part_file = open(os.path.join(directory, part), 'wb')
- k.get_contents_to_file(part_file)
- part_file.close()
+ directory = self.args.get('directory') or tempfile.mkdtemp()
+ if not os.path.isdir(directory):
+ raise ArgumentError("location '%s' is either not a directory or " \
+ "does not exist." % directory)
- def main(self):
- bucket_instance = self.ensure_bucket(self.bucket)
- manifests = self.get_manifests(bucket_instance, self.prefix)
- self.download_manifests(bucket_instance, manifests, self.directory)
- self.download_parts(bucket_instance, manifests, self.directory)
+ if self.args.get('manifest'):
+ self._download_by_local_manifest(directory)
+ else:
+ self._download_by_prefix(directory)
- def main_cli(self):
- self.main()
+ # Print location if we used a temp directory
+ if not self.args.get('directory'):
+ print >> sys.stderr, "Bundle downloaded to '%s'" % directory
diff --git a/euca2ools/commands/bundle/helpers.py b/euca2ools/commands/bundle/helpers.py
new file mode 100644
index 0000000..b746013
--- /dev/null
+++ b/euca2ools/commands/bundle/helpers.py
@@ -0,0 +1,90 @@
+# Software License Agreement (BSD License)
+#
+# Copyright (c) 2013, Eucalyptus Systems, Inc.
+# All rights reserved.
+#
+# Redistribution and use of this software in source and binary forms, with or
+# without modification, are permitted provided that the following conditions
+# are met:
+#
+# Redistributions of source code must retain the above
+# copyright notice, this list of conditions and the
+# following disclaimer.
+#
+# Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the
+# following disclaimer in the documentation and/or other
+# materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+from euca2ools.commands.walrus.getobject import GetObject
+from euca2ools.commands.walrus.listbucket import ListBucket
+import os
+import sys
+from xml.dom import minidom
+
+
+def get_manifest_parts(manifest, bucket=None):
+ """Gets a list object containing the filenames of parts in the manifest.
+ Returns a list of parts contained in the manifest.
+ :param manifest: name of the local manifest file to parse.
+ :param bucket: (optional) bucket name to append to the part key.
+ """
+ part_paths = []
+ # try:
+ dom = minidom.parse(manifest)
+ elem = dom.getElementsByTagName('manifest')[0]
+ for tag in elem.getElementsByTagName('filename'):
+ for node in tag.childNodes:
+ if node.nodeType == node.TEXT_NODE:
+ if bucket:
+ part_paths.append(os.path.join(bucket, node.data))
+ else:
+ part_paths.append(node.data)
+ # except:
+ # print >> sys.stderr, 'problem parsing: %s' % manifest
+ return part_paths
+
+
+def get_manifest_keys(bucket, prefix=None, **kwargs):
+ """Gets the key names for manifests in the specified bucket with optional
+ prefix.
+ Returns list of manifest keys in the bucket.
+ :param bucket: bucket to search for manifest keys.
+ :param prefix: (optional) only return keys with this prefix.
+ :param kwargs: (optional) extra options passed to ListBucket.
+ """
+ manifests = []
+ kwargs.update(paths=[bucket])
+ response = ListBucket(**kwargs).main()
+ for item in response.get('Contents'):
+ key = item.get('Key')
+ if key.endswith('.manifest.xml'):
+ if prefix:
+ if key.startswith(prefix):
+ manifests.append(key)
+ else:
+ manifests.append(key)
+ return manifests
+
+def download_files(bucket, keys, directory, **kwargs):
+ """Download manifests from a Walrus bucket to a local directory.
+ :param bucket: The bucket to download manifests from.
+ :param keys: keys of the files to download.
+ :param directory: location to put downloaded manifests.
+ :param kwargs: (optional) extra arguments passed to GetObject.
+ """
+ paths = [os.path.join(bucket, key) for key in keys]
+ kwargs.update(paths=paths, opath=directory)
+ GetObject(**kwargs).main()
--
managing cloud instances for Eucalyptus
More information about the pkg-eucalyptus-commits
mailing list