[Pkg-escience-soc2009] [SCM] Tool for creating VM images. branch, master, updated. e8d2b53afcdef695b7415a3f04f04a5add0e5117
David Wendt
dcrkid at yahoo.com
Wed Aug 12 04:19:04 UTC 2009
The following commit has been merged in the master branch:
commit e8d2b53afcdef695b7415a3f04f04a5add0e5117
Author: David Wendt <dcrkid at yahoo.com>
Date: Wed Aug 12 00:18:30 2009 -0400
A CRAPLOAD of changes, all of them me fixing up this automagically-make-us-kernels code.
Also, Boto likes to choke on my cloud's URL, but euca2ools works just fine, so I've switched to use euca-register (which I'm not sure has an ec2-api equivalent.)
Code isn't complete yet; still have some bugs to iron out.
diff --git a/VMBuilder/plugins/debian/distro.py b/VMBuilder/plugins/debian/distro.py
index fa5e141..90ba50a 100644
--- a/VMBuilder/plugins/debian/distro.py
+++ b/VMBuilder/plugins/debian/distro.py
@@ -19,7 +19,7 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
import logging
-import os
+import os, os.path
import socket
import types
import VMBuilder
@@ -239,20 +239,54 @@ EOT''')
return self.xen_ramdisk_path()
def __kernel_path(self):
- path = '/boot/vmlinuz-%s-%s' % (self.find_linux_kernel(self.suite, flavour = None, arch = self.vm.arch), self.suite.default_flavour[self.vm.arch])
- return path
+ kernels = run_cmd("ls", "%s/boot/" % self.vm.installdir)
+
+ for kernel in kernels.split("\n"):
+ k_filename_parts = kernel.split("-")
+
+ if k_filename_parts[0] == "vmlinuz":
+ if k_filename_parts[3] in self.suite.valid_flavours[self.vm.arch]:
+ return "/boot/%s" % kernel
+
+ return None
def __ramdisk_path(self):
- path = '/boot/initrd.img-%s-%s' % (self.find_linux_kernel(self.suite, flavour = None, arch = self.vm.arch), self.suite.default_flavour[self.vm.arch])
- return path
+ kernels = run_cmd("ls", "%s/boot/" % self.vm.installdir)
+
+ for kernel in kernels.split("\n"):
+ k_filename_parts = kernel.split("-")
+
+ if k_filename_parts[0] == "initrd.img":
+ if k_filename_parts[3] in self.suite.valid_flavours[self.vm.arch]:
+ return "/boot/%s" % kernel
+
+ return None
def xen_kernel_path(self):
- path = '/boot/vmlinuz-%s-%s-%s' % (self.xen_kernel_version(), self.suite.xen_kernel_flavour, self.suite.default_flavour[self.vm.arch])
- return path
+ kernels = run_cmd("ls", "%s/boot/" % self.vm.installdir)
+
+ for kernel in kernels.split("\n"):
+ k_filename_parts = kernel.split("-")
+
+ if k_filename_parts[0] == "vmlinuz":
+ if k_filename_parts[3] == self.suite.xen_kernel_flavour:
+ if k_filename_parts[4] in self.suite.valid_flavours[self.vm.arch]:
+ return "/boot/%s" % kernel
+
+ return None
def xen_ramdisk_path(self):
- path = '/boot/initrd.img-%s-%s-%s' % (self.xen_kernel_version(), self.suite.xen_kernel_flavour, self.suite.default_flavour[self.vm.arch])
- return path
+ kernels = run_cmd("ls", "%s/boot/" % self.vm.installdir)
+
+ for kernel in kernels.split("\n"):
+ k_filename_parts = kernel.split("-")
+
+ if k_filename_parts[0] == "initrd.img":
+ if k_filename_parts[3] == self.suite.xen_kernel_flavour:
+ if k_filename_parts[4] in self.suite.valid_flavours[self.vm.arch]:
+ return "/boot/%s" % kernel
+
+ return None
def get_ec2_kernel(self):
if self.suite.ec2_kernel_info:
diff --git a/VMBuilder/plugins/ec2/__init__.py b/VMBuilder/plugins/ec2/__init__.py
index ee657b8..9b6bd40 100644
--- a/VMBuilder/plugins/ec2/__init__.py
+++ b/VMBuilder/plugins/ec2/__init__.py
@@ -45,6 +45,8 @@ class EC2(Plugin):
group.add_option('--ec2-no-amazon-tools', action='store_true', help='Do not use Amazon\'s EC2 AMI tools.')
group.add_option('--ec2-no-euca-tools', action='store_true', help='Do not use Eucalyptus\' EC2 AMI tools.')
group.add_option('--ec2-url', metavar='EC2_URL', help='URL of EC2 or compatible cluster.')
+ group.add_option('--ec2-s3-url', metavar='S3_URL', help='URL of EC2 or compatible cluster\'s S3 server.')
+ group.add_option('--ec2-cloud-cert', metavar='EUCALYPTUS_CERT', help='Certificate of EC2-compatible cluster.')
group.add_option('--ec2-is-eucalyptus', action='store_true', help='Allow the use of Eucalyptus-specific features like KVM.')
group.add_option('--ec2-bundle-kernel', action='store_true', help='Bundle the kernel and ramdisk.')
group.add_option('--ec2-upload-kernel', action='store_true', help='Upload the kernel and ramdisk.')
@@ -55,7 +57,7 @@ class EC2(Plugin):
if not getattr(self.vm, 'ec2', False):
return True
- #NOW, actually check if we can do EC2
+ #NOW, actually check if we can do EC2. Also check if we can take advantage of Euca's KVM support
if not isinstance(self.vm.hypervisor, VMBuilder.plugins.xen.Xen):
if not (self.vm.ec2_is_eucalyptus and isinstance(self.vm.hypervisor, VMBuilder.plugins.kvm.KVM)):
raise VMBuilderUserError("You must use either Xen or KVM on Eucalyptus")
@@ -105,6 +107,15 @@ class EC2(Plugin):
if not self.vm.ec2_name:
raise VMBuilderUserError('When building for EC2 you must supply the name for the image.')
+ if not self.vm.ec2_url and "EC2_URL" in os.environ:
+ self.vm.ec2_url = os.environ["EC2_URL"]
+
+ if not self.vm.ec2_s3_url and "S3_URL" in os.environ:
+ self.vm.ec2_s3_url = os.environ["S3_URL"]
+
+ if not self.vm.ec2_cloud_cert and "EUCALYPTUS_CERT" in os.environ:
+ self.vm.ec2_cloud_cert = os.environ["EUCALYPTUS_CERT"]
+
if not self.vm.ec2_cert:
if "EC2_CERT" in os.environ:
self.vm.ec2_cert = os.environ["EC2_CERT"]
@@ -135,7 +146,7 @@ class EC2(Plugin):
if not self.vm.ec2_secret_key:
if "EC2_SECRET_KEY" in os.environ:
- self.vm.ec2_access_key = os.environ["EC2_SECRET_KEY"]
+ self.vm.ec2_secret_key = os.environ["EC2_SECRET_KEY"]
else:
raise VMBuilderUserError('When building for EC2 you must provide your AWS secret access key.')
@@ -168,89 +179,177 @@ class EC2(Plugin):
kernel_loc = self.vm.distro.kernel_path()
ramdisk_loc = self.vm.distro.ramdisk_path()
- bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', "%s%s" % (installdir, kernel_loc), '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', self.vm.ec2_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.workdir, '--kernel', 'true']
+ kernel_name = os.path.split(kernel_loc)[1]
+
+ if self.vm.ec2_cloud_cert:
+ bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', "%s%s" % (installdir, kernel_loc), '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', kernel_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.destdir, '--kernel', 'true', '--ec2cert', self.vm.ec2_cloud_cert]
+ else:
+ bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', "%s%s" % (installdir, kernel_loc), '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', kernel_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.destdir, '--kernel', 'true']
+
run_cmd(*bundle_cmdline)
- kernel_name = os.path.split(kernel_loc)[1]
- kernel_manifest = '%s/%s.manifest.xml' % (self.vm.workdir, kernel_name)
+ kernel_manifest = '%s/%s.manifest.xml' % (self.vm.destdir, kernel_name)
if ramdisk_loc != None:
- bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', "%s%s" % (installdir, ramdisk_loc), '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', self.vm.ec2_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.workdir, '--ramdisk', 'true']
+ ramdisk_name = os.path.split(ramdisk_loc)[1]
- run_cmd(*bundle_cmdline)
+ if self.vm.ec2_cloud_cert:
+ bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', "%s%s" % (installdir, ramdisk_loc), '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', ramdisk_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.destdir, '--ramdisk', 'true', '--ec2cert', self.vm.ec2_cloud_cert]
+ else:
+ bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', "%s%s" % (installdir, ramdisk_loc), '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', ramdisk_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.destdir, '--ramdisk', 'true']
- ramdisk_name = os.path.split(ramdisk_loc)[1]
- ramdisk_manifest = '%s/%s.manifest.xml' % (self.vm.workdir, ramdisk_name)
+ run_cmd(*bundle_cmdline)
+ ramdisk_manifest = '%s/%s.manifest.xml' % (self.vm.destdir, ramdisk_name)
if self.vm.ec2_upload_kernel:
logging.info("Uploading EC2 kernel bundle")
- upload_cmdline = ['%supload-bundle' % self.ec2_tools_prefix, '--retry', '--manifest', kernel_manifest, '--bucket', self.vm.ec2_bucket, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
- run_cmd(*bundle_cmdline)
+ upload_cmdline = ['%supload-bundle' % self.ec2_tools_prefix, '--manifest', kernel_manifest, '--bucket', self.vm.ec2_bucket, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
+
+ if self.vm.ec2_s3_url:
+ upload_cmdline += ['--url', self.vm.ec2_s3_url]
+
+ if self.vm.ec2_cloud_cert:
+ upload_cmdline += ['--ec2cert', self.vm.ec2_cloud_cert]
+
+ run_cmd(*upload_cmdline)
if ramdisk_loc != None:
- upload_cmdline = ['%supload-bundle' % self.ec2_tools_prefix, '--retry', '--manifest', ramdisk_manifest, '--bucket', self.vm.ec2_bucket, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
- run_cmd(*bundle_cmdline)
+ upload_cmdline = ['%supload-bundle' % self.ec2_tools_prefix, '--manifest', ramdisk_manifest, '--bucket', self.vm.ec2_bucket, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
+
+ if self.vm.ec2_s3_url:
+ upload_cmdline += ['--url', self.vm.ec2_s3_url]
+
+ if self.vm.ec2_cloud_cert:
+ upload_cmdline += ['--ec2cert', self.vm.ec2_cloud_cert]
+
+ run_cmd(*upload_cmdline)
- if self.vm.ec2.register_kernel:
+ if self.vm.ec2_register_kernel:
logging.info("Registering EC2 kernel bundle")
- from boto.ec2.connection import EC2Connection
-
- #Code added to support Eucalyptus and other EC2-compatible clusters
- if self.vm.ec2_url:
- import urlparse
- parsed_url = urlparse.urlparse(self.vm.ec2_url)
- conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key,
- host = parsed_url.hostname, port = parsed_url.port, path = parsed_url.path)
- else:
- conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key)
+ uploaded_kernel_manifest = '%s/%s.manifest.xml' % (self.vm.ec2_bucket, kernel_name)
+ register_cmdline = ['%sregister' % self.ec2_tools_prefix, uploaded_kernel_manifest, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
- kernel_id = conn.register_image('%s/%s.manifest.xml' % (self.vm.ec2_bucket, kernel_name))
+ if self.vm.ec2_s3_url:
+ register_cmdline += ['--url', self.vm.ec2_s3_url]
- logging.info("EC2 kernel ID: %s" % kernel_id)
+ registered_eki = run_cmd(*register_cmdline)
- self.vm.ec2_kernel = kernel_id
+ self.vm.ec2_kernel = registered_eki.split("\t")[1].strip()
+ logging.info("EC2 kernel ID: %s" % self.vm.ec2_kernel)
if ramdisk_loc != None:
- ramdisk_id = conn.register_image('%s/%s.manifest.xml' % (self.vm.ec2_bucket, ramdisk_name))
+ uploaded_ramdisk_manifest = '%s/%s.manifest.xml' % (self.vm.ec2_bucket, ramdisk_name)
+
+ register_cmdline = ['%sregister' % self.ec2_tools_prefix, uploaded_ramdisk_manifest, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
+
+ if self.vm.ec2_s3_url:
+ register_cmdline += ['--url', self.vm.ec2_s3_url]
+
+ registered_eri = run_cmd(*register_cmdline)
+
+ self.vm.ec2_ramdisk = registered_eri.split("\t")[1].strip()
+ logging.info("EC2 ramdisk ID: %s" % self.vm.ec2_ramdisk)
+
+# This is the way we used to do this. Now we just refer to ec2/euca2ools since Boto won't stop choking on my local hostname!!
+# from boto.ec2.connection import EC2Connection
+# from boto.ec2.regioninfo import RegionInfo
+
+# rinfo = None
+# if self.vm.ec2_is_eucalyptus: #default boto regioninfo assumes Amazon EC2
+# rinfo = RegionInfo(None, "eucalyptus", self.vm.ec2_url)
+
+# #Code added to support Eucalyptus and other EC2-compatible clusters
+# if self.vm.ec2_url:
+# import urlparse
+# parsed_url = urlparse.urlparse(self.vm.ec2_url)
+# is_secure = False
+
+# if parsed_url.scheme == 'https':
+# is_secure = True
+
+# conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key,
+# host = parsed_url.hostname, port = parsed_url.port, path = parsed_url.path,
+# is_secure = is_secure, region = rinfo)
+# else:
+# conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key, region = rinfo)
+
+# kernel_id = conn.register_image('%s/%s.manifest.xml' % (self.vm.ec2_bucket, kernel_name))
- logging.info("EC2 ramdisk ID: %s" % ramdisk_id)
- self.vm.ec2_ramdisk = ramdisk_id
+ else: #No uploading, keep the Mani Manifest
+ self.vm.result_files.append(kernel_manifest)
+
+ if ramdisk_loc != None:
+ self.vm.result_files.append(ramdisk_manifest)
if not self.vm.ec2_kernel:
self.vm.ec2_kernel = self.vm.distro.get_ec2_kernel()
- logging.debug('%s - to be used for AKI.' %(self.vm.ec2_kernel))
+
+ logging.debug('%s - to be used for AKI.' %(self.vm.ec2_kernel))
if not self.vm.ec2_ramdisk:
self.vm.ec2_ramdisk = self.vm.distro.ec2_ramdisk_id()
- logging.debug('%s - to be use for the ARI.' %(self.vm.ec2_ramdisk))
+
+ logging.debug('%s - to be use for the ARI.' %(self.vm.ec2_ramdisk))
if self.vm.ec2_bundle:
logging.info("Building EC2 bundle")
bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', self.vm.filesystems[0].filename, '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', self.vm.ec2_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.workdir, '--kernel', self.vm.ec2_kernel, '--ramdisk', self.vm.ec2_ramdisk]
+
+ if self.vm.ec2_cloud_cert:
+ bundle_cmdline += ['--ec2cert', self.vm.ec2_cloud_cert]
+
run_cmd(*bundle_cmdline)
manifest = '%s/%s.manifest.xml' % (self.vm.workdir, self.vm.ec2_name)
if self.vm.ec2_upload:
logging.info("Uploading EC2 bundle")
- upload_cmdline = ['%supload-bundle' % self.ec2_tools_prefix, '--retry', '--manifest', manifest, '--bucket', self.vm.ec2_bucket, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
+ upload_cmdline = ['%supload-bundle' % self.ec2_tools_prefix, '--manifest', manifest, '--bucket', self.vm.ec2_bucket, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
+
+ if self.vm.ec2_s3_url:
+ upload_cmdline += ['--s3_url', self.vm.ec2_s3_url]
+
+ if self.vm.ec2_cloud_cert:
+ upload_cmdline += ['--ec2cert', self.vm.ec2_cloud_cert]
+
run_cmd(*upload_cmdline)
if self.vm.ec2_register:
- from boto.ec2.connection import EC2Connection
-
- #Code added to support Eucalyptus and other EC2-compatible clusters
- if self.vm.ec2_url:
- import urlparse
- parsed_url = urlparse.urlparse(self.vm.ec2_url)
- conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key,
- host = parsed_url.hostname, port = parsed_url.port, path = parsed_url.path)
- else:
- conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key)
+ logging.info("Registering EC2 bundle")
+ uploaded_image_manifest = '%s/%s.manifest.xml' % (self.vm.ec2_bucket, self.vm.ec2_name)
+ register_cmdline = ['%sregister' % self.ec2_tools_prefix, uploaded_image_manifest, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
+
+ if self.vm.ec2_s3_url:
+ register_cmdline += ['--url', self.vm.ec2_s3_url]
- image_id = conn.register_image('%s/%s.manifest.xml' % (self.vm.ec2_bucket, self.vm.ec2_name))
+ registered_ami = run_cmd(*register_cmdline)
+ image_id = registered_ami.split("\t")[1].strip()
logging.info("EC2 image ID: %s" % image_id)
+
+# Again, changing strategies call for lots of commented out code
+# from boto.ec2.connection import EC2Connection
+# from boto.ec2.regioninfo import RegionInfo
+
+# rinfo = None
+# if self.vm.ec2_is_eucalyptus: #default boto regioninfo assumes Amazon EC2
+# rinfo = RegionInfo(None, "eucalyptus", self.vm.ec2_url)
+
+# #Code added to support Eucalyptus and other EC2-compatible clusters
+# if self.vm.ec2_url:
+# import urlparse
+# parsed_url = urlparse.urlparse(self.vm.ec2_url)
+# is_secure = False
+
+# if parsed_url.scheme == 'https':
+# is_secure = True
+
+# conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key,
+# host = parsed_url.hostname, port = parsed_url.port, path = parsed_url.path, is_secure = is_secure, region = rinfo)
+# else:
+# conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key, region = rinfo)
+
+# image_id = conn.register_image('%s/%s.manifest.xml' % (self.vm.ec2_bucket, self.vm.ec2_name))
else:
self.vm.result_files.append(manifest)
else:
--
Tool for creating VM images.
More information about the Pkg-escience-soc2009
mailing list