[Pkg-escience-soc2009] [SCM] Tool for creating VM images. branch, master, updated. 6f1b989746275565c73fff48c83d10c3195fdad8
David Wendt
dcrkid at yahoo.com
Sat Aug 8 00:23:12 UTC 2009
The following commit has been merged in the master branch:
commit 6f1b989746275565c73fff48c83d10c3195fdad8
Author: David Wendt <dcrkid at yahoo.com>
Date: Fri Aug 7 20:18:49 2009 -0400
A bunch of work on stuff. Now have a working means to build EKI/ERI.
Only problem is that I've recently discovered that building for Xen doesn't give us a kernel. :/
diff --git a/VMBuilder/plugins/debian/distro.py b/VMBuilder/plugins/debian/distro.py
index 11e6ab7..e990112 100644
--- a/VMBuilder/plugins/debian/distro.py
+++ b/VMBuilder/plugins/debian/distro.py
@@ -40,6 +40,7 @@ class Debian(Distro):
'i386' : [ 'i386' ] }
xen_kernel = ''
+ kernel = ''
def register_options(self):
group = self.vm.setting_group('Package options')
@@ -120,7 +121,7 @@ class Debian(Distro):
' '.join(self.valid_archs[self.host_arch])))
if not self.vm.components:
- self.vm.components = ['main', 'restricted', 'universe']
+ self.vm.components = ['main', 'contrib']
else:
if type(self.vm.components) is str:
self.vm.components = self.vm.components.split(',')
@@ -165,7 +166,10 @@ setup (hd0)
EOT''')
def find_linux_kernel(self, suite, flavour, arch):
- rmad = run_cmd('rmadison', 'linux-image-2.6-%s-%s' % (flavour, arch))
+ if flavour == None:
+ rmad = run_cmd('rmadison', 'linux-image-2.6-%s' % (arch))
+ else:
+ rmad = run_cmd('rmadison', 'linux-image-2.6-%s-%s' % (flavour, arch))
version = ['0', '0','0', '0']
for line in rmad.splitlines():
@@ -190,13 +194,13 @@ EOT''')
if version[0] != '0':
return '%s.%s.%s-%s' % (version[0],version[1],version[2],version[3])
-
- return None
+ else:
+ return None
def xen_kernel_version(self):
if self.suite.xen_kernel_flavour:
if not self.xen_kernel:
- logging.debug("Searching for %s-%s flavour Xen kernel..." % (self.suite.xen_kernel_flavour, self.self.suite.default_flavour[self.vm.arch]))
+ logging.debug("Searching for %s-%s flavour Xen kernel..." % (self.suite.xen_kernel_flavour, self.suite.default_flavour[self.vm.arch]))
xen_kernel = self.find_linux_kernel(self.vm.suite, self.suite.xen_kernel_flavour, self.suite.default_flavour[self.vm.arch])
if xen_kernel == None:
@@ -220,6 +224,28 @@ EOT''')
else:
raise VMBuilderUserError('There is no valid xen kernel for the suite selected.')
+ #Ugly kludge to avoid breaking old code that relies on xen_kernel_path
+
+ def kernel_path(self):
+ if not isinstance(self.vm.hypervisor, VMBuilder.plugins.xen.Xen):
+ return self.__kernel_path()
+ else:
+ return self.xen_kernel_path()
+
+ def ramdisk_path(self):
+ if not isinstance(self.vm.hypervisor, VMBuilder.plugins.xen.Xen):
+ return self.__ramdisk_path()
+ else:
+ return self.xen_ramdisk_path()
+
+ def __kernel_path(self):
+ path = '/boot/vmlinuz-%s' % self.find_linux_kernel(self.suite, flavour = None, arch = self.vm.arch)
+ return path
+
+ def __ramdisk_path(self):
+ path = '/boot/initrd.img-%s' % self.find_linux_kernel(self.suite, flavour = None, arch = self.vm.arch)
+ return path
+
def xen_kernel_path(self):
path = '/boot/vmlinuz-%s-%s' % (self.xen_kernel_version(), self.suite.xen_kernel_flavour)
return path
diff --git a/VMBuilder/plugins/debian/etch.py b/VMBuilder/plugins/debian/etch.py
index a73fa73..d5a3326 100644
--- a/VMBuilder/plugins/debian/etch.py
+++ b/VMBuilder/plugins/debian/etch.py
@@ -46,6 +46,7 @@ class Etch(suite.Suite):
virtio_net = False
ec2_kernel_info = { 'i386' : 'aki-9b00e5f2', 'amd64' : 'aki-9800e5f1' }
+ ec2_ramdisk_info = { 'i386' : 'ari-cant-be-arsed', 'amd64' : 'ari-to-look-that-up' }
def check_kernel_flavour(self, arch, flavour):
return flavour in self.valid_flavours[arch]
@@ -349,8 +350,6 @@ class Etch(suite.Suite):
def install_ec2(self):
#Mostly ported over from the Ubuntu install_ec2 commands
- self.run_in_target('apt-get' ,'--force-yes', '-y', 'install', 'libc6-xen')
- self.run_in_target('apt-get','--purge','--force-yes', '-y', 'remove', 'libc6-i686')
#TODO: etch doesn't use Upstart, port over to init scripts
#Basically, it makes it setup a getty console on xvc0
#self.install_from_template('/etc/event.d/xvc0', 'upstart', { 'console' : 'xvc0' })
diff --git a/VMBuilder/plugins/ubuntu/templates/xen-ld-so-conf.tmpl b/VMBuilder/plugins/debian/templates/xen-ld-so-conf.tmpl
similarity index 100%
copy from VMBuilder/plugins/ubuntu/templates/xen-ld-so-conf.tmpl
copy to VMBuilder/plugins/debian/templates/xen-ld-so-conf.tmpl
diff --git a/VMBuilder/plugins/ec2/__init__.py b/VMBuilder/plugins/ec2/__init__.py
index a6e74b4..ee657b8 100644
--- a/VMBuilder/plugins/ec2/__init__.py
+++ b/VMBuilder/plugins/ec2/__init__.py
@@ -26,10 +26,7 @@ class EC2(Plugin):
name = 'EC2 integration'
def register_options(self):
- # Don't pretend like we can do EC2
- if not isinstance(self.vm.hypervisor, VMBuilder.plugins.xen.Xen):
- return
- group = self.vm.setting_group('EC2 integation')
+ group = self.vm.setting_group('EC2 and Eucalyptus support')
group.add_option('--ec2', action='store_true', help='Build for EC2')
group.add_option('--ec2-name','--ec2-prefix', metavar='EC2_NAME', help='Name for the EC2 image.')
group.add_option('--ec2-cert', metavar='CERTFILE', help='PEM encoded public certificate for EC2.')
@@ -48,12 +45,23 @@ class EC2(Plugin):
group.add_option('--ec2-no-amazon-tools', action='store_true', help='Do not use Amazon\'s EC2 AMI tools.')
group.add_option('--ec2-no-euca-tools', action='store_true', help='Do not use Eucalyptus\' EC2 AMI tools.')
group.add_option('--ec2-url', metavar='EC2_URL', help='URL of EC2 or compatible cluster.')
+ group.add_option('--ec2-is-eucalyptus', action='store_true', help='Allow the use of Eucalyptus-specific features like KVM.')
+ group.add_option('--ec2-bundle-kernel', action='store_true', help='Bundle the kernel and ramdisk.')
+ group.add_option('--ec2-upload-kernel', action='store_true', help='Upload the kernel and ramdisk.')
+ group.add_option('--ec2-register-kernel', action='store_true', help='Register the kernel and ramdisk')
self.vm.register_setting_group(group)
def preflight_check(self):
if not getattr(self.vm, 'ec2', False):
return True
+ #NOW, actually check if we can do EC2
+ if not isinstance(self.vm.hypervisor, VMBuilder.plugins.xen.Xen):
+ if not (self.vm.ec2_is_eucalyptus and isinstance(self.vm.hypervisor, VMBuilder.plugins.kvm.KVM)):
+ raise VMBuilderUserError("You must use either Xen or KVM on Eucalyptus")
+ else:
+ raise VMBuilderUserError("You must use Xen on EC2")
+
if self.vm.ec2_no_amazon_tools:
logging.info("Not using Amazon ec2-tools.")
awsec2_installed = False
@@ -110,25 +118,26 @@ class EC2(Plugin):
raise VMBuilderUserError('When building for EC2 you must provide your PEM encoded private key file')
if not self.vm.ec2_user:
- raise VMBuilderUserError('When building for EC2 you must provide your EC2 user ID (your AWS account number, not your AWS access key ID)')
-
- if not self.vm.ec2_kernel:
- self.vm.ec2_kernel = self.vm.distro.get_ec2_kernel()
- logging.debug('%s - to be used for AKI.' %(self.vm.ec2_kernel))
-
- if not self.vm.ec2_ramdisk:
- self.vm.ec2_ramdisk = self.vm.distro.ec2_ramdisk_id()
- logging.debug('%s - to be use for the ARI.' %(self.vm.ec2_ramdisk))
+ if "EC2_USER_ID" in os.environ:
+ self.vm.ec2_user = os.environ["EC2_USER_ID"]
+ else:
+ raise VMBuilderUserError('When building for EC2 you must provide your EC2 user ID (your AWS account number, not your AWS access key ID)')
if self.vm.ec2_upload:
if not self.vm.ec2_bucket:
raise VMBuilderUserError('When building for EC2 you must provide an S3 bucket to hold the AMI')
if not self.vm.ec2_access_key:
- raise VMBuilderUserError('When building for EC2 you must provide your AWS access key ID.')
+ if "EC2_ACCESS_KEY" in os.environ:
+ self.vm.ec2_access_key = os.environ["EC2_ACCESS_KEY"]
+ else:
+ raise VMBuilderUserError('When building for EC2 you must provide your AWS access key ID.')
if not self.vm.ec2_secret_key:
- raise VMBuilderUserError('When building for EC2 you must provide your AWS secret access key.')
+ if "EC2_SECRET_KEY" in os.environ:
+ self.vm.ec2_access_key = os.environ["EC2_SECRET_KEY"]
+ else:
+ raise VMBuilderUserError('When building for EC2 you must provide your AWS secret access key.')
if not self.vm.ec2_version:
raise VMBuilderUserError('When building for EC2 you must provide version info.')
@@ -147,6 +156,75 @@ class EC2(Plugin):
if not getattr(self.vm, 'ec2', False):
return False
+ #We take the time now to upload the kernel and ramdisk, if the user wants it.
+ if self.vm.ec2_bundle_kernel:
+ logging.info("Building EC2 kernel bundle")
+
+ if self.vm.in_place:
+ installdir = self.vm.rootmnt
+ else:
+ installdir = self.vm.tmproot
+
+ kernel_loc = self.vm.distro.kernel_path()
+ ramdisk_loc = self.vm.distro.ramdisk_path()
+
+ bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', "%s%s" % (installdir, kernel_loc), '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', self.vm.ec2_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.workdir, '--kernel', 'true']
+ run_cmd(*bundle_cmdline)
+
+ kernel_name = os.path.split(kernel_loc)[1]
+ kernel_manifest = '%s/%s.manifest.xml' % (self.vm.workdir, kernel_name)
+
+ if ramdisk_loc != None:
+ bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', "%s%s" % (installdir, ramdisk_loc), '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', self.vm.ec2_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.workdir, '--ramdisk', 'true']
+
+ run_cmd(*bundle_cmdline)
+
+ ramdisk_name = os.path.split(ramdisk_loc)[1]
+ ramdisk_manifest = '%s/%s.manifest.xml' % (self.vm.workdir, ramdisk_name)
+
+ if self.vm.ec2_upload_kernel:
+ logging.info("Uploading EC2 kernel bundle")
+
+ upload_cmdline = ['%supload-bundle' % self.ec2_tools_prefix, '--retry', '--manifest', kernel_manifest, '--bucket', self.vm.ec2_bucket, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
+ run_cmd(*bundle_cmdline)
+
+ if ramdisk_loc != None:
+ upload_cmdline = ['%supload-bundle' % self.ec2_tools_prefix, '--retry', '--manifest', ramdisk_manifest, '--bucket', self.vm.ec2_bucket, '--access-key', self.vm.ec2_access_key, '--secret-key', self.vm.ec2_secret_key]
+ run_cmd(*bundle_cmdline)
+
+ if self.vm.ec2.register_kernel:
+ logging.info("Registering EC2 kernel bundle")
+ from boto.ec2.connection import EC2Connection
+
+ #Code added to support Eucalyptus and other EC2-compatible clusters
+ if self.vm.ec2_url:
+ import urlparse
+ parsed_url = urlparse.urlparse(self.vm.ec2_url)
+ conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key,
+ host = parsed_url.hostname, port = parsed_url.port, path = parsed_url.path)
+ else:
+ conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key)
+
+ kernel_id = conn.register_image('%s/%s.manifest.xml' % (self.vm.ec2_bucket, kernel_name))
+
+ logging.info("EC2 kernel ID: %s" % kernel_id)
+
+ self.vm.ec2_kernel = kernel_id
+
+ if ramdisk_loc != None:
+ ramdisk_id = conn.register_image('%s/%s.manifest.xml' % (self.vm.ec2_bucket, ramdisk_name))
+
+ logging.info("EC2 ramdisk ID: %s" % ramdisk_id)
+ self.vm.ec2_ramdisk = ramdisk_id
+
+ if not self.vm.ec2_kernel:
+ self.vm.ec2_kernel = self.vm.distro.get_ec2_kernel()
+ logging.debug('%s - to be used for AKI.' %(self.vm.ec2_kernel))
+
+ if not self.vm.ec2_ramdisk:
+ self.vm.ec2_ramdisk = self.vm.distro.ec2_ramdisk_id()
+ logging.debug('%s - to be use for the ARI.' %(self.vm.ec2_ramdisk))
+
if self.vm.ec2_bundle:
logging.info("Building EC2 bundle")
bundle_cmdline = ['%sbundle-image' % self.ec2_tools_prefix, '--image', self.vm.filesystems[0].filename, '--cert', self.vm.ec2_cert, '--privatekey', self.vm.ec2_key, '--user', self.vm.ec2_user, '--prefix', self.vm.ec2_name, '-r', ['i386', 'x86_64'][self.vm.arch == 'amd64'], '-d', self.vm.workdir, '--kernel', self.vm.ec2_kernel, '--ramdisk', self.vm.ec2_ramdisk]
@@ -170,7 +248,9 @@ class EC2(Plugin):
else:
conn = EC2Connection(self.vm.ec2_access_key, self.vm.ec2_secret_key)
- conn.register_image('%s/%s.manifest.xml' % (self.vm.ec2_bucket, self.vm.ec2_name))
+ image_id = conn.register_image('%s/%s.manifest.xml' % (self.vm.ec2_bucket, self.vm.ec2_name))
+
+ logging.info("EC2 image ID: %s" % image_id)
else:
self.vm.result_files.append(manifest)
else:
diff --git a/VMBuilder/plugins/ubuntu/hardy.py b/VMBuilder/plugins/ubuntu/hardy.py
index 7da2e9f..6e978a0 100644
--- a/VMBuilder/plugins/ubuntu/hardy.py
+++ b/VMBuilder/plugins/ubuntu/hardy.py
@@ -16,7 +16,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
-import suite
+from VMBuilder import suite
import logging
import VMBuilder
from VMBuilder.util import run_cmd
diff --git a/VMBuilder/vm.py b/VMBuilder/vm.py
index 93b8214..9096242 100644
--- a/VMBuilder/vm.py
+++ b/VMBuilder/vm.py
@@ -108,9 +108,6 @@ class VM(object):
def add_clean_cb(self, cb):
self._cleanup_cbs.insert(0, cb)
- def remove_clean_cb(self, cb):
- self._cleanup_cbs.remove(cb)
-
def add_clean_cmd(self, *argv, **kwargs):
cb = lambda : util.run_cmd(*argv, **kwargs)
self.add_clean_cb(cb)
@@ -118,7 +115,7 @@ class VM(object):
def remove_clean_cmd(self, *argv, **kwargs):
cb = lambda : util.run_cmd(*argv, **kwargs)
- self.remove_clean_cb(cb)
+ self.cancel_cleanup(cb)
return cb
def cancel_cleanup(self, cb):
--
Tool for creating VM images.
More information about the Pkg-escience-soc2009
mailing list