# Copyright (c) 2013-2017 Hewlett Packard Enterprise Development LP # # Redistribution and use of this software in source and binary forms, # with or without modification, are permitted provided that the following # conditions are met: # # Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import atexit import base64 import os.path import random import subprocess import sys import tempfile from requestbuilder import Arg, MutuallyExclusiveArgList from requestbuilder.exceptions import ArgumentError import six import euca2ools.bundle.manifest import euca2ools.bundle.util from euca2ools.commands.argtypes import (b64encoded_file_contents, delimited_list, filesize, manifest_block_device_mappings) from euca2ools.commands.bootstrap.describeservicecertificates import \ DescribeServiceCertificates from euca2ools.commands.s3.checkbucket import CheckBucket from euca2ools.commands.s3.createbucket import CreateBucket from euca2ools.commands.s3.getobject import GetObject from euca2ools.commands.s3.postobject import PostObject from euca2ools.commands.s3.putobject import PutObject from euca2ools.exceptions import AWSError EC2_BUNDLE_SIZE_LIMIT = 10 * 2 ** 30 # 10 GiB class BundleCreatingMixin(object): ARGS = [Arg('-i', '--image', metavar='FILE', required=True, help='file containing the image to bundle (required)'), Arg('-p', '--prefix', help='''the file name prefix to give the bundle's files (required when bundling stdin; otherwise defaults to the image's file name)'''), Arg('-d', '--destination', metavar='DIR', help='''location to place the bundle's files (default: dir named by TMPDIR, TEMP, or TMP environment variables, or otherwise /var/tmp)'''), Arg('-r', '--arch', required=True, choices=('i386', 'x86_64', 'armhf', 'ppc', 'ppc64', 'ppc64le'), help="the image's architecture (required)"), # User- and cloud-specific stuff Arg('-k', '--privatekey', metavar='FILE', help='''file containing your private key to sign the bundle's manifest with. If one is not available the bundle will not be signed.'''), Arg('-c', '--cert', metavar='FILE', help='''file containing your X.509 certificate. If one is not available it will not be possible to unbundle the bundle without cloud administrator assistance.'''), Arg('--ec2cert', metavar='FILE', help='''file containing the cloud's X.509 certificate. If one is not available locally it must be available from the bootstrap service.'''), Arg('-u', '--user', metavar='ACCOUNT', help='your account ID'), Arg('--kernel', metavar='IMAGE', help='''ID of the kernel image to associate with this machine image'''), Arg('--ramdisk', metavar='IMAGE', help='''ID of the ramdisk image to associate with this machine image'''), Arg('--bootstrap-url', route_to=None, help='''[Eucalyptus only] bootstrap service endpoint URL (used for obtaining --ec2cert automatically'''), Arg('--bootstrap-service', route_to=None, help=argparse.SUPPRESS), Arg('--bootstrap-auth', route_to=None, help=argparse.SUPPRESS), # Obscurities Arg('-B', '--block-device-mappings', metavar='VIRTUAL1=DEVICE1,VIRTUAL2=DEVICE2,...', type=manifest_block_device_mappings, help='''block device mapping scheme with which to launch instances of this machine image'''), Arg('--productcodes', metavar='CODE1,CODE2,...', type=delimited_list(','), default=[], help='comma-separated list of product codes for the image'), Arg('--image-type', choices=('machine', 'kernel', 'ramdisk'), default='machine', help=argparse.SUPPRESS), # Stuff needed to fill out TarInfo when input comes from stdin. # # We technically could ask for a lot more, but most of it is # unnecessary since owners/modes/etc will be ignored at unbundling # time anyway. # # When bundling stdin we interpret --prefix as the image's file # name. Arg('--image-size', type=filesize, help='''the image's size (required when bundling stdin)'''), # Overrides for debugging and other entertaining uses Arg('--part-size', type=filesize, default=10485760, # 10M help=argparse.SUPPRESS), Arg('--enc-key', type=(lambda s: int(s, 16)), help=argparse.SUPPRESS), # a hex string Arg('--enc-iv', type=(lambda s: int(s, 16)), help=argparse.SUPPRESS), # a hex string # Noop, for compatibility Arg('--batch', action='store_true', help=argparse.SUPPRESS)] # CONFIG METHODS # def configure_bundle_creds(self): # User's account ID (user-level) if not self.args.get('user'): config_val = self.config.get_user_option('account-id') if 'EC2_USER_ID' in os.environ: self.log.debug('using account ID from environment') self.args['user'] = os.getenv('EC2_USER_ID') elif config_val: self.log.debug('using account ID from configuration') self.args['user'] = config_val if self.args.get('user'): self.args['user'] = self.args['user'].replace('-', '') if not self.args.get('user'): raise ArgumentError( 'missing account ID; please supply one with --user') self.log.debug('account ID: %s', self.args['user']) # User's X.509 certificate (user-level in config) if not self.args.get('cert'): config_val = self.config.get_user_option('certificate') if 'EC2_CERT' in os.environ: self.log.debug('using certificate from environment') self.args['cert'] = os.getenv('EC2_CERT') elif 'EUCA_CERT' in os.environ: # used by the NC self.log.debug('using certificate from environment') self.args['cert'] = os.getenv('EUCA_CERT') elif config_val: self.log.debug('using certificate from configuration') self.args['cert'] = config_val if self.args.get('cert'): self.args['cert'] = os.path.expanduser(os.path.expandvars( self.args['cert'])) _assert_is_file(self.args['cert'], 'user certificate') self.log.debug('certificate: %s', self.args.get('cert')) # User's private key (user-level in config) if not self.args.get('privatekey'): config_val = self.config.get_user_option('private-key') if 'EC2_PRIVATE_KEY' in os.environ: self.log.debug('using private key from environment') self.args['privatekey'] = os.getenv('EC2_PRIVATE_KEY') if 'EUCA_PRIVATE_KEY' in os.environ: # used by the NC self.log.debug('using private key from environment') self.args['privatekey'] = os.getenv('EUCA_PRIVATE_KEY') elif config_val: self.log.debug('using private key from configuration') self.args['privatekey'] = config_val if self.args.get('privatekey'): self.args['privatekey'] = os.path.expanduser(os.path.expandvars( self.args['privatekey'])) _assert_is_file(self.args['privatekey'], 'private key') self.log.debug('private key: %s', self.args.get('privatekey')) # Cloud's X.509 cert (region-level in config) if not self.args.get('ec2cert'): config_val = self.config.get_region_option('certificate') if 'EUCALYPTUS_CERT' in os.environ: # This has no EC2 equivalent since they just bundle their cert. self.log.debug('using cloud certificate from environment') self.args['ec2cert'] = os.getenv('EUCALYPTUS_CERT') elif config_val: self.log.debug('using cloud certificate from configuration') self.args['ec2cert'] = config_val elif (self.args.get('bootstrap_service') and self.args.get('bootstrap_auth')): # Sending requests during configure() can be precarious. # Pay close attention to ordering to ensure all # of this request's dependencies have been fulfilled. try: fetched_cert = self.__get_bundle_certificate( self.args['bootstrap_service'], self.args['bootstrap_auth']) except AWSError as err: self.log.debug('failed to fetch ec2cert', exc_info=True) if err.response.status_code == 403: msg = ('permission error retrieving cloud ' 'certificate; please supply one with ' '--ec2cert or obtain an IAM policy that ' 'allows "euserv:DescribeServiceCertificates"') else: msg = ('error retrieving cloud certificate; please ' 'supply one with --ec2cert') six.raise_from(ArgumentError(msg), err) if fetched_cert: self.log.debug('using cloud certificate from ' 'bootstrap service') self.args['ec2cert'] = fetched_cert if self.args.get('ec2cert'): self.args['ec2cert'] = os.path.expanduser(os.path.expandvars( self.args['ec2cert'])) _assert_is_file(self.args['ec2cert'], 'cloud certificate') if not self.args.get('ec2cert'): raise ArgumentError( 'missing cloud certificate; please supply one with ' '--ec2cert or use --bootstrap-url and access keys to ' 'fetch one automatically') self.log.debug('cloud certificate: %s', self.args['ec2cert']) def configure_bundle_output(self): if (self.args.get('destination') and os.path.exists(self.args['destination']) and not os.path.isdir(self.args['destination'])): raise ArgumentError("argument -d/--destination: '{0}' is not a " "directory".format(self.args['destination'])) if self.args['image'] == '-': self.args['image'] = os.fdopen(os.dup(sys.stdin.fileno())) if not self.args.get('prefix'): raise ArgumentError( 'argument --prefix is required when bundling stdin') if not self.args.get('image_size'): raise ArgumentError( 'argument --image-size is required when bundling stdin') elif isinstance(self.args['image'], six.string_types): if not self.args.get('prefix'): self.args['prefix'] = os.path.basename(self.args['image']) if not self.args.get('image_size'): self.args['image_size'] = euca2ools.util.get_filesize( self.args['image']) self.args['image'] = open(self.args['image']) else: # Assume it is already a file object if not self.args.get('prefix'): raise ArgumentError('argument --prefix is required when ' 'bundling a file object') if not self.args.get('image_size'): raise ArgumentError('argument --image-size is required when ' 'bundling a file object') if self.args['image_size'] > EC2_BUNDLE_SIZE_LIMIT: self.log.warn( 'image is incompatible with EC2 due to its size (%i > %i)', self.args['image_size'], EC2_BUNDLE_SIZE_LIMIT) def configure_bundle_properties(self): if self.args.get('kernel') == 'true': self.args['image_type'] = 'kernel' if self.args.get('ramdisk') == 'true': self.args['image_type'] = 'ramdisk' if self.args['image_type'] == 'kernel': if self.args.get('kernel') and self.args['kernel'] != 'true': raise ArgumentError("argument --kernel: not compatible with " "image type 'kernel'") if self.args.get('ramdisk'): raise ArgumentError("argument --ramdisk: not compatible with " "image type 'kernel'") if self.args.get('block_device_mappings'): raise ArgumentError("argument -B/--block-device-mappings: not " "compatible with image type 'kernel'") if self.args['image_type'] == 'ramdisk': if self.args.get('kernel'): raise ArgumentError("argument --kernel: not compatible with " "image type 'ramdisk'") if self.args.get('ramdisk') and self.args['ramdisk'] != 'true': raise ArgumentError("argument --ramdisk: not compatible with " "image type 'ramdisk'") if self.args.get('block_device_mappings'): raise ArgumentError("argument -B/--block-device-mappings: not " "compatible with image type 'ramdisk'") def generate_encryption_keys(self): srand = random.SystemRandom() if self.args.get('enc_key'): self.log.info('using preexisting encryption key') enc_key_i = self.args['enc_key'] else: enc_key_i = srand.getrandbits(128) if self.args.get('enc_iv'): self.log.info('using preexisting encryption IV') enc_iv_i = self.args['enc_iv'] else: enc_iv_i = srand.getrandbits(128) self.args['enc_key'] = '{0:0>32x}'.format(enc_key_i) self.args['enc_iv'] = '{0:0>32x}'.format(enc_iv_i) def __get_bundle_certificate(self, bootstrap_service, bootstrap_auth): self.log.info('attempting to obtain cloud certificate from ' 'bootstrap service') req = DescribeServiceCertificates( config=self.config, loglevel=self.log.level, service=bootstrap_service, auth=bootstrap_auth, Format='pem', FingerprintDigest='SHA-256') response = req.main() for cert in response.get('serviceCertificates') or []: if (cert.get('certificateUsage') == 'image-bundling' and cert.get('serviceType') == 'compute'): cert_file = tempfile.NamedTemporaryFile(delete=False) cert_file.write(cert['certificate']) cert_file.file.flush() self.args['ec2cert'] = cert_file.name atexit.register(os.remove, cert_file.name) return cert_file.name # MANIFEST GENERATION METHODS # def build_manifest(self, digest, partinfo): manifest = euca2ools.bundle.manifest.BundleManifest( loglevel=self.log.level) manifest.image_arch = self.args['arch'] manifest.kernel_id = self.args.get('kernel') manifest.ramdisk_id = self.args.get('ramdisk') if self.args.get('block_device_mappings'): manifest.block_device_mappings.update( self.args['block_device_mappings']) if self.args.get('productcodes'): manifest.product_codes.extend(self.args['productcodes']) manifest.image_name = self.args['prefix'] manifest.account_id = self.args['user'] manifest.image_type = self.args['image_type'] manifest.image_digest = digest manifest.image_digest_algorithm = 'SHA1' # shouldn't be hardcoded here manifest.image_size = self.args['image_size'] manifest.bundled_image_size = sum(part.size for part in partinfo) manifest.enc_key = self.args['enc_key'] manifest.enc_iv = self.args['enc_iv'] manifest.enc_algorithm = 'AES-128-CBC' # shouldn't be hardcoded here manifest.image_parts = partinfo return manifest def dump_manifest_to_file(self, manifest, filename, pretty_print=False): with open(filename, 'w') as manifest_file: manifest_file.write(self.dump_manifest_to_str( manifest, pretty_print=pretty_print)) def dump_manifest_to_str(self, manifest, pretty_print=False): return manifest.dump_to_str(self.args['privatekey'], self.args['cert'], self.args['ec2cert'], pretty_print=pretty_print) class BundleUploadingMixin(object): ARGS = [Arg('-b', '--bucket', metavar='BUCKET[/PREFIX]', required=True, help='bucket to upload the bundle to (required)'), Arg('--acl', default='aws-exec-read', choices=('public-read', 'aws-exec-read', 'ec2-bundle-read'), help='''canned ACL policy to apply to the bundle (default: aws-exec-read)'''), MutuallyExclusiveArgList( Arg('--upload-policy', dest='upload_policy', metavar='POLICY', type=base64.b64encode, help='upload policy to use for authorization'), Arg('--upload-policy-file', dest='upload_policy', metavar='FILE', type=b64encoded_file_contents, help='''file containing an upload policy to use for authorization''')), Arg('--upload-policy-signature', metavar='SIGNATURE', help='''signature for the upload policy (required when an 'upload policy is used)'''), Arg('--location', help='''location constraint of the destination bucket (default: inferred from s3-location-constraint in configuration, or otherwise none)'''), Arg('--retry', dest='retries', action='store_const', const=5, default=0, help='retry failed uploads up to 5 times')] def configure_bundle_upload_auth(self): if self.args.get('upload_policy'): if not self.args.get('key_id'): raise ArgumentError('-I/--access-key-id is required when ' 'using an upload policy') if not self.args.get('upload_policy_signature'): raise ArgumentError('--upload-policy-signature is required ' 'when using an upload policy') self.auth = None self.AUTH_CLASS = None def get_bundle_key_prefix(self): (bucket, _, prefix) = self.args['bucket'].partition('/') if prefix and not prefix.endswith('/'): prefix += '/' return bucket + '/' + prefix def ensure_dest_bucket_exists(self): if self.args.get('upload_policy'): # We won't have creds to sign our own requests self.log.info('using an upload policy; not verifying bucket ' 'existence') return bucket = self.args['bucket'].split('/', 1)[0] try: req = CheckBucket.from_other(self, bucket=bucket) req.main() except AWSError as err: if err.status_code == 404: # No such bucket self.log.info("creating bucket '%s'", bucket) req = CreateBucket.from_other( self, bucket=bucket, location=self.args.get('location')) req.main() else: raise # At this point we know we can at least see the bucket, but it's still # possible that we can't write to it with the desired key names. So # many policies are in play here that it isn't worth trying to be # proactive about it. def upload_bundle_file(self, source, dest, show_progress=False, **putobj_kwargs): if self.args.get('upload_policy'): if show_progress: # PostObject does not yet support show_progress print source, 'uploading...' if self.args.get('security_token'): postobj_kwargs = \ {'x-amz-security-token': self.args['security_token']} else: postobj_kwargs = {} postobj_kwargs.update(putobj_kwargs) req = PostObject.from_other( self, source=source, dest=dest, acl=self.args.get('acl') or 'aws-exec-read', Policy=self.args['upload_policy'], Signature=self.args['upload_policy_signature'], AWSAccessKeyId=self.args['key_id'], **postobj_kwargs) else: req = PutObject.from_other( self, source=source, dest=dest, acl=self.args.get('acl') or 'aws-exec-read', retries=self.args.get('retries') or 0, show_progress=show_progress, **putobj_kwargs) req.main() def upload_bundle_parts(self, partinfo_in_mpconn, key_prefix, partinfo_out_mpconn=None, part_write_sem=None, **putobj_kwargs): try: while True: part = partinfo_in_mpconn.recv() dest = key_prefix + os.path.basename(part.filename) self.upload_bundle_file(part.filename, dest, **putobj_kwargs) if part_write_sem is not None: # Allow something that's waiting for the upload to finish # to continue part_write_sem.release() if partinfo_out_mpconn is not None: partinfo_out_mpconn.send(part) except EOFError: return finally: partinfo_in_mpconn.close() if partinfo_out_mpconn is not None: partinfo_out_mpconn.close() class BundleDownloadingMixin(object): # When fetching the manifest from the server there are two ways to get # its path: # -m: BUCKET[/PREFIX]/MANIFEST # -p: BUCKET[/PREFIX]/PREFIX.manifest.xml (the PREFIXes are different) # # In all cases, after we obtain the manifest (whether it is local or not) # we choose key names for parts based on the file names in the manifest: # BUCKET[/PREFIX]/PART ARGS = [Arg('-b', '--bucket', metavar='BUCKET[/PREFIX]', required=True, route_to=None, help='''the bucket that contains the bundle, with an optional path prefix (required)'''), MutuallyExclusiveArgList( Arg('-m', '--manifest', dest='manifest', route_to=None, help='''the manifest's complete file name, not including any path that may be specified using -b'''), Arg('-p', '--prefix', dest='manifest', route_to=None, type=(lambda x: x + '.manifest.xml'), help='''the portion of the manifest's file name that precedes ".manifest.xml"'''), Arg('--local-manifest', dest='local_manifest', metavar='FILE', route_to=None, help='''use a manifest on disk and ignore any that appear on the server''')) .required()] def fetch_manifest(self, s3_service, privkey_filename=None): if self.args.get('local_manifest'): _assert_is_file(self.args['local_manifest'], 'manifest') return euca2ools.bundle.manifest.BundleManifest.read_from_file( self.args['local_manifest'], privkey_filename=privkey_filename) # It's on the server, so do things the hard way manifest_s3path = self.get_manifest_s3path() with tempfile.TemporaryFile() as manifest_tempfile: self.log.info('reading manifest from %s', manifest_s3path) req = GetObject.from_other( self, service=s3_service, source=manifest_s3path, dest=manifest_tempfile) try: req.main() except AWSError as err: if err.status_code == 404: self.log.debug('failed to fetch manifest', exc_info=True) raise ValueError("manifest '{0}' does not exist on the " "server".format(manifest_s3path)) raise manifest_tempfile.flush() manifest_tempfile.seek(0) return euca2ools.bundle.manifest.BundleManifest.read_from_fileobj( manifest_tempfile, privkey_filename=privkey_filename) def get_manifest_s3path(self): if self.args.get('manifest'): return '/'.join((self.args['bucket'], self.args['manifest'])) else: # With a local manifest we can't divine the manifest's key name is return None def download_bundle_to_dir(self, manifest, dest_dir, s3_service): parts = self.map_bundle_parts_to_s3paths(manifest) for part, part_s3path in parts: part.filename = os.path.join(dest_dir, os.path.basename(part_s3path)) self.log.info('downloading part %s to %s', part_s3path, part.filename) req = GetObject.from_other( self, service=s3_service, source=part_s3path, dest=part.filename, show_progress=self.args.get('show_progress', False)) response = req.main() self.__check_part_sha1(part, part_s3path, response) manifest_s3path = self.get_manifest_s3path() if manifest_s3path: # Can't download a manifest if we're using a local one manifest_dest = os.path.join(dest_dir, os.path.basename(manifest_s3path)) self.log.info('downloading manifest %s to %s', manifest_s3path, manifest_dest) req = GetObject.from_other( self, service=s3_service, source=manifest_s3path, dest=manifest_dest, show_progress=self.args.get('show_progress', False)) req.main() return manifest_dest return None def download_bundle_to_fileobj(self, manifest, fileobj, s3_service): # We can skip downloading the manifest since we're just writing all # parts to a file object. parts = self.map_bundle_parts_to_s3paths(manifest) for part, part_s3path in parts: self.log.info('downloading part %s', part_s3path) req = GetObject.from_other( self, service=s3_service, source=part_s3path, dest=fileobj, show_progress=self.args.get('show_progress', False)) response = req.main() self.__check_part_sha1(part, part_s3path, response) def map_bundle_parts_to_s3paths(self, manifest): parts = [] for part in manifest.image_parts: parts.append((part, '/'.join((self.args['bucket'], part.filename)))) return parts def __check_part_sha1(self, part, part_s3path, response): if response[part_s3path]['sha1'] != part.hexdigest: self.log.error('rejecting download due to manifest SHA1 ' 'mismatch (expected: %s, actual: %s)', part.hexdigest, response[part_s3path]['sha1']) raise RuntimeError('downloaded file {0} appears to be corrupt ' '(expected SHA1: {0}, actual: {1}' .format(part.hexdigest, response[part_s3path]['sha1'])) def _assert_is_file(filename, filetype): if not os.path.exists(filename): raise ArgumentError("{0} file '{1}' does not exist" .format(filetype, filename)) if not os.path.isfile(filename): raise ArgumentError("{0} file '{1}' is not a file" .format(filetype, filename)) def _get_cert_fingerprint(cert_content): popen = subprocess.Popen(('openssl', 'x509', '-fingerprint', '-noout'), stdin=subprocess.PIPE, stdout=subprocess.PIPE) return popen.communicate(cert_content)[0].strip()