# Copyright (c) 2009-2017 Hewlett Packard Enterprise Development LP # # Redistribution and use of this software in source and binary forms, # with or without modification, are permitted provided that the following # conditions are met: # # Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import argparse import os.path import pipes import shutil import subprocess import sys import tempfile import time from requestbuilder import Arg, MutuallyExclusiveArgList from requestbuilder.command import BaseCommand from requestbuilder.exceptions import ArgumentError, ClientError from requestbuilder.mixins import (FileTransferProgressBarMixin, RegionConfigurableMixin) import requests import euca2ools from euca2ools.commands import Euca2ools, SYSCONFDIR from euca2ools.commands.argtypes import (delimited_list, filesize, manifest_block_device_mappings) from euca2ools.commands.bootstrap import BootstrapRequest from euca2ools.commands.bundle.bundleimage import BundleImage ALLOWED_FILESYSTEM_TYPES = ['btrfs', 'ext2', 'ext3', 'ext4', 'jfs', 'xfs'] EXCLUDES_FILE = os.path.join(SYSCONFDIR, 'bundle-vol', 'excludes') FSTAB_TEMPLATE_FILE = os.path.join(SYSCONFDIR, 'bundle-vol', 'fstab') class BundleVolume(BaseCommand, FileTransferProgressBarMixin, RegionConfigurableMixin): SUITE = Euca2ools DESCRIPTION = ("Prepare this machine's filesystem for use in the cloud\n\n" "This command must be run as the superuser.") REGION_ENVVAR = ('EUCA_DEFAULT_REGION', 'AWS_DEFAULT_REGION') ARGS = [Arg('-p', '--prefix', help='''the file name prefix to give the bundle's files (default: image)'''), Arg('-d', '--destination', metavar='DIR', help='''location to place the bundle's files (default: dir named by TMPDIR, TEMP, or TMP environment variables, or otherwise /var/tmp)'''), # -r/--arch is required, but to keep the UID check we do at the # beginning of configure() first we enforce that there instead. Arg('-r', '--arch', help="the image's architecture (required)", choices=('i386', 'x86_64', 'armhf', 'ppc', 'ppc64', 'ppc64le')), Arg('-e', '--exclude', metavar='PATH,...', type=delimited_list(','), help='comma-separated list of paths to exclude'), Arg('-i', '--include', metavar='PATH,...', type=delimited_list(','), help='comma-separated list of paths to include'), Arg('-s', '--size', metavar='MiB', type=int, default=10240, help='size of the image to create (default: 10240 MiB)'), Arg('--no-filter', action='store_true', help='do not filter out sensitive/system files'), Arg('--all', action='store_true', help='''include all filesystems regardless of type (default: only include local filesystems)'''), MutuallyExclusiveArgList( Arg('--inherit', dest='inherit', action='store_true', help='''use the metadata service to provide metadata for the bundle (this is the default)'''), Arg('--no-inherit', dest='inherit', action='store_false', help='''do not use the metadata service for bundle metadata''')), Arg('-v', '--volume', metavar='DIR', default='/', help='''location of the volume from which to create the bundle (default: /)'''), Arg('-P', '--partition', choices=('mbr', 'gpt', 'none'), help='''the type of partition table to create (default: attempt to guess based on the existing disk)'''), Arg('--script', metavar='FILE', help='''location of a script to run immediately before bundling. It will receive the volume's mount point as its only argument.'''), MutuallyExclusiveArgList( Arg('--fstab', metavar='FILE', help='''location of an fstab(5) file to copy into the bundled image'''), Arg('--generate-fstab', action='store_true', help='''automatically generate an fstab(5) file for the bundled image''')), Arg('--grub-config', metavar='FILE', help='''location of a GRUB 1 configuration file to copy to /boot/grub/menu.lst on the bundled image'''), # User- and cloud-specific stuff for bundling Arg('-k', '--privatekey', metavar='FILE', help='''file containing your private key to sign the bundle's manifest with. This private key will also be required to unbundle the image in the future.'''), Arg('-c', '--cert', metavar='FILE', help='file containing your X.509 certificate'), Arg('--ec2cert', metavar='FILE', help='''file containing the cloud's X.509 certificate'''), Arg('-u', '--user', metavar='ACCOUNT', help='your account ID'), Arg('--kernel', metavar='IMAGE', help='''ID of the kernel image to associate with this machine image'''), Arg('--ramdisk', metavar='IMAGE', help='''ID of the ramdisk image to associate with this machine image'''), Arg('--bootstrap-url', route_to=None, help='''[Eucalyptus only] bootstrap service endpoint URL (used for obtaining --ec2cert automatically'''), Arg('--bootstrap-service', route_to=None, help=argparse.SUPPRESS), Arg('--bootstrap-auth', route_to=None, help=argparse.SUPPRESS), BootstrapRequest.AUTH_CLASS.ARGS, # Obscurities Arg('-B', '--block-device-mappings', metavar='VIRTUAL1=DEVICE1,VIRTUAL2=DEVICE2,...', type=manifest_block_device_mappings, help='''block device mapping scheme with which to launch instances of this machine image'''), Arg('--productcodes', metavar='CODE1,CODE2,...', type=delimited_list(','), default=[], help='comma-separated list of product codes for the image'), # Overrides for debugging and other entertaining uses Arg('--part-size', type=filesize, default=10485760, # 10MB help=argparse.SUPPRESS), Arg('--enc-key', type=(lambda s: int(s, 16)), help=argparse.SUPPRESS), # a hex string Arg('--enc-iv', type=(lambda s: int(s, 16)), help=argparse.SUPPRESS)] # a hex string def configure(self): if os.geteuid() != 0: raise RuntimeError('must be superuser') if not self.args.get('arch'): raise ArgumentError('argument -r/--arch is required') # Farm all the bundle arg validation out to BundleImage self.__build_bundle_command('/dev/null', image_size=1) root_device = _get_root_device() if self.args.get('inherit'): self.__populate_args_from_metadata() if not self.args.get('partition'): self.args['partition'] = _get_partition_table_type(root_device) if not self.args['partition']: self.log.warn('could not determine the partition table type ' 'for root device %s', root_device) raise ArgumentError( 'could not determine the type of partition table to use ' 'for disk {0}; specify one with -P/--partition' .format(root_device)) self.log.info('discovered partition table type %s', self.args['partition']) if not self.args.get('fstab') and not self.args.get('generate_fstab'): self.args['fstab'] = '/etc/fstab' def main(self): if self.args.get('destination'): destdir = self.args['destination'] else: destdir = euca2ools.util.mkdtemp_for_large_files(prefix='bundle-') image = os.path.join(destdir, self.args.get('prefix') or 'image') mountpoint = tempfile.mkdtemp(prefix='target-', dir=destdir) # Prepare the disk image device = self.__create_disk_image(image, self.args['size']) try: fsinfo = self.__create_and_mount_filesystem(device, mountpoint) try: # Copy files exclude_opts = self.__get_exclude_and_include_args() exclude_opts.extend(['--exclude', image, '--exclude', mountpoint]) self.__copy_to_target_dir(mountpoint, exclude_opts) self.__insert_fstab(mountpoint) self.__insert_grub_config(mountpoint) if self.args.get('script'): cmd = [self.args['script'], mountpoint] self.log.info("running user script ``%s''", _quote_cmd(cmd)) subprocess.check_call(cmd) except KeyboardInterrupt: self.log.info('received ^C; skipping to cleanup') msg = ('\n\nCleaning up after ^C\nWARNING: pressing ' '^C again will result in the need for manual ' 'device cleanup\n\n') print >> sys.stderr, msg raise # Cleanup finally: time.sleep(0.2) self.__unmount_filesystem(device) os.rmdir(mountpoint) self.__update_filesystem_ids(device, fsinfo) finally: self.__detach_disk_image(image, device) bundle_cmd = self.__build_bundle_command(image) result = bundle_cmd.main() os.remove(image) return result # pylint: disable=no-self-use def print_result(self, result): for manifest_filename in result[1]: print 'Wrote manifest', manifest_filename # pylint: enable=no-self-use def __build_bundle_command(self, image_filename, image_size=None): bundle_args = ('prefix', 'destination', 'arch', 'privatekey', 'cert', 'ec2cert', 'user', 'kernel', 'ramdisk', 'block_device_mappings', 'productcodes', 'part_size', 'enc_key', 'enc_iv', 'show_progress', 'key_id', 'secret_key', 'security_token', 'bootstrap_url', 'bootstrap_service', 'bootstrap_auth', 'region') bundle_args_dict = dict((key, self.args.get(key)) for key in bundle_args) return BundleImage.from_other(self, image=image_filename, image_size=image_size, image_type='machine', **bundle_args_dict) # INSTANCE METADATA # def __read_metadata_value(self, path): self.log.debug("reading metadata service value '%s'", path) url = 'http://169.254.169.254/2012-01-12/meta-data/' + path response = requests.get(url, timeout=1) if response.status_code == 200: return response.text return None def __read_metadata_list(self, path): value = self.__read_metadata_value(path) if value: return [line.rstrip('/') for line in value.splitlines() if line] return [] def __read_metadata_dict(self, path): metadata = {} if not path.endswith('/'): path += '/' keys = self.__read_metadata_list(path) for key in keys: if key: metadata[key] = self.__read_metadata_value(path + key) return metadata def __populate_args_from_metadata(self): """ Populate missing/empty values in self.args using info obtained from the metadata service. """ try: if not self.args.get('kernel'): self.args['kernel'] = self.__read_metadata_value('kernel-id') self.log.info('inherited kernel: %s', self.args['kernel']) if not self.args.get('ramdisk'): self.args['ramdisk'] = self.__read_metadata_value('ramdisk-id') self.log.info('inherited ramdisk: %s', self.args['ramdisk']) if not self.args.get('productcodes'): self.args['productcodes'] = self.__read_metadata_list( 'product-codes') if self.args['productcodes']: self.log.info('inherited product codes: %s', ','.join(self.args['productcodes'])) if not self.args.get('block_device_mappings'): self.args['block_device_mappings'] = {} for key, val in (self.__read_metadata_dict( 'block-device-mapping') or {}).items(): if not key.startswith('ebs'): self.args['block_device_mappings'][key] = val for key, val in self.args['block_device_mappings'].items(): self.log.info('inherited block device mapping: %s=%s', key, val) except requests.exceptions.Timeout: raise ClientError('metadata service is absent or unresponsive; ' 'use --no-inherit to proceed without it') # DISK MANAGEMENT # def __create_disk_image(self, image, size_in_mb): subprocess.check_call(['dd', 'if=/dev/zero', 'of={0}'.format(image), 'bs=1M', 'count=1', 'seek={0}'.format(int(size_in_mb) - 1)]) if self.args['partition'] == 'mbr': # Why use sfdisk when we can use parted? :-) parted_script = ( b'unit s', b'mklabel msdos', b'mkpart primary 64 -1s', b'set 1 boot on', b'print', b'quit') subprocess.check_call(['parted', '-s', image, '--', ' '.join(parted_script)]) elif self.args['partition'] == 'gpt': # type 0xef02 == BIOS boot (we'll put it at the end of the list) subprocess.check_call( ['sgdisk', '--new', '128:1M:+1M', '--typecode', '128:ef02', '--change-name', '128:BIOS Boot', image]) # type 0x8300 == Linux filesystem data subprocess.check_call( ['sgdisk', '--largest-new=1', '--typecode', '1:8300', '--change-name', '1:Image', image]) subprocess.check_call(['sgdisk', '--print', image]) mapped = self.__map_disk_image(image) assert os.path.exists(mapped) return mapped def __map_disk_image(self, image): if self.args['partition'] in ('mbr', 'gpt'): # Create /dev/mapper/loopXpY and return that. # We could do this with losetup -Pf as well, but that isn't # available on RHEL 6. self.log.debug('mapping partitioned image %s', image) kpartx = subprocess.Popen(['kpartx', '-s', '-v', '-a', image], stdout=subprocess.PIPE) try: for line in kpartx.stdout.readlines(): line_split = line.split() if line_split[:2] == ['add', 'map']: device = line_split[2] if device.endswith('p1'): return '/dev/mapper/{0}'.format(device) self.log.error('failed to get usable map output from kpartx') raise RuntimeError('device mapping failed') finally: # Make sure the process exits kpartx.communicate() else: # No partition table self.log.debug('mapping unpartitioned image %s', image) losetup = subprocess.Popen(['losetup', '-f', image, '--show'], stdout=subprocess.PIPE) loopdev, _ = losetup.communicate() return loopdev.strip() def __create_and_mount_filesystem(self, device, mountpoint): root_device = _get_root_device() fsinfo = _get_filesystem_info(root_device) self.log.info('creating filesystem on %s using metadata from %s: %s', device, root_device, fsinfo) fs_cmds = [['mkfs', '-t', fsinfo['type']]] for fs_cmd in fs_cmds: fs_cmd.append(device) self.log.info("formatting with ``%s''", _quote_cmd(fs_cmd)) subprocess.check_call(fs_cmd) self.log.info('mounting %s filesystem %s at %s', fsinfo['type'], device, mountpoint) subprocess.check_call(['mount', '-t', fsinfo['type'], device, mountpoint]) return fsinfo def __unmount_filesystem(self, device): self.log.info('unmounting %s', device) subprocess.check_call(['sync']) time.sleep(0.2) subprocess.check_call(['umount', device]) def __update_filesystem_ids(self, device, fsinfo): """ Apply filesystem identifiers to an unmounted filesystem. To avoid UUID conflicts, run this only after the filesystem no longer needs to be mounted on the running system. """ options = [] if fsinfo.get('label'): options.extend(('-L', fsinfo['label'])) if fsinfo.get('uuid'): options.extend(('-U', fsinfo['uuid'])) if fsinfo.get('type') in ('ext2', 'ext3', 'ext4'): # Time-based checking doesn't make much sense for cloud images options.extend(('-i', '0')) if options: if fsinfo.get('type') in ('ext2', 'ext3', 'ext4'): cmd = ['tune2fs'] + options elif fsinfo.get('type') == 'jfs': cmd = ['jfs_admin'] + options elif fsinfo.get('type') == 'xfs': cmd = ['xfs_admin'] + options cmd.append(device) self.log.info("updating device %s filesystem IDs with ``%s''", device, _quote_cmd(cmd)) subprocess.check_call(cmd) def __detach_disk_image(self, image, device): subprocess.check_call(['sync']) if self.args['partition'] in ('mbr', 'gpt'): self.log.debug('unmapping partitioned image %s', image) cmd = ['kpartx', '-s', '-d', image] else: self.log.debug('unmapping unpartitioned device %s', device) cmd = ['losetup', '-d', device] subprocess.check_call(cmd) # FILE MANAGEMENT # def __get_exclude_and_include_args(self): args = [] for exclude in self.args.get('exclude') or []: args.extend(['--exclude', exclude]) for include in self.args.get('include') or []: args.extend(['--include', include]) # Exclude remote filesystems if not self.args.get('all'): for device, mountpoint, fstype in _get_all_mounts(): if fstype not in ALLOWED_FILESYSTEM_TYPES: self.log.debug('excluding %s filesystem %s at %s', fstype, device, mountpoint) args.extend(['--exclude', os.path.join(mountpoint, '**')]) # Add pre-defined exclusions if not self.args.get('no_filter') and os.path.isfile(EXCLUDES_FILE): self.log.debug('adding path exclusions from %s', EXCLUDES_FILE) args.extend(['--exclude-from', EXCLUDES_FILE]) return args def __copy_to_target_dir(self, dest, exclude_opts): source = self.args.get('volume') or '/' if not source.endswith('/'): source += '/' if not dest.endswith('/'): dest += '/' rsync_opts = ['-rHlpogDtS'] if self.args.get('show_progress'): rsync = subprocess.Popen(['rsync', '--version'], stdout=subprocess.PIPE) out, _ = rsync.communicate() rsync_version = (out.partition('version ')[2] or '\0').split()[0] if rsync_version >= '3.1.0': # Use the new summarizing version rsync_opts.append('--info=progress2') else: rsync_opts.append('--progress') else: rsync_opts.append('--quiet') cmd = ['rsync', '-X'] + rsync_opts + exclude_opts + [source, dest] self.log.info("copying files with ``%s''", _quote_cmd(cmd)) print 'Copying files...' rsync = subprocess.Popen(cmd) rsync.wait() if rsync.returncode == 1: # Try again without xattrs self.log.info('rsync exited with code %i; retrying without xattrs', rsync.returncode) print 'Retrying without extended attributes' cmd = ['rsync'] + rsync_opts + exclude_opts + [source, dest] rsync = subprocess.Popen(cmd) rsync.wait() if rsync.returncode not in (0, 23): self.log.error('rsync exited with code %i', rsync.returncode) raise subprocess.CalledProcessError(rsync.returncode, 'rsync') def __insert_fstab(self, mountpoint): fstab_filename = os.path.join(mountpoint, 'etc', 'fstab') if os.path.exists(fstab_filename): fstab_bak = fstab_filename + '.bak' self.log.debug('backing up original fstab file as %s', fstab_bak) _copy_with_xattrs(fstab_filename, fstab_bak) if self.args.get('generate_fstab'): # This isn't really a template, but if the need arises we # can add something of that sort later. self.log.info('generating fstab file from %s', self.args['fstab']) _copy_with_xattrs(FSTAB_TEMPLATE_FILE, fstab_filename) elif self.args.get('fstab'): self.log.info('using fstab file %s', self.args['fstab']) _copy_with_xattrs(self.args['fstab'], fstab_filename) def __insert_grub_config(self, mountpoint): if self.args.get('grub_config'): grub_filename = os.path.join(mountpoint, 'boot', 'grub', 'menu.lst') if os.path.exists(grub_filename): grub_back = grub_filename + '.bak' self.log.debug('backing up original grub1 config file as %s', grub_back) _copy_with_xattrs(grub_filename, grub_back) self.log.info('using grub1 config file %s', self.args['grub_config']) _copy_with_xattrs(self.args['grub_config'], grub_filename) def _get_all_mounts(): # This implementation is Linux-specific # We first load everything into a dict based on mount points so we # can return only the last filesystem to be mounted in each # location. This is important for / on Linux, where a rootfs # volume has a real filesystem mounted on top of it, because # returning both of them will cause / to get excluded due to its # filesystem type. filesystems_dict = {} with open('/proc/mounts') as mounts: for line in mounts: device, mountpoint, fstype, _ = line.split(None, 3) filesystems_dict[mountpoint] = (device, fstype) filesystems_list = [] for mountpoint, (device, fstype) in filesystems_dict.items(): filesystems_list.append((device, mountpoint, fstype)) return filesystems_list def _get_filesystem_info(device): blkid = subprocess.Popen(['blkid', '-o', 'export', device], stdout=subprocess.PIPE) fsinfo = {} for line in blkid.stdout: key, _, val = line.strip().partition('=') if key == 'LABEL': fsinfo['label'] = val elif key == 'TYPE': fsinfo['type'] = val elif key == 'UUID': fsinfo['uuid'] = val blkid.wait() if blkid.returncode != 0: raise subprocess.CalledProcessError(blkid.returncode, 'blkid') return fsinfo def _get_partition_table_type(device, debug=False): if device[-1] in '0123456789': if device[-2] == 'd': # /dev/sda1, /dev/xvda1, /dev/vda1, etc. device = device[:-1] elif device[-2] == 'p': # /dev/loop0p1, /dev/sr0p1, etc. device = device[:-2] if debug: stderr_dest = subprocess.PIPE else: stderr_dest = None parted = subprocess.Popen(['parted', '-m', '-s', device, 'print'], stdout=subprocess.PIPE, stderr=stderr_dest) stdout, _ = parted.communicate() for line in stdout.splitlines(): if line.startswith('/dev/'): # /dev/sda:500GB:scsi:512:512:msdos:ATA WDC WD5003ABYX-1; line_bits = line.split(':') if line_bits[5] == 'msdos': return 'mbr' elif line_bits[5] == 'gpt': return 'gpt' else: return 'none' def _get_root_device(): root_device = None for device, mountpoint, _ in _get_all_mounts(): if mountpoint == '/' and os.path.exists(device): root_device = device # Do not skip the rest of the mount points. Another # / filesystem, such as a btrfs subvolume, may be # mounted on top of that. if not root_device: raise KeyError('no / filesystem found') return root_device def _quote_cmd(cmd): return ' '.join(pipes.quote(arg) for arg in cmd) def _copy_with_xattrs(source, dest): """ shutil.copy2 doesn't preserve xattrs until python 3.3, so here we attempt to leverage the cp command to do it for us. """ try: subprocess.check_call(['cp', '-a', source, dest]) except subprocess.CalledProcessError: shutil.copy2(source, dest)