On Thu, 2017-08-17 at 10:39 +0100, Radostin Stoyanov wrote: > Use the python bindings of libguestfs to create qcow2 image with > backing chains to mimic the layers of container image. > > This commit also changes the behaviour of FileSource when 'qcow2' > output format is used. Now the string layer-0.qcow2 will be used > as name of the output file. > > This change is appied in the test suite as an update to the function typo: 'applied' > get_image_path(). > > Note: The following commits which introduce UID/GID mapping > and setting root password for 'qcow2' with FileSource will apply > these changes to layer-1.qcow2 image with backing file layer-0.qcow2. What is the interest of this note for this commit? The info could be interesting in the corresponding commit though. > --- > src/virtBootstrap/sources/docker_source.py | 10 +- > src/virtBootstrap/sources/file_source.py | 14 +-- > src/virtBootstrap/utils.py | 146 +++++++++++++++++------------ > tests/file_source.py | 8 +- > 4 files changed, 106 insertions(+), 72 deletions(-) > > diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py > index 58e30ce..99247ab 100644 > --- a/src/virtBootstrap/sources/docker_source.py > +++ b/src/virtBootstrap/sources/docker_source.py > @@ -275,7 +275,15 @@ class DockerSource(object): > elif self.output_format == 'qcow2': > self.progress("Extracting container layers into qcow2 images", > value=50, logger=logger) > - utils.extract_layers_in_qcow2(self.layers, dest, self.progress) > + > + img = utils.BuildImage( > + layers=self.layers, > + dest=dest, > + progress=self.progress > + ) > + img.create_base_layer() > + img.create_backing_chains() > + > else: > raise Exception("Unknown format:" + self.output_format) > > diff --git a/src/virtBootstrap/sources/file_source.py b/src/virtBootstrap/sources/file_source.py > index 412db8a..69f024c 100644 > --- a/src/virtBootstrap/sources/file_source.py > +++ b/src/virtBootstrap/sources/file_source.py > @@ -64,14 +64,16 @@ class FileSource(object): > utils.untar_layers(layer, dest, self.progress) > > elif self.output_format == 'qcow2': > - # Remove the old path > - file_name = os.path.basename(self.path) > - qcow2_file = os.path.realpath('{}/{}.qcow2'.format(dest, > - file_name)) > - > self.progress("Extracting files into qcow2 image", value=0, > logger=logger) > - utils.create_qcow2(self.path, qcow2_file) > + > + img = utils.BuildImage( > + layers=layer, > + dest=dest, > + progress=self.progress > + ) > + img.create_base_layer() > + > else: > raise Exception("Unknown format:" + self.output_format) > > diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py > index 77177a3..482e4aa 100644 > --- a/src/virtBootstrap/utils.py > +++ b/src/virtBootstrap/utils.py > @@ -33,6 +33,7 @@ import tempfile > import logging > import re > > +import guestfs > import passlib.hosts > > # pylint: disable=invalid-name > @@ -42,6 +43,8 @@ logger = logging.getLogger(__name__) > DEFAULT_OUTPUT_FORMAT = 'dir' > # Default virtual size of qcow2 image > DEF_QCOW2_SIZE = '5G' > +DEF_BASE_IMAGE_SIZE = 5 * 1024 * 1024 * 1024 > + > if os.geteuid() == 0: > LIBVIRT_CONN = "lxc:///" > DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images" > @@ -51,6 +54,88 @@ else: > DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images" > > > +class BuildImage(object): > + """ > + Use guestfs-python to create qcow2 disk images. > + """ > + > + def __init__(self, layers, dest, progress): > + """ > + @param tar_files: Tarballs to be converted to qcow2 images > + @param dest: Directory where the qcow2 images will be created > + @param progress: Instance of the progress module > + """ > + self.g = guestfs.GuestFS(python_return_dict=True) > + self.layers = layers > + self.nlayers = len(layers) > + self.dest = dest > + self.progress = progress > + self.qcow2_files = [] > + > + def create_base_layer(self, fmt='qcow2', size=DEF_BASE_IMAGE_SIZE): > + """ > + Create and format qcow2 disk image which represnts the base layer. > + """ > + self.qcow2_files = [os.path.join(self.dest, 'layer-0.qcow2')] > + self.progress("Creating base layer", logger=logger) > + self.g.disk_create(self.qcow2_files[0], fmt, size) > + self.g.add_drive(self.qcow2_files[0], format=fmt) > + self.g.launch() > + self.progress("Formating disk image", logger=logger) > + self.g.mkfs("ext3", '/dev/sda') > + self.extract_layer(0, '/dev/sda') > + # Shutdown qemu instance to avoid hot-plugging of devices. > + self.g.shutdown() > + > + def create_backing_chains(self): > + """ > + Convert other layers to qcow2 images linked as backing chains. > + """ > + for i in range(1, self.nlayers): > + self.qcow2_files.append( > + os.path.join(self.dest, 'layer-%d.qcow2' % i) > + ) > + self.progress( > + "Creating image (%d/%d)" % (i + 1, self.nlayers), > + logger=logger > + ) > + self.g.disk_create( > + filename=self.qcow2_files[i], > + format='qcow2', > + size=-1, > + backingfile=self.qcow2_files[i - 1], > + backingformat='qcow2' > + ) > + self.g.add_drive(self.qcow2_files[i], format='qcow2') > + self.g.launch() > + devices = self.g.list_devices() > + # Tar-in layers (skip the base layer) > + for index in range(1, self.nlayers): > + self.extract_layer(index, devices[index - 1]) > + self.g.shutdown() > + > + def extract_layer(self, index, dev): > + """ > + Extract tarball of layer to device > + """ > + tar_file, tar_size = self.layers[index] > + log_layer_extract( > + tar_file, tar_size, index + 1, self.nlayers, self.progress > + ) > + self.tar_in(dev, tar_file) > + > + def tar_in(self, dev, tar_file): > + """ > + Common pattern used to tar-in archive into image > + """ > + self.g.mount(dev, '/') > + # Restore extended attributes, SELinux contexts and POSIX ACLs > + # from tar file. > + self.g.tar_in(tar_file, '/', get_compression_type(tar_file), > + xattrs=True, selinux=True, acls=True) > + self.g.umount('/') > + > + > def get_compression_type(tar_file): > """ > Get compression type of tar file. > @@ -209,67 +294,6 @@ def get_mime_type(path): > return output.read().decode('utf-8').split()[1] > > > -def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE): > - """ > - Create qcow2 image from tarball. > - """ > - qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size] > - > - if not backing_file: > - logger.info("Creating base qcow2 image") > - execute(qemu_img_cmd) > - > - logger.info("Formatting qcow2 image") > - execute(['virt-format', > - '--format=qcow2', > - '--partition=none', > - '--filesystem=ext3', > - '-a', layer_file]) > - else: > - # Add backing chain > - qemu_img_cmd.insert(2, "-b") > - qemu_img_cmd.insert(3, backing_file) > - > - logger.info("Creating qcow2 image with backing chain") > - execute(qemu_img_cmd) > - > - # Extract tarball using "tar-in" command from libguestfs > - tar_in_cmd = ["guestfish", > - "-a", layer_file, > - '-m', '/dev/sda', > - 'tar-in', tar_file, "/"] > - > - # Check if tarball is compressed > - compression = get_compression_type(tar_file) > - if compression is not None: > - tar_in_cmd.append('compress:' + compression) > - > - # Execute virt-tar-in command > - execute(tar_in_cmd) > - > - > -def extract_layers_in_qcow2(layers_list, dest_dir, progress): > - """ > - Extract docker layers in qcow2 images with backing chains. > - """ > - qcow2_backing_file = None > - > - nlayers = len(layers_list) > - for index, layer in enumerate(layers_list): > - tar_file, tar_size = layer > - log_layer_extract(tar_file, tar_size, index + 1, nlayers, progress) > - > - # Name format for the qcow2 image > - qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index) > - # Create the image layer > - create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file) > - # Keep the file path for the next layer > - qcow2_backing_file = qcow2_layer_file > - > - # Update progress value > - progress(value=(float(index + 1) / nlayers * 50) + 50) > - > - > def get_image_dir(no_cache=False): > """ > Get the directory where image layers are stored. > diff --git a/tests/file_source.py b/tests/file_source.py > index dd7fd00..2a0f964 100644 > --- a/tests/file_source.py > +++ b/tests/file_source.py > @@ -50,13 +50,13 @@ class Qcow2BuildImage(BuildTarFiles): > g.umount('/') > g.shutdown() > > - def get_image_path(self): > + def get_image_path(self, n=0): > """ > Returns the path where the qcow2 image will be stored. > """ > return os.path.join( > self.dest_dir, > - "%s.qcow2" % os.path.basename(self.tar_file) > + "layer-%d.qcow2" % n > ) > > def runTest(self): > @@ -92,7 +92,7 @@ class Qcow2OwnershipMapping(Qcow2BuildImage): > gid_map=self.gid_map > ) > self.apply_mapping() > - self.check_qcow2_images(self.get_image_path()) > + self.check_qcow2_images(self.get_image_path(1)) > > > @unittest.skip("Not implemented") > @@ -114,7 +114,7 @@ class Qcow2SettingRootPassword(Qcow2BuildImage): > root_password=self.root_password > ) > self.check_image = self.validate_shadow_file_in_image > - self.check_qcow2_images(self.get_image_path()) > + self.check_qcow2_images(self.get_image_path(1)) > > > @unittest.skipIf(os.geteuid() != 0, "Root privileges required") ACK -- Cedric _______________________________________________ virt-tools-list mailing list virt-tools-list@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/virt-tools-list