On Sat, 2017-08-26 at 21:42 +0100, Radostin Stoyanov wrote:
> Use the python bindings of libguestfs to create qcow2 image with
> backing chains to mimic the layers of container image.
>
> This commit also changes the behavior of FileSource when 'qcow2'
> output format is used. Now the string layer-0.qcow2 will be used
> as name of the output file.
>
> This change is applied in the test suite as an update to the function
> get_image_path().
> ---
> src/virtBootstrap/sources/docker_source.py | 10 +-
> src/virtBootstrap/sources/file_source.py | 14 +--
> src/virtBootstrap/utils.py | 146 +++++++++++++++++------------
> tests/file_source.py | 29 ++++++
> 4 files changed, 131 insertions(+), 68 deletions(-)
>
> diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py
> index 2dadb42..a6ea3e6 100644
> --- a/src/virtBootstrap/sources/docker_source.py
> +++ b/src/virtBootstrap/sources/docker_source.py
> @@ -272,7 +272,15 @@ class DockerSource(object):
> elif self.output_format == 'qcow2':
> self.progress("Extracting container layers into qcow2 images",
> value=50, logger=logger)
> - utils.extract_layers_in_qcow2(self.layers, dest, self.progress)
> +
> + img = utils.BuildImage(
> + layers=self.layers,
> + dest=dest,
> + progress=self.progress
> + )
> + img.create_base_layer()
> + img.create_backing_chains()
> +
> else:
> raise Exception("Unknown format:" + self.output_format)
>
> diff --git a/src/virtBootstrap/sources/file_source.py b/src/virtBootstrap/sources/file_source.py
> index 412db8a..69f024c 100644
> --- a/src/virtBootstrap/sources/file_source.py
> +++ b/src/virtBootstrap/sources/file_source.py
> @@ -64,14 +64,16 @@ class FileSource(object):
> utils.untar_layers(layer, dest, self.progress)
>
> elif self.output_format == 'qcow2':
> - # Remove the old path
> - file_name = os.path.basename(self.path)
> - qcow2_file = os.path.realpath('{}/{}.qcow2'.format(dest,
> - file_name))
> -
> self.progress("Extracting files into qcow2 image", value=0,
> logger=logger)
> - utils.create_qcow2(self.path, qcow2_file)
> +
> + img = utils.BuildImage(
> + layers=layer,
> + dest=dest,
> + progress=self.progress
> + )
> + img.create_base_layer()
> +
> else:
> raise Exception("Unknown format:" + self.output_format)
>
> diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py
> index be9133c..5fb5d8c 100644
> --- a/src/virtBootstrap/utils.py
> +++ b/src/virtBootstrap/utils.py
> @@ -33,6 +33,7 @@ import tempfile
> import logging
> import re
>
> +import guestfs
> import passlib.hosts
>
> # pylint: disable=invalid-name
> @@ -42,6 +43,8 @@ logger = logging.getLogger(__name__)
> DEFAULT_OUTPUT_FORMAT = 'dir'
> # Default virtual size of qcow2 image
> DEF_QCOW2_SIZE = '5G'
> +DEF_BASE_IMAGE_SIZE = 5 * 1024 * 1024 * 1024
> +
> if os.geteuid() == 0:
> LIBVIRT_CONN = "lxc:///"
> DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images"
> @@ -51,6 +54,88 @@ else:
> DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images"
>
>
> +class BuildImage(object):
> + """
> + Use guestfs-python to create qcow2 disk images.
> + """
> +
> + def __init__(self, layers, dest, progress):
> + """
> + @param tar_files: Tarballs to be converted to qcow2 images
> + @param dest: Directory where the qcow2 images will be created
> + @param progress: Instance of the progress module
> + """
> + self.g = guestfs.GuestFS(python_return_dict=True)
> + self.layers = layers
> + self.nlayers = len(layers)
> + self.dest = dest
> + self.progress = progress
> + self.qcow2_files = []
> +
> + def create_base_layer(self, fmt='qcow2', size=DEF_BASE_IMAGE_SIZE):
> + """
> + Create and format qcow2 disk image which represnts the base layer.
> + """
> + self.qcow2_files = [os.path.join(self.dest, 'layer-0.qcow2')]
> + self.progress("Creating base layer", logger=logger)
> + self.g.disk_create(self.qcow2_files[0], fmt, size)
> + self.g.add_drive(self.qcow2_files[0], format=fmt)
> + self.g.launch()
> + self.progress("Formating disk image", logger=logger)
> + self.g.mkfs("ext3", '/dev/sda')
> + self.extract_layer(0, '/dev/sda')
> + # Shutdown qemu instance to avoid hot-plugging of devices.
> + self.g.shutdown()
> +
> + def create_backing_chains(self):
> + """
> + Convert other layers to qcow2 images linked as backing chains.
> + """
> + for i in range(1, self.nlayers):
> + self.qcow2_files.append(
> + os.path.join(self.dest, 'layer-%d.qcow2' % i)
> + )
> + self.progress(
> + "Creating image (%d/%d)" % (i + 1, self.nlayers),
> + logger=logger
> + )
> + self.g.disk_create(
> + filename=self.qcow2_files[i],
> + format='qcow2',
> + size=-1,
> + backingfile=self.qcow2_files[i - 1],
> + backingformat='qcow2'
> + )
> + self.g.add_drive(self.qcow2_files[i], format='qcow2')
> + self.g.launch()
> + devices = self.g.list_devices()
> + # Tar-in layers (skip the base layer)
> + for index in range(1, self.nlayers):
> + self.extract_layer(index, devices[index - 1])
> + self.g.shutdown()
> +
> + def extract_layer(self, index, dev):
> + """
> + Extract tarball of layer to device
> + """
> + tar_file, tar_size = self.layers[index]
> + log_layer_extract(
> + tar_file, tar_size, index + 1, self.nlayers, self.progress
> + )
> + self.tar_in(dev, tar_file)
> +
> + def tar_in(self, dev, tar_file):
> + """
> + Common pattern used to tar-in archive into image
> + """
> + self.g.mount(dev, '/')
> + # Restore extended attributes, SELinux contexts and POSIX ACLs
> + # from tar file.
> + self.g.tar_in(tar_file, '/', get_compression_type(tar_file),
> + xattrs=True, selinux=True, acls=True)
> + self.g.umount('/')
> +
> +
> def get_compression_type(tar_file):
> """
> Get compression type of tar file.
> @@ -212,67 +297,6 @@ def get_mime_type(path):
> return output.read().decode('utf-8').split()[1]
>
>
> -def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE):
> - """
> - Create qcow2 image from tarball.
> - """
> - qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size]
> -
> - if not backing_file:
> - logger.info("Creating base qcow2 image")
> - execute(qemu_img_cmd)
> -
> - logger.info("Formatting qcow2 image")
> - execute(['virt-format',
> - '--format=qcow2',
> - '--partition=none',
> - '--filesystem=ext3',
> - '-a', layer_file])
> - else:
> - # Add backing chain
> - qemu_img_cmd.insert(2, "-b")
> - qemu_img_cmd.insert(3, backing_file)
> -
> - logger.info("Creating qcow2 image with backing chain")
> - execute(qemu_img_cmd)
> -
> - # Extract tarball using "tar-in" command from libguestfs
> - tar_in_cmd = ["guestfish",
> - "-a", layer_file,
> - '-m', '/dev/sda',
> - 'tar-in', tar_file, "/"]
> -
> - # Check if tarball is compressed
> - compression = get_compression_type(tar_file)
> - if compression is not None:
> - tar_in_cmd.append('compress:' + compression)
> -
> - # Execute virt-tar-in command
> - execute(tar_in_cmd)
> -
> -
> -def extract_layers_in_qcow2(layers_list, dest_dir, progress):
> - """
> - Extract docker layers in qcow2 images with backing chains.
> - """
> - qcow2_backing_file = None
> -
> - nlayers = len(layers_list)
> - for index, layer in enumerate(layers_list):
> - tar_file, tar_size = layer
> - log_layer_extract(tar_file, tar_size, index + 1, nlayers, progress)
> -
> - # Name format for the qcow2 image
> - qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index)
> - # Create the image layer
> - create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file)
> - # Keep the file path for the next layer
> - qcow2_backing_file = qcow2_layer_file
> -
> - # Update progress value
> - progress(value=(float(index + 1) / nlayers * 50) + 50)
> -
> -
> def get_image_dir(no_cache=False):
> """
> Get the directory where image layers are stored.
> diff --git a/tests/file_source.py b/tests/file_source.py
> index 79bb234..391ca48 100644
> --- a/tests/file_source.py
> +++ b/tests/file_source.py
> @@ -27,6 +27,7 @@ import unittest
> from . import mock
> from . import virt_bootstrap
> from . import ImageAccessor
> +from . import Qcow2ImageAccessor
> from . import NOT_ROOT
>
>
> @@ -76,3 +77,31 @@ class TestDirFileSource(ImageAccessor):
> self.root_password = 'my secret root password'
> self.call_bootstrap()
> self.validate_shadow_file()
> +
> +
> +class TestQcow2FileSource(Qcow2ImageAccessor):
> + """
> + Test cases for the class FileSource used with qcow2 output format.
> + """
> +
> + def call_bootstrap(self):
> + """
> + Execute the bootstrap method from virtBootstrap.
> + """
> + virt_bootstrap.bootstrap(
> + uri=self.tar_file,
> + dest=self.dest_dir,
> + fmt='qcow2',
> + progress_cb=mock.Mock(),
> + uid_map=self.uid_map,
> + gid_map=self.gid_map,
> + root_password=self.root_password
> + )
> +
> + def test_qcow2_extract_rootfs(self):
> + """
> + Ensures root file system of tar archive is converted to single
> + partition qcow2 image.
> + """
> + self.call_bootstrap()
> + self.check_qcow2_images(self.get_image_path())
ACK
--
Cedric
_______________________________________________
virt-tools-list mailing list
virt-tools-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/virt-tools-list
