---
setup.py | 9 +-
src/virtBootstrap/sources/docker_source.py | 4 +-
src/virtBootstrap/sources/file_source.py | 7 +-
src/virtBootstrap/utils.py | 200 +++++++++++++++++------------
4 files changed, 132 insertions(+), 88 deletions(-)
diff --git a/setup.py b/setup.py
index 54e24a2..f6aa9c4 100755
--- a/setup.py
+++ b/setup.py
@@ -115,8 +115,13 @@ setuptools.setup(
# virt-bootstrap uses passlib to compute the hash of
# root password for root file system.
- install_requires=['passlib>=1.6.1'],
-
+ install_requires=[
+ 'passlib>=1.6.1',
+ 'guestfs'
+ ],
+ dependency_links=[
+ 'http://download.libguestfs.org/python#guestfs'
+ ],
tests_require=['mock>=2.0'],
extras_require={
diff --git a/src/virtBootstrap/sources/docker_source.py b/src/virtBootstrap/sources/docker_source.py
index 1b5bc31..9220e13 100644
--- a/src/virtBootstrap/sources/docker_source.py
+++ b/src/virtBootstrap/sources/docker_source.py
@@ -65,6 +65,7 @@ class DockerSource(object):
self.images_dir = utils.get_image_dir(self.no_cache)
self.manifest = None
self.layers = []
+ self.tar_files = []
if self.username and not self.password:
self.password = getpass.getpass()
@@ -96,6 +97,7 @@ class DockerSource(object):
sum_type, layer_sum = layer_digest.split(':')
file_path = os.path.join(self.images_dir, layer_sum + '.tar')
+ self.tar_files.append(file_path)
self.layers.append([sum_type, layer_sum, file_path, layer_size])
def gen_valid_uri(self, uri):
@@ -260,7 +262,7 @@ class DockerSource(object):
elif self.output_format == 'qcow2':
self.progress("Extracting container layers into qcow2 images",
value=50, logger=logger)
- utils.extract_layers_in_qcow2(self.layers, dest, self.progress)
+ utils.Build_QCOW2_Image(self.tar_files, dest, self.progress)
else:
raise Exception("Unknown format:" + self.output_format)
diff --git a/src/virtBootstrap/sources/file_source.py b/src/virtBootstrap/sources/file_source.py
index f8cf6c4..87589f6 100644
--- a/src/virtBootstrap/sources/file_source.py
+++ b/src/virtBootstrap/sources/file_source.py
@@ -62,14 +62,9 @@ class FileSource(object):
utils.safe_untar(self.path, dest)
elif self.output_format == 'qcow2':
- # Remove the old path
- file_name = os.path.basename(self.path)
- qcow2_file = os.path.realpath('{}/{}.qcow2'.format(dest,
- file_name))
-
self.progress("Extracting files into qcow2 image", value=0,
logger=logger)
- utils.create_qcow2(self.path, qcow2_file)
+ utils.Build_QCOW2_Image([self.path], dest, self.progress)
else:
raise Exception("Unknown format:" + self.output_format)
diff --git a/src/virtBootstrap/utils.py b/src/virtBootstrap/utils.py
index 2c79d6b..6369127 100644
--- a/src/virtBootstrap/utils.py
+++ b/src/virtBootstrap/utils.py
@@ -33,13 +33,16 @@ import logging
import re
from subprocess import CalledProcessError, PIPE, Popen
+import guestfs
import passlib.hosts
# pylint: disable=invalid-name
# Create logger
logger = logging.getLogger(__name__)
+
# Default virtual size of qcow2 image
-DEF_QCOW2_SIZE = '5G'
+DEF_QCOW2_SIZE = 5 * 1024 * 1024 * 1024
+
if os.geteuid() == 0:
LIBVIRT_CONN = "lxc:///"
DEFAULT_IMG_DIR = "/var/lib/virt-bootstrap/docker_images"
@@ -49,6 +52,117 @@ else:
DEFAULT_IMG_DIR += "/.local/share/virt-bootstrap/docker_images"
+class Build_QCOW2_Image(object):
+ """
+ Create qcow2 image with backing chains from list of tar files.
+ """
+ def __init__(self, tar_files, dest, progress):
+ """
+ Initialize guestfs
+ """
+ if not isinstance(tar_files, list):
+ raise ValueError('tar_files must be list not %s' % type(tar_files))
+ self.tar_files = tar_files
+ self.nlayers = len(tar_files)
+ self.progress = progress
+ self.fmt = 'qcow2'
+ self.qcow2_files = [os.path.join(dest, 'layer-%s.qcow2' % i)
+ for i in range(self.nlayers)]
+
+ self.g = guestfs.GuestFS(python_return_dict=True)
+ self.create_base_qcow2_layer(self.tar_files[0], self.qcow2_files[0])
+ if len(self.tar_files) > 1:
+ self.create_backing_chains()
+ self.g.shutdown()
+
+ def create_and_add_disk(self, qcow2_file, backingfile=None,
+ readonly=False):
+ """
+ Create and add qcow2 disk image.
+ """
+ if backingfile is not None:
+ size = -1
+ backingformat = self.fmt
+ else:
+ size = DEF_QCOW2_SIZE
+ backingformat = None
+
+ self.g.disk_create(qcow2_file, self.fmt, size, backingfile,
+ backingformat)
+ self.g.add_drive_opts(qcow2_file, readonly, self.fmt)
+
+ def tar_in(self, tar_file, dev):
+ """
+ Extract tar file in disk device.
+ """
+ self.g.mount(dev, '/')
+ # Restore extended attributes, SELinux contexts and POSIX ACLs
+ # from tar file.
+ self.g.tar_in(tar_file, '/', get_compression_type(tar_file),
+ xattrs=True, selinux=True, acls=True)
+ # Shutdown guestfs instance to avoid hot-plugging of devices.
+ self.g.umount('/')
+
+ def create_base_qcow2_layer(self, tar_file, qcow2_file):
+ """
+ Create and format base qcow2 layer.
+
+ Do this separatelly when extracting multiple layers to avoid
+ hot-plugging of devices.
+ """
+ self.progress("Creating base layer", logger=logger)
+ self.create_and_add_disk(qcow2_file)
+ self.g.launch()
+ dev = self.g.list_devices()[0]
+ self.progress("Formating disk image", logger=logger)
+ self.g.mkfs("ext3", dev)
+ self.tar_in(tar_file, dev)
+ self.progress("Extracting content of base layer", logger=logger)
+ self.g.shutdown()
+
+ def create_backing_chains(self):
+ """
+ Create backing chains for all layers after following the first
+ and tar-in the content.
+ """
+ for i in range(1, self.nlayers):
+ self.progress("Creating layer %d" % i, logger=logger)
+ self.create_and_add_disk(self.qcow2_files[i],
+ backingfile=self.qcow2_files[i - 1])
+
+ self.g.launch()
+ devices = self.g.list_devices()
+ # Iterate trough tar files of layers and skip the base layer
+ for i, tar_file in enumerate(self.tar_files[1:]):
+ self.progress("Extracting content of layer %d" % (i + 1),
+ logger=logger)
+ self.tar_in(tar_file, devices[i])
+
+
+def get_compression_type(tar_file):
+ """
+ Get compression type of tar file.
+ """
+ # Get mime type of archive
+ mime_tar_file = get_mime_type(tar_file)
+ logger.debug("Detected mime type of archive: %s", mime_tar_file)
+
+ compression_fmts = {
+ 'x-gzip': 'gzip',
+ 'gzip': 'gzip',
+ 'x-xz': 'xz',
+ 'x-bzip2': 'bzip2',
+ 'x-compress': 'compress',
+ 'x-lzop': 'lzop'
+ }
+
+ # Check if tarball is compressed
+ mime_type, mime_subtype = mime_tar_file.split('/')
+ if mime_type == 'application' and mime_subtype in compression_fmts:
+ return compression_fmts[mime_subtype]
+ return None
+
+
def checksum(path, sum_type, sum_expected):
"""
Validate file using checksum.
@@ -148,22 +262,6 @@ def log_layer_extract(layer, current, total, progress):
logger.debug('Untar layer: (%s:%s) %s', sum_type, sum_value, layer_file)
-def untar_layers(layers_list, dest_dir, progress):
- """
- Untar each of layers from container image.
- """
- nlayers = len(layers_list)
- for index, layer in enumerate(layers_list):
- log_layer_extract(layer, index + 1, nlayers, progress)
- layer_file = layer[2]
-
- # Extract layer tarball into destination directory
- safe_untar(layer_file, dest_dir)
-
- # Update progress value
- progress(value=(float(index + 1) / nlayers * 50) + 50)
-
-
def get_mime_type(path):
"""
Get the mime type of a file.
@@ -172,73 +270,17 @@ def get_mime_type(path):
.stdout.read().decode('utf-8').split()[1])
-def create_qcow2(tar_file, layer_file, backing_file=None, size=DEF_QCOW2_SIZE):
- """
- Create qcow2 image from tarball.
- """
- qemu_img_cmd = ["qemu-img", "create", "-f", "qcow2", layer_file, size]
-
- if not backing_file:
- logger.info("Creating base qcow2 image")
- execute(qemu_img_cmd)
-
- logger.info("Formatting qcow2 image")
- execute(['virt-format',
- '--format=qcow2',
- '--partition=none',
- '--filesystem=ext3',
- '-a', layer_file])
- else:
- # Add backing chain
- qemu_img_cmd.insert(2, "-b")
- qemu_img_cmd.insert(3, backing_file)
-
- logger.info("Creating qcow2 image with backing chain")
- execute(qemu_img_cmd)
-
- # Get mime type of archive
- mime_tar_file = get_mime_type(tar_file)
- logger.debug("Detected mime type of archive: %s", mime_tar_file)
-
- # Extract tarball using "tar-in" command from libguestfs
- tar_in_cmd = ["guestfish",
- "-a", layer_file,
- '-m', '/dev/sda',
- 'tar-in', tar_file, "/"]
-
- compression_fmts = {'x-gzip': 'gzip', 'gzip': 'gzip',
- 'x-xz': 'xz',
- 'x-bzip2': 'bzip2',
- 'x-compress': 'compress',
- 'x-lzop': 'lzop'}
-
- # Check if tarball is compressed
- mime_parts = mime_tar_file.split('/')
- if mime_parts[0] == 'application' and \
- mime_parts[1] in compression_fmts:
- tar_in_cmd.append('compress:' + compression_fmts[mime_parts[1]])
-
- # Execute virt-tar-in command
- execute(tar_in_cmd)
-
-
-def extract_layers_in_qcow2(layers_list, dest_dir, progress):
+def untar_layers(layers_list, dest_dir, progress):
"""
- Extract docker layers in qcow2 images with backing chains.
+ Untar each of layers from container image.
"""
- qcow2_backing_file = None
-
nlayers = len(layers_list)
for index, layer in enumerate(layers_list):
log_layer_extract(layer, index + 1, nlayers, progress)
- tar_file = layer[2]
-
- # Name format for the qcow2 image
- qcow2_layer_file = "{}/layer-{}.qcow2".format(dest_dir, index)
- # Create the image layer
- create_qcow2(tar_file, qcow2_layer_file, qcow2_backing_file)
- # Keep the file path for the next layer
- qcow2_backing_file = qcow2_layer_file
+ layer_file = layer[2]
+
+ # Extract layer tarball into destination directory
+ safe_untar(layer_file, dest_dir)
# Update progress value
progress(value=(float(index + 1) / nlayers * 50) + 50)
--
2.13.3
_______________________________________________
virt-tools-list mailing list
virt-tools-list@xxxxxxxxxx
https://www.redhat.com/mailman/listinfo/virt-tools-list
