On Thu, 2017-08-17 at 10:39 +0100, Radostin Stoyanov wrote: > Add implementation for virt-builder source which aims to create > container root file system from VM image build with virt-builder. 'build' -> 'built' > > Usage examples: > virt-bootstrap virt-builder://fedora-25 /tmp/foo > virt-bootstrap virt-builder://ubuntu-16.04 /tmp/bar --root-password secret > virt-bootstrap virt-builder://fedora-25 /tmp/foo -f qcow2 --idmap 0:1000:10 > sudo virt-bootstrap virt-builder://fedora-25 /tmp/foo --idmap 0:1000:10 > > Tests are also introduced along with the implementation. They cover > creation of root file system and UID/GID mapping for 'dir' and 'qcow2' > output format by mocking the build_image() method to avoid the time > consuming call to virt-builder which might also require network > connection with function which creates dummy disk image. > Setting root password is handled by virt-builder and hence the > introduced test only ensures that the password string is passed > correctly. Testing a virt-builder source without any call to virt-builder sounds fishy. I'ld rather not see virt-builder mocked up for real tests. > --- > src/virtBootstrap/sources/__init__.py | 1 + > src/virtBootstrap/sources/virt_builder_source.py | 148 +++++++++++++++ > src/virtBootstrap/virt_bootstrap.py | 4 +- > tests/__init__.py | 23 +++ > tests/file_source.py | 28 +-- > tests/virt_builder_source.py | 228 +++++++++++++++++++++++ > 6 files changed, 406 insertions(+), 26 deletions(-) > create mode 100644 src/virtBootstrap/sources/virt_builder_source.py > create mode 100644 tests/virt_builder_source.py > > diff --git a/src/virtBootstrap/sources/__init__.py b/src/virtBootstrap/sources/__init__.py > index e891e9b..be6b25c 100644 > --- a/src/virtBootstrap/sources/__init__.py > +++ b/src/virtBootstrap/sources/__init__.py > @@ -24,3 +24,4 @@ sources - Class definitions which process container image or > > from virtBootstrap.sources.file_source import FileSource > from virtBootstrap.sources.docker_source import DockerSource > +from virtBootstrap.sources.virt_builder_source import VirtBuilderSource > diff --git a/src/virtBootstrap/sources/virt_builder_source.py b/src/virtBootstrap/sources/virt_builder_source.py > new file mode 100644 > index 0000000..780ffb1 > --- /dev/null > +++ b/src/virtBootstrap/sources/virt_builder_source.py > @@ -0,0 +1,148 @@ > +# -*- coding: utf-8 -*- > +# Authors: Radostin Stoyanov <rstoyanov1@xxxxxxxxx> > +# > +# Copyright (C) 2017 Radostin Stoyanov > +# > +# This program is free software: you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation, either version 3 of the License, or > +# (at your option) any later version. > + > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > + > +# You should have received a copy of the GNU General Public License > +# along with this program. If not, see <http://www.gnu.org/licenses/>. > + > +""" > +VirtBuilderSource aim is to extract the root file system from VM image > +build with virt-builder from template. > +""" > + > +import os > +import logging > +import subprocess > +import tempfile > + > +import guestfs > +from virtBootstrap import utils > + > + > +# pylint: disable=invalid-name > +# Create logger > +logger = logging.getLogger(__name__) > + > + > +class VirtBuilderSource(object): > + """ > + Extract root file system from image build with virt-builder. > + """ > + def __init__(self, **kwargs): > + """ > + Create container rootfs by building VM from virt-builder template > + and extract the rootfs. > + > + @param uri: Template name > + @param fmt: Format used to store the output [dir, qcow2] > + @param uid_map: Mappings for UID of files in rootfs > + @param gid_map: Mappings for GID of files in rootfs > + @param root_password: Root password to set in rootfs > + @param progress: Instance of the progress module > + """ > + # Parsed URIs: > + # - "virt-builder:///<template>" > + # - "virt-builder://<template>" > + # - "virt-builder:/<template>" > + self.template = kwargs['uri'].netloc or kwargs['uri'].path[1:] > + self.output_format = kwargs.get('fmt', utils.DEFAULT_OUTPUT_FORMAT) > + self.uid_map = kwargs.get('uid_map', []) > + self.gid_map = kwargs.get('gid_map', []) > + self.root_password = kwargs.get('root_password', None) > + self.progress = kwargs['progress'].update_progress > + > + def build_image(self, output_file): > + """ > + Build VM from virt-builder template > + """ > + cmd = ['virt-builder', self.template, We could make this more configurable for testability. For example, we could add a self._builder_cmd member that contains ['virt-builder'] for normal case, but the test case could add some '--source', '/path/to/test/source'. That would have us test against virt-builder but get rid of the download length and network requirement. > + '-o', output_file, > + '--no-network', > + '--delete', '/dev/*', > + '--delete', '/boot/*', > + # Comment out all lines in fstab > + '--edit', '/etc/fstab:s/^/#/'] > + if self.root_password is not None: > + cmd += ['--root-password', "password:%s" % self.root_password] > + subprocess.check_call(cmd) > + > + def unpack(self, dest): > + """ > + Build image and extract root file system > + > + @param dest: Directory path where output files will be stored. > + """ > + > + with tempfile.NamedTemporaryFile(prefix='bootstrap_') as tmp_file: > + if self.output_format == 'dir': > + self.progress("Building image", value=0, logger=logger) > + self.build_image(tmp_file.name) > + self.progress("Extracting rootfs", value=50, logger=logger) > + g = guestfs.GuestFS(python_return_dict=True) > + g.add_drive_opts(tmp_file.name, readonly=False, format='raw') > + g.launch() > + > + # Get the device with file system > + root_dev = g.inspect_os() > + if not root_dev: > + raise Exception("No file system was found") > + g.mount(root_dev[0], '/') > + > + # Extract file system to destination directory > + g.copy_out('/', dest) > + > + g.umount('/') > + g.shutdown() > + > + self.progress("Extraction completed successfully!", > + value=100, logger=logger) > + logger.info("Files are stored in: %s", dest) > + > + elif self.output_format == 'qcow2': > + output_file = os.path.join(dest, 'layer-0.qcow2') > + > + self.progress("Building image", value=0, logger=logger) > + self.build_image(tmp_file.name) > + > + self.progress("Extracting rootfs", value=50, logger=logger) > + g = guestfs.GuestFS(python_return_dict=True) > + g.add_drive_opts(tmp_file.name, readonly=True, format='raw') > + # Create qcow2 disk image > + g.disk_create( > + filename=output_file, > + format='qcow2', > + size=os.path.getsize(tmp_file.name) > + ) > + g.add_drive_opts(output_file, readonly=False, format='qcow2') > + g.launch() > + # Get the device with file system > + root_dev = g.inspect_os() > + if not root_dev: > + raise Exception("No file system was found") > + output_dev = g.list_devices()[1] > + # Copy the file system to the new qcow2 disk > + g.copy_device_to_device(root_dev[0], output_dev, sparse=True) > + g.shutdown() > + > + # UID/GID mapping > + if self.uid_map or self.gid_map: > + logger.info("Mapping UID/GID") > + utils.map_id_in_image(1, dest, self.uid_map, self.gid_map) > + > + self.progress("Extraction completed successfully!", value=100, > + logger=logger) > + logger.info("Image is stored in: %s", output_file) > + > + else: > + raise Exception("Unknown format:" + self.output_format) > diff --git a/src/virtBootstrap/virt_bootstrap.py b/src/virtBootstrap/virt_bootstrap.py > index e387842..68e6516 100755 > --- a/src/virtBootstrap/virt_bootstrap.py > +++ b/src/virtBootstrap/virt_bootstrap.py > @@ -62,7 +62,7 @@ def get_source(source_type): > Get object which match the source type > """ > try: > - class_name = "%sSource" % source_type.capitalize() > + class_name = "%sSource" % source_type.title().replace('-', '') > clazz = getattr(sources, class_name) > return clazz > except Exception: > @@ -179,6 +179,8 @@ def main(): > docker://docker.io/fedora > docker://privateregistry:5000/image > file:///path/to/local/rootfs.tar.xz > + virt-builder://fedora-25 > + virt-builder://ubuntu-16.04 > ---------------------------------------- > > '''))) > diff --git a/tests/__init__.py b/tests/__init__.py > index 7a53c38..8888a0d 100644 > --- a/tests/__init__.py > +++ b/tests/__init__.py > @@ -29,6 +29,8 @@ import tarfile > import unittest > import passlib.hosts > > +import guestfs > + > try: > import mock > except ImportError: > @@ -434,3 +436,24 @@ class BuildTarFiles(unittest.TestCase): > self.check_image_content(g, user, 'dirs', g.is_dir) > # Check files > self.check_image_content(g, user, 'files', g.is_file) > + > + def check_qcow2_image(self, image_path): > + """ > + Ensures that qcow2 images contain all files. > + """ > + g = guestfs.GuestFS(python_return_dict=True) > + g.add_drive_opts(image_path, readonly=True) > + g.launch() > + g.mount('/dev/sda', '/') > + self.check_image(g) > + g.umount('/') > + g.shutdown() > + > + def get_image_path(self, n=0): > + """ > + Returns the path where the qcow2 image will be stored. > + """ > + return os.path.join( > + self.dest_dir, > + "layer-%d.qcow2" % n > + ) > diff --git a/tests/file_source.py b/tests/file_source.py > index 8851c3f..895ba9e 100644 > --- a/tests/file_source.py > +++ b/tests/file_source.py > @@ -24,7 +24,6 @@ with FileSource. > > import os > import unittest > -import guestfs > > from . import virt_bootstrap > from . import BuildTarFiles > @@ -38,27 +37,6 @@ class Qcow2BuildImage(BuildTarFiles): > works as expected. > """ > > - def check_qcow2_images(self, image_path): > - """ > - Ensures that qcow2 images contain all files. > - """ > - g = guestfs.GuestFS(python_return_dict=True) > - g.add_drive_opts(image_path, readonly=True) > - g.launch() > - g.mount('/dev/sda', '/') > - self.check_image(g) > - g.umount('/') > - g.shutdown() > - > - def get_image_path(self, n=0): > - """ > - Returns the path where the qcow2 image will be stored. > - """ > - return os.path.join( > - self.dest_dir, > - "layer-%d.qcow2" % n > - ) > - > def runTest(self): > """ > Create qcow2 image from each dummy tarfile. > @@ -69,7 +47,7 @@ class Qcow2BuildImage(BuildTarFiles): > fmt='qcow2', > progress_cb=mock.Mock() > ) > - self.check_qcow2_images(self.get_image_path()) > + self.check_qcow2_image(self.get_image_path()) > > > class Qcow2OwnershipMapping(Qcow2BuildImage): > @@ -91,7 +69,7 @@ class Qcow2OwnershipMapping(Qcow2BuildImage): > gid_map=self.gid_map > ) > self.apply_mapping() > - self.check_qcow2_images(self.get_image_path(1)) > + self.check_qcow2_image(self.get_image_path(1)) > > > class Qcow2SettingRootPassword(Qcow2BuildImage): > @@ -112,7 +90,7 @@ class Qcow2SettingRootPassword(Qcow2BuildImage): > root_password=self.root_password > ) > self.check_image = self.validate_shadow_file_in_image > - self.check_qcow2_images(self.get_image_path(1)) > + self.check_qcow2_image(self.get_image_path(1)) > > > @unittest.skipIf(os.geteuid() != 0, "Root privileges required") > diff --git a/tests/virt_builder_source.py b/tests/virt_builder_source.py > new file mode 100644 > index 0000000..4fe7713 > --- /dev/null > +++ b/tests/virt_builder_source.py > @@ -0,0 +1,228 @@ > +# -*- coding: utf-8 -*- > +# Authors: Radostin Stoyanov <rstoyanov1@xxxxxxxxx> > +# > +# Copyright (C) 2017 Radostin Stoyanov > +# > +# This program is free software: you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation, either version 3 of the License, or > +# (at your option) any later version. > + > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > + > +# You should have received a copy of the GNU General Public License > +# along with this program. If not, see <http://www.gnu.org/licenses/>. > + > + > +""" > +Regression tests which aim to excercise the creation of root file system typo: 'exercise' > +with VirtBuilderSource. > +""" > + > +import os > +import unittest > + > +import guestfs > + > +from . import BuildTarFiles > +from . import virt_bootstrap > +from . import mock > + > + > +# pylint: disable=invalid-name > +class DirExtractRootFS(BuildTarFiles): > + """ > + This test is replacing the method build_image() from VirtBuilderSource > + with a function which generates gummy disk image. > + > + It ensures that all files and directories from the created root file > + systems is extracted correctly. > + """ > + def create_tar_files(self): > + """ > + Don't need to build tar files for these tests. > + """ > + pass > + > + def create_dummy_disk(self, output_file): > + """ > + Create raw disk image with dummy root file system > + """ > + g = guestfs.GuestFS(python_return_dict=True) > + g.disk_create(output_file, format='raw', size=(1 * 1024 * 1024)) > + g.add_drive(output_file, readonly=False, format='raw') > + g.launch() > + g.mkfs('ext3', '/dev/sda') > + g.mount('/dev/sda', '/') > + for user in self.rootfs_tree: > + > + usr_uid = self.rootfs_tree[user]['uid'] > + usr_gid = self.rootfs_tree[user]['gid'] > + usr_dirs = self.rootfs_tree[user]['dirs'] > + usr_files = self.rootfs_tree[user]['files'] > + > + for member in usr_dirs: > + dir_name = '/' + member > + g.mkdir_p(dir_name) > + g.chown(usr_uid, usr_gid, dir_name) > + > + for member in usr_files: > + if isinstance(member, tuple): > + file_name = '/' + member[0] > + g.write(file_name, member[2]) > + g.chmod(member[1] & 0o777, file_name) > + else: > + file_name = '/' + member > + g.touch(file_name) > + g.chmod(0o755 & 0o777, file_name) > + g.chown(usr_uid, usr_gid, file_name) > + > + def runTest(self): > + """ > + Mock the function build_image() and call bootstrap(). > + Then check the extracted root file system. > + """ > + build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image' > + with mock.patch(build_image) as m_build_image: > + m_build_image.side_effect = self.create_dummy_disk > + virt_bootstrap.bootstrap( > + uri='virt-builder://foobar', > + dest=self.dest_dir, > + fmt='dir', > + progress_cb=mock.Mock() > + ) > + self.check_rootfs(skip_ownership=(os.geteuid() != 0)) > + The test would thus be: * Create dummy disk and minimal source index file (no need to gpg sign it) * something along this: virtBootstrap.sources.VirtBuilderSource._builder_args.append += ['--source', index_path] * virt_bootstrap.bootstrap(...) Same applies for the other tests. -- Cedric > + > +@unittest.skipIf(os.geteuid() != 0, "Root privileges required") > +class DirOwnershipMapping(DirExtractRootFS): > + """ > + Ensures that UID/GID mapping for extracted root file system are applied > + correctly. > + """ > + def runTest(self): > + """ > + Mock the function build_image() and call bootstrap() with uid/gid > + mapping values. > + Then check the ownership of the extracted root file system. > + """ > + self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]] > + self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]] > + build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image' > + with mock.patch(build_image) as m_build_image: > + m_build_image.side_effect = self.create_dummy_disk > + virt_bootstrap.bootstrap( > + progress_cb=mock.Mock(), > + uri='virt-builder://foobar', > + dest=self.dest_dir, > + fmt='dir', > + uid_map=self.uid_map, > + gid_map=self.gid_map > + ) > + self.apply_mapping() > + self.check_rootfs(skip_ownership=(os.geteuid() != 0)) > + > + > +class DirSettingRootPassword(DirExtractRootFS): > + """ > + The root password is set by virt-builder in this test we do not call > + virt-builder as this is time consuming job and might require network > + connection to download the VM template. > + > + Instead we only check if the root password is passed to virt-builder > + and if virt-bootstrap extracts the shadow file correctly. > + """ > + def verify_virt_builder_cmd(self, cmd): > + """ > + Ensures that virt-builder is called with valid command and the root > + password is passed. > + """ > + self.assertEqual( > + 'virt-builder', > + cmd[0], > + "virt-builder command does not start with 'virt-builder'" > + ) > + self.assertIn( > + '--root-password', > + cmd, > + "The flag '--root-password' is missing in virt-builder command" > + ) > + self.assertEqual( > + 'password:%s' % self.root_password, > + cmd[cmd.index('--root-password') + 1], > + "Root password doesn't match" > + ) > + self.create_dummy_disk(cmd[cmd.index('-o') + 1]) > + > + def runTest(self): > + """ > + Mock the function subprocess.check_call() and call bootstrap(). > + Then check the extracted root file system. > + """ > + self.root_password = "Root password" > + with mock.patch('subprocess.check_call') as m_check_call: > + m_check_call.side_effect = self.verify_virt_builder_cmd > + virt_bootstrap.bootstrap( > + uri='virt-builder://foobar', > + dest=self.dest_dir, > + fmt='dir', > + root_password=self.root_password, > + progress_cb=mock.Mock() > + ) > + m_check_call.assert_called_once() > + self.validate_shadow_file( > + os.path.join(self.dest_dir, 'etc/shadow'), > + skip_ownership=(os.geteuid() != 0), > + skip_hash=True > + ) > + > + > +class Qcow2BuildImage(DirExtractRootFS): > + """ > + Ensures that the file system is copied correctly to the output qcow2 image. > + """ > + def runTest(self): > + """ > + Mock the function build_image() and call bootstrap(). > + Then check the content of the new image. > + """ > + build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image' > + with mock.patch(build_image) as m_build_image: > + m_build_image.side_effect = self.create_dummy_disk > + virt_bootstrap.bootstrap( > + progress_cb=mock.Mock(), > + uri='virt-builder://foobar', > + dest=self.dest_dir, > + fmt='qcow2' > + ) > + self.check_qcow2_image(self.get_image_path()) > + > + > +class Qcow2OwnershipMapping(DirExtractRootFS): > + """ > + Ensures that UID/GID mapping is applied correctly with qcow2 conversion. > + """ > + def runTest(self): > + """ > + Mock the function build_image() and call bootstrap() with uid/gid > + mapping values. > + Then check the ownership of the extracted root file system. > + """ > + self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]] > + self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]] > + build_image = 'virtBootstrap.sources.VirtBuilderSource.build_image' > + with mock.patch(build_image) as m_build_image: > + m_build_image.side_effect = self.create_dummy_disk > + virt_bootstrap.bootstrap( > + uri='virt-builder://foobar', > + dest=self.dest_dir, > + fmt='qcow2', > + uid_map=self.uid_map, > + gid_map=self.gid_map, > + progress_cb=mock.Mock() > + ) > + self.apply_mapping() > + self.check_qcow2_image(self.get_image_path(1)) _______________________________________________ virt-tools-list mailing list virt-tools-list@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/virt-tools-list
