On Sat, 2017-08-26 at 21:42 +0100, Radostin Stoyanov wrote: > Add implementation for virt-builder source which aims to create > container root file system from VM image build with virt-builder. > > Usage examples: > $ virt-bootstrap virt-builder://fedora-25 /tmp/foo > $ virt-bootstrap virt-builder://ubuntu-16.04 /tmp/bar --root-password secret > $ virt-bootstrap virt-builder://fedora-25 /tmp/foo -f qcow2 --idmap 0:1000:10 > $ sudo virt-bootstrap virt-builder://fedora-25 /tmp/foo --idmap 0:1000:10 > > Tests are also introduced along with the implementation. They cover > creation of root file system and UID/GID mapping for 'dir' and 'qcow2' > output format by mocking the build_image() method to avoid the time > consuming call to virt-builder which might also require network > connection with function which creates dummy disk image. > Setting root password is handled by virt-builder and hence the > introduced test only ensures that the password string is passed > correctly. > --- > src/virtBootstrap/sources/__init__.py | 1 + > src/virtBootstrap/sources/virt_builder_source.py | 154 ++++++++++++++ > src/virtBootstrap/virt_bootstrap.py | 2 +- > tests/virt_builder_source.py | 243 +++++++++++++++++++++++ > 4 files changed, 399 insertions(+), 1 deletion(-) > create mode 100644 src/virtBootstrap/sources/virt_builder_source.py > create mode 100644 tests/virt_builder_source.py > > diff --git a/src/virtBootstrap/sources/__init__.py b/src/virtBootstrap/sources/__init__.py > index e891e9b..be6b25c 100644 > --- a/src/virtBootstrap/sources/__init__.py > +++ b/src/virtBootstrap/sources/__init__.py > @@ -24,3 +24,4 @@ sources - Class definitions which process container image or > > from virtBootstrap.sources.file_source import FileSource > from virtBootstrap.sources.docker_source import DockerSource > +from virtBootstrap.sources.virt_builder_source import VirtBuilderSource > diff --git a/src/virtBootstrap/sources/virt_builder_source.py b/src/virtBootstrap/sources/virt_builder_source.py > new file mode 100644 > index 0000000..0343091 > --- /dev/null > +++ b/src/virtBootstrap/sources/virt_builder_source.py > @@ -0,0 +1,154 @@ > +# -*- coding: utf-8 -*- > +# Authors: Radostin Stoyanov <rstoyanov1@xxxxxxxxx> > +# > +# Copyright (C) 2017 Radostin Stoyanov > +# > +# This program is free software: you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation, either version 3 of the License, or > +# (at your option) any later version. > + > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > + > +# You should have received a copy of the GNU General Public License > +# along with this program. If not, see <http://www.gnu.org/licenses/>. > + > +""" > +VirtBuilderSource aim is to extract the root file system from VM image > +build with virt-builder from template. > +""" > + > +import os > +import logging > +import subprocess > +import tempfile > + > +import guestfs > +from virtBootstrap import utils > + > + > +# pylint: disable=invalid-name > +# Create logger > +logger = logging.getLogger(__name__) > + > + > +class VirtBuilderSource(object): > + """ > + Extract root file system from image build with virt-builder. > + """ > + def __init__(self, **kwargs): > + """ > + Create container rootfs by building VM from virt-builder template > + and extract the rootfs. > + > + @param uri: Template name > + @param fmt: Format used to store the output [dir, qcow2] > + @param uid_map: Mappings for UID of files in rootfs > + @param gid_map: Mappings for GID of files in rootfs > + @param root_password: Root password to set in rootfs > + @param progress: Instance of the progress module > + """ > + # Parsed URIs: > + # - "virt-builder:///<template>" > + # - "virt-builder://<template>" > + # - "virt-builder:/<template>" > + self.template = kwargs['uri'].netloc or kwargs['uri'].path[1:] > + self.output_format = kwargs.get('fmt', utils.DEFAULT_OUTPUT_FORMAT) > + self.uid_map = kwargs.get('uid_map', []) > + self.gid_map = kwargs.get('gid_map', []) > + self.root_password = kwargs.get('root_password', None) > + self.progress = kwargs['progress'].update_progress > + > + def build_image(self, output_file): > + """ > + Build VM from virt-builder template > + """ > + cmd = ['virt-builder', self.template, > + '-o', output_file, > + '--no-network', > + '--delete', '/dev/*', > + '--delete', '/boot/*', > + # Comment out all lines in fstab > + '--edit', '/etc/fstab:s/^/#/'] > + if self.root_password is not None: > + cmd += ['--root-password', "password:%s" % self.root_password] > + self.run_buider(cmd) > + > + def run_buider(self, cmd): typo: s/run_buider/run_builder/ > + """ > + Execute virt-builder command > + """ > + subprocess.check_call(cmd) > + > + def unpack(self, dest): > + """ > + Build image and extract root file system > + > + @param dest: Directory path where output files will be stored. > + """ > + > + with tempfile.NamedTemporaryFile(prefix='bootstrap_') as tmp_file: > + if self.output_format == 'dir': > + self.progress("Building image", value=0, logger=logger) > + self.build_image(tmp_file.name) > + self.progress("Extracting rootfs", value=50, logger=logger) > + g = guestfs.GuestFS(python_return_dict=True) > + g.add_drive_opts(tmp_file.name, readonly=False, format='raw') > + g.launch() > + > + # Get the device with file system > + root_dev = g.inspect_os() > + if not root_dev: > + raise Exception("No file system was found") > + g.mount(root_dev[0], '/') > + > + # Extract file system to destination directory > + g.copy_out('/', dest) > + > + g.umount('/') > + g.shutdown() > + > + self.progress("Extraction completed successfully!", > + value=100, logger=logger) > + logger.info("Files are stored in: %s", dest) > + > + elif self.output_format == 'qcow2': > + output_file = os.path.join(dest, 'layer-0.qcow2') > + > + self.progress("Building image", value=0, logger=logger) > + self.build_image(tmp_file.name) > + > + self.progress("Extracting rootfs", value=50, logger=logger) > + g = guestfs.GuestFS(python_return_dict=True) > + g.add_drive_opts(tmp_file.name, readonly=True, format='raw') > + # Create qcow2 disk image > + g.disk_create( > + filename=output_file, > + format='qcow2', > + size=os.path.getsize(tmp_file.name) > + ) > + g.add_drive_opts(output_file, readonly=False, format='qcow2') > + g.launch() > + # Get the device with file system > + root_dev = g.inspect_os() > + if not root_dev: > + raise Exception("No file system was found") > + output_dev = g.list_devices()[1] > + # Copy the file system to the new qcow2 disk > + g.copy_device_to_device(root_dev[0], output_dev, sparse=True) > + g.shutdown() > + > + # UID/GID mapping > + if self.uid_map or self.gid_map: > + logger.info("Mapping UID/GID") > + utils.map_id_in_image(1, dest, self.uid_map, self.gid_map) > + > + self.progress("Extraction completed successfully!", value=100, > + logger=logger) > + logger.info("Image is stored in: %s", output_file) > + > + else: > + raise Exception("Unknown format:" + self.output_format) > diff --git a/src/virtBootstrap/virt_bootstrap.py b/src/virtBootstrap/virt_bootstrap.py > index e387842..fe95f5e 100755 > --- a/src/virtBootstrap/virt_bootstrap.py > +++ b/src/virtBootstrap/virt_bootstrap.py > @@ -62,7 +62,7 @@ def get_source(source_type): > Get object which match the source type > """ > try: > - class_name = "%sSource" % source_type.capitalize() > + class_name = "%sSource" % source_type.title().replace('-', '') > clazz = getattr(sources, class_name) > return clazz > except Exception: > diff --git a/tests/virt_builder_source.py b/tests/virt_builder_source.py > new file mode 100644 > index 0000000..3cd812b > --- /dev/null > +++ b/tests/virt_builder_source.py > @@ -0,0 +1,243 @@ > +# -*- coding: utf-8 -*- > +# Authors: Radostin Stoyanov <rstoyanov1@xxxxxxxxx> > +# > +# Copyright (C) 2017 Radostin Stoyanov > +# > +# This program is free software: you can redistribute it and/or modify > +# it under the terms of the GNU General Public License as published by > +# the Free Software Foundation, either version 3 of the License, or > +# (at your option) any later version. > + > +# This program is distributed in the hope that it will be useful, > +# but WITHOUT ANY WARRANTY; without even the implied warranty of > +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the > +# GNU General Public License for more details. > + > +# You should have received a copy of the GNU General Public License > +# along with this program. If not, see <http://www.gnu.org/licenses/>. > + > + > +""" > +Tests which aim to exercise the extraction of root file system from disk image > +created with virt-builder. > + > +Brief description of these tests: > +1. Create dummy root file system on raw disk image. > +2. Create index file of local repository for virt-builder. > +3. Call bootstrap() with modified virt-builder commnad to use local repository > + as source. > +4. Check the result. > +""" > + > +import copy > +import platform > +import os > +import shutil > +import tempfile > +import unittest > +import subprocess > + > +import guestfs > + > +from . import virt_bootstrap > +from . import mock > +from . import DEFAULT_FILE_MODE > +from . import ROOTFS_TREE > +from . import Qcow2ImageAccessor > +from . import NOT_ROOT > + > + > +# pylint: disable=invalid-name, too-many-instance-attributes > +class TestVirtBuilderSource(Qcow2ImageAccessor): > + """ > + Test cases for virt-builder source. > + """ > + > + def create_local_repository(self): > + """ > + Create raw disk image with dummy root file system and index file which > + contains the metadata used by virt-builder. > + """ > + g = guestfs.GuestFS(python_return_dict=True) > + g.disk_create( > + self.image['path'], > + format=self.image['format'], > + size=self.image['size'] > + ) > + g.add_drive( > + self.image['path'], > + readonly=False, > + format=self.image['format'] > + ) > + g.launch() > + g.mkfs('ext2', '/dev/sda') > + g.mount('/dev/sda', '/') > + for user in self.rootfs_tree: > + usr_uid = self.rootfs_tree[user]['uid'] > + usr_gid = self.rootfs_tree[user]['gid'] > + > + for member in self.rootfs_tree[user]['dirs']: > + dir_name = '/' + member > + g.mkdir_p(dir_name) > + g.chown(usr_uid, usr_gid, dir_name) > + > + for member in self.rootfs_tree[user]['files']: > + if isinstance(member, tuple): > + m_name, m_permissions, m_data = member > + file_name = '/' + m_name > + g.write(file_name, m_data) > + g.chmod(m_permissions & 0o777, file_name) > + else: > + file_name = '/' + member > + g.touch(file_name) > + g.chmod(DEFAULT_FILE_MODE & 0o777, file_name) > + > + g.chown(usr_uid, usr_gid, file_name) > + > + # Create index file > + with open(self.repo_index, 'w') as index_file: > + index_file.write( > + '[{template}]\n' > + 'name=Test\n' > + 'arch={arch}\n' > + 'file={filename}\n' # Relative (not real) path must be used. > + 'format={format}\n' > + 'expand=/dev/sda\n' > + 'size={size}\n'.format(**self.image) > + # The new line at the end of the index file is required. > + # Otherwise, virt-builder will return "syntax error". > + ) > + > + def setUp(self): > + self.rootfs_tree = copy.deepcopy(ROOTFS_TREE) > + > + self.fmt = None > + self.uid_map = None > + self.gid_map = None > + self.root_password = None > + self.checked_members = set() > + > + self.dest_dir = tempfile.mkdtemp('_bootstrap_dest') > + self.repo_dir = tempfile.mkdtemp('_local_builder_repo') > + # Set permissions for tmp directories to avoid > + # "Permission denied" errors from Libvirt. > + os.chmod(self.repo_dir, 0o755) > + os.chmod(self.dest_dir, 0o755) > + self.repo_index = os.path.join(self.repo_dir, 'index') > + > + self.image = { > + 'template': 'test', > + 'filename': 'test.img', > + 'path': os.path.join(self.repo_dir, 'test.img'), > + 'format': 'raw', > + 'size': (1 * 1024 * 1024), > + 'arch': platform.processor(), > + } > + self.create_local_repository() > + > + def mocked_run_builder(self, cmd): > + """ > + Modify the virt-builder command to use the dummy disk image > + and capture the 'stdout'. > + """ > + subprocess.check_call( > + cmd + [ > + '--source', > + 'file://%s' % self.repo_index, > + '--no-check-signature', > + '--no-cache' > + ], > + stdout=subprocess.PIPE > + ) > + > + def tearDown(self): > + """ > + Clean up > + """ > + shutil.rmtree(self.repo_dir) > + shutil.rmtree(self.dest_dir) > + > + def call_bootstrap(self): > + """ > + Mock out run_builder() with mocked_run_builder() and > + call bootstrap() method from virtBootstrap. > + """ > + # By default virt-builder sets random root password which leads to > + # modification in /etc/shadow file. If we don't test this we simplify > + # the test by not adding shadow file in our dummy root file system. > + if not self.root_password: > + self.rootfs_tree['root']['files'] = ['etc/hosts', 'etc/fstab'] > + > + target = ('virtBootstrap.sources.VirtBuilderSource.run_buider') typo again: s/run_buider/run_builder/ > + with mock.patch(target) as m_run_builder: > + m_run_builder.side_effect = self.mocked_run_builder > + > + virt_bootstrap.bootstrap( > + progress_cb=mock.Mock(), > + uri='virt-builder://%s' % self.image['template'], > + dest=self.dest_dir, > + fmt=self.fmt, > + gid_map=self.gid_map, > + uid_map=self.uid_map, > + root_password=self.root_password > + ) > + > + def test_dir_extract_rootfs(self): > + """ > + Ensures that the root file system is extracted correctly. > + """ > + self.fmt = 'dir' > + self.call_bootstrap() > + self.check_rootfs(skip_ownership=NOT_ROOT) > + > + @unittest.skipIf(NOT_ROOT, "Root privileges required") > + def test_dir_ownership_mapping(self): > + """ > + Ensures that UID/GID mapping is applied to extracted root file system. > + """ > + self.fmt = 'dir' > + self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]] > + self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]] > + self.call_bootstrap() > + self.apply_mapping() > + self.check_rootfs() > + > + def test_dir_setting_root_password(self): > + """ > + Ensures that password for root is set correctly. > + """ > + self.root_password = 'my secret root password' > + self.fmt = 'dir' > + self.call_bootstrap() > + self.validate_shadow_file() > + > + def test_qcow2_build_image(self): > + """ > + Ensures that the root file system is copied correctly within single > + partition qcow2 image. > + """ > + self.fmt = 'qcow2' > + self.call_bootstrap() > + self.check_qcow2_images(self.get_image_path()) > + > + def test_qcow2_ownership_mapping(self): > + """ > + Ensures that UID/GID mapping is applied in qcow2 image "layer-1.qcow2". > + """ > + self.fmt = 'qcow2' > + self.gid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]] > + self.uid_map = [[1000, 2000, 10], [0, 1000, 10], [500, 500, 10]] > + self.call_bootstrap() > + self.apply_mapping() > + self.check_qcow2_images(self.get_image_path(1)) > + > + def test_qcow2_setting_root_password(self): > + """ > + Ensures that the root password is set in the shadow file of > + "layer-1.qcow2" > + """ > + self.fmt = 'qcow2' > + self.root_password = "My secret password" > + self.call_bootstrap() > + self.check_image = self.validate_shadow_file_in_image > + self.check_qcow2_images(self.get_image_path()) ACK with the typos fixed -- Cedric _______________________________________________ virt-tools-list mailing list virt-tools-list@xxxxxxxxxx https://www.redhat.com/mailman/listinfo/virt-tools-list
