From: Cleber Rosa <crosa@xxxxxxxxxx>
The ongoing refactor work on the virt installer has led the creation
of helper classes. There are basically three types of helper classes:
* Content helper classes, that ease and provide a common interface
for fetching content, usually source code, either local or remote.
These include GitRepo, LocalSource, LocalTar and RemoteTar.
* Build helper classes, that deal with building content, usually
compiling source code into binary files. This currently is
represented by a single class: GnuSourceBuildHelper
* Patch helper classes, that deal with patching content, again,
usually source code, with patch files that live either remotely
or locally. There's currently also only one Patch helper class,
PatchHelper. The main puporse here is to share code among
different installers.
For every *Helper class, there's also a *ParamHelper class, that
instead of instantiating with individual params, can read them
directly from the Param dict for the current test.
Changes from v1:
* Minimal changes in coding style (change some old style classes
into new style ones, make exception raise conform with the coding
style)
Signed-off-by: Cleber Rosa <crosa@xxxxxxxxxx>
---
client/virt/virt_utils.py | 696 ++++++++++++++++++++++++++++++++++++++++++++-
1 files changed, 694 insertions(+), 2 deletions(-)
diff --git a/client/virt/virt_utils.py b/client/virt/virt_utils.py
index 7026492..1966d81 100644
--- a/client/virt/virt_utils.py
+++ b/client/virt/virt_utils.py
@@ -6,7 +6,7 @@ KVM test utility functions.
import time, string, random, socket, os, signal, re, logging, commands, cPickle
import fcntl, shelve, ConfigParser, threading, sys, UserDict, inspect, tarfile
-import struct
+import struct, shutil
from autotest_lib.client.bin import utils, os_dep
from autotest_lib.client.common_lib import error, logging_config
import rss_client, aexpect
@@ -2110,7 +2110,7 @@ def get_default_koji_tag():
return DEFAULT_KOJI_TAG
-class KojiPkgSpec:
+class KojiPkgSpec(object):
'''
A package specification syntax parser for Koji
@@ -2405,6 +2405,698 @@ def mount(src, mount_point, type, perm="rw"):
return False
+class GitRepoHelper(object):
+ '''
+ Helps to deal with git repos, mostly fetching content from a repo
+ '''
+ def __init__(self, uri, branch, destination_dir, commit=None, lbranch=None):
+ '''
+ Instantiates a new GitRepoHelper
+
+ @type uri: string
+ @param uri: git repository url
+ @type branch: string
+ @param branch: git remote branch
+ @type destination_dir: string
+ @param destination_dir: path of a dir where to save downloaded code
+ @type commit: string
+ @param commit: specific commit to download
+ @type lbranch: string
+ @param lbranch: git local branch name, if different from remote
+ '''
+ self.uri = uri
+ self.branch = branch
+ self.destination_dir = destination_dir
+ self.commit = commit
+ if self.lbranch is None:
+ self.lbranch = branch
+
+
+ def init(self):
+ '''
+ Initializes a directory for receiving a verbatim copy of git repo
+
+ This creates a directory if necessary, and either resets or inits
+ the repo
+ '''
+ if not os.path.exists(self.destination_dir):
+ logging.debug('Creating directory %s for git repo %s',
+ self.destination_dir, self.uri)
+ os.makedirs(self.destination_dir)
+
+ os.chdir(self.destination_dir)
+
+ if os.path.exists('.git'):
+ logging.debug('Resetting previously existing git repo at %s for '
+ 'receiving git repo %s',
+ self.destination_dir, self.uri)
+ utils.system('git reset --hard')
+ else:
+ logging.debug('Initializing new git repo at %s for receiving '
+ 'git repo %s',
+ self.destination_dir, self.uri)
+ utils.system('git init')
+
+
+ def fetch(self):
+ '''
+ Performs a git fetch from the remote repo
+ '''
+ logging.info("Fetching git [REP '%s' BRANCH '%s'] -> %s",
+ self.uri, self.branch, self.destination_dir)
+ os.chdir(self.destination_dir)
+ utils.system("git fetch -q -f -u -t %s %s:%s" % (self.uri,
+ self.branch,
+ self.lbranch))
+
+
+ def checkout(self):
+ '''
+ Performs a git checkout for a given branch and start point (commit)
+ '''
+ os.chdir(self.destination_dir)
+
+ logging.debug('Checking out local branch %s', self.lbranch)
+ utils.system("git checkout %s" % self.lbranch)
+
+ if self.commit is not None:
+ logging.debug('Checking out commit %s', self.commit)
+ utils.system("git checkout %s" % self.commit)
+
+ h = utils.system_output('git log --pretty=format:"%H" -1').strip()
+ try:
+ desc = "tag %s" % utils.system_output("git describe")
+ except error.CmdError:
+ desc = "no tag found"
+
+ logging.info("Commit hash for %s is %s (%s)", self.name, h, desc)
+
+
+ def execute(self):
+ '''
+ Performs all steps necessary to initialize and download a git repo
+
+ This includes the init, fetch and checkout steps in one single
+ utility method.
+ '''
+ self.init()
+ self.fetch()
+ self.checkout()
+
+
+class GitRepoParamHelper(GitRepoHelper):
+ '''
+ Helps to deal with git repos specified in cartersian config files
+
+ This class attempts to make it simple to manage a git repo, by using a
+ naming standard that follows this basic syntax:
+
+ <prefix>_name_<suffix>
+
+ <prefix> is always 'git_repo' and <suffix> sets options for this git repo.
+ Example for repo named foo:
+
+ git_repo_foo_uri = git://git.foo.org/foo.git
+ git_repo_foo_branch = master
+ git_repo_foo_lbranch = master
+ git_repo_foo_commit = bb5fb8e678aabe286e74c4f2993dc2a9e550b627
+ '''
+ def __init__(self, params, name, destination_dir):
+ '''
+ Instantiates a new GitRepoParamHelper
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to this repo
+
+ This method currently does everything that the parent class __init__()
+ method does, that is, sets all instance variables needed by other
+ methods. That means it's not strictly necessary to call parent's
+ __init__().
+ '''
+ config_prefix = 'git_repo_%s' % self.name
+ logging.debug('Parsing parameters for git repo %s, configuration '
+ 'prefix is %s' % (self.name, config_prefix))
+
+ self.uri = self.params.get('%s_uri' % config_prefix)
+ logging.debug('Git repo %s uri: %s' % (self.name, self.uri))
+
+ self.branch = self.params.get('%s_branch' % config_prefix, 'master')
+ logging.debug('Git repo %s branch: %s' % (self.name, self.branch))
+
+ self.lbranch = self.params.get('%s_lbranch' % config_prefix)
+ if self.lbranch is None:
+ self.lbranch = self.branch
+ logging.debug('Git repo %s lbranch: %s' % (self.name, self.lbranch))
+
+ self.commit = self.params.get('%s_commit' % config_prefix)
+ if self.commit is None:
+ logging.debug('Git repo %s commit is not set' % self.name)
+ else:
+ logging.debug('Git repo %s commit: %s' % (self.name, self.commit))
+
+
+class LocalSourceDirHelper(object):
+ '''
+ Helper class to deal with source code sitting somewhere in the filesystem
+ '''
+ def __init__(self, source_dir, destination_dir):
+ '''
+ @param source_dir:
+ @param destination_dir:
+ @return: new LocalSourceDirHelper instance
+ '''
+ self.source = source_dir
+ self.destination = destination_dir
+
+
+ def execute(self):
+ '''
+ Copies the source directory to the destination directory
+ '''
+ if os.path.isdir(self.destination):
+ shutil.rmtree(self.destination)
+
+ if os.path.isdir(self.source):
+ shutil.copytree(self.source, self.destination)
+
+
+class LocalSourceDirParamHelper(LocalSourceDirHelper):
+ '''
+ Helps to deal with source dirs specified in cartersian config files
+
+ This class attempts to make it simple to manage a source dir, by using a
+ naming standard that follows this basic syntax:
+
+ <prefix>_name_<suffix>
+
+ <prefix> is always 'local_src' and <suffix> sets options for this source
+ dir. Example for source dir named foo:
+
+ local_src_foo_path = /home/user/foo
+ '''
+ def __init__(self, params, name, destination_dir):
+ '''
+ Instantiate a new LocalSourceDirParamHelper
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to source dir
+ '''
+ config_prefix = 'local_src_%s' % self.name
+ logging.debug('Parsing parameters for local source %s, configuration '
+ 'prefix is %s' % (self.name, config_prefix))
+
+ self.path = self.params.get('%s_path' % config_prefix)
+ logging.debug('Local source directory %s path: %s' % (self.name,
+ self.path))
+ self.source = self.path
+ self.destination = self.destination_dir
+
+
+class LocalTarHelper(object):
+ '''
+ Helper class to deal with source code in a local tarball
+ '''
+ def __init__(self, source_file, destination_dir):
+ self.source = source
+ self.destination = destination_dir
+
+
+ def extract(self):
+ '''
+ Extracts the tarball into the destination directory
+ '''
+ if os.path.isdir(self.destination):
+ shutil.rmtree(self.destination)
+
+ if os.path.isfile(self.source) and tarfile.is_tarfile(self.source):
+
+ name = os.path.basename(self.destination)
+ temp_dir = os.path.join(os.path.dirname(self.destination),
+ '%s.tmp' % name)
+ logging.debug('Temporary directory for extracting tarball is %s' %
+ temp_dir)
+
+ if not os.path.isdir(temp_dir):
+ os.makedirs(temp_dir)
+
+ tarball = tarfile.open(self.source)
+ tarball.extractall(temp_dir)
+
+ #
+ # If there's a directory at the toplevel of the tarfile, assume
+ # it's the root for the contents, usually source code
+ #
+ tarball_info = tarball.members[0]
+ if tarball_info.isdir():
+ content_path = os.path.join(temp_dir,
+ tarball_info.name)
+ else:
+ content_path = temp_dir
+
+ #
+ # Now move the content directory to the final destination
+ #
+ shutil.move(content_path, self.destination)
+
+ else:
+ raise OSError("%s is not a file or tar file" % self.source)
+
+
+ def execute(self):
+ '''
+ Executes all action this helper is suposed to perform
+
+ This is the main entry point method for this class, and all other
+ helper classes.
+ '''
+ self.extract()
+
+
+class LocalTarParamHelper(LocalTarHelper):
+ '''
+ Helps to deal with source tarballs specified in cartersian config files
+
+ This class attempts to make it simple to manage a tarball with source code,
+ by using a naming standard that follows this basic syntax:
+
+ <prefix>_name_<suffix>
+
+ <prefix> is always 'local_tar' and <suffix> sets options for this source
+ tarball. Example for source tarball named foo:
+
+ local_tar_foo_path = /tmp/foo-1.0.tar.gz
+ '''
+ def __init__(self, params, name, destination_dir):
+ '''
+ Instantiates a new LocalTarParamHelper
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to this local tar helper
+ '''
+ config_prefix = 'local_tar_%s' % self.name
+ logging.debug('Parsing parameters for local tar %s, configuration '
+ 'prefix is %s' % (self.name, config_prefix))
+
+ self.path = self.params.get('%s_path' % config_prefix)
+ logging.debug('Local source tar %s path: %s' % (self.name,
+ self.path))
+ self.source = self.path
+ self.destination = self.destination_dir
+
+
+class RemoteTarHelper(LocalTarHelper):
+ '''
+ Helper that fetches a tarball and extracts it locally
+ '''
+ def __init__(self, source_uri, destination_dir):
+ self.source = source_uri
+ self.destination = destination_dir
+
+
+ def execute(self):
+ '''
+ Executes all action this helper class is suposed to perform
+
+ This is the main entry point method for this class, and all other
+ helper classes.
+
+ This implementation fetches the remote tar file and then extracts
+ it using the functionality present in the parent class.
+ '''
+ name = os.path.basename(self.source)
+ base_dest = os.path.dirname(self.destination_dir)
+ dest = os.path.join(base_dest, name)
+ utils.get_file(self.source, dest)
+ self.source = dest
+ self.extract()
+
+
+class RemoteTarParamHelper(RemoteTarHelper):
+ '''
+ Helps to deal with remote source tarballs specified in cartersian config
+
+ This class attempts to make it simple to manage a tarball with source code,
+ by using a naming standard that follows this basic syntax:
+
+ <prefix>_name_<suffix>
+
+ <prefix> is always 'local_tar' and <suffix> sets options for this source
+ tarball. Example for source tarball named foo:
+
+ remote_tar_foo_uri = http://foo.org/foo-1.0.tar.gz
+ '''
+ def __init__(self, params, name, destination_dir):
+ '''
+ Instantiates a new RemoteTarParamHelper instance
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to this remote tar helper
+ '''
+ config_prefix = 'remote_tar_%s' % self.name
+ logging.debug('Parsing parameters for remote tar %s, configuration '
+ 'prefix is %s' % (self.name, config_prefix))
+
+ self.uri = self.params.get('%s_uri' % config_prefix)
+ logging.debug('Remote source tar %s uri: %s' % (self.name,
+ self.uri))
+ self.source = self.uri
+ self.destination = self.destination_dir
+
+
+class PatchHelper(object):
+ '''
+ Helper that encapsulates the patching of source code with patch files
+ '''
+ def __init__(self, source_dir, patches):
+ '''
+ Initializes a new PatchHelper
+ '''
+ self.source_dir = source_dir
+ self.patches = patches
+
+
+ def download(self):
+ '''
+ Copies patch files from remote locations to the source directory
+ '''
+ for patch in self.patches:
+ utils.get_file(patch, os.path.join(self.source_dir,
+ os.path.basename(patch)))
+
+
+ def patch(self):
+ '''
+ Patches the source dir with all patch files
+ '''
+ os.chdir(self.source_dir)
+ for patch in self.patches:
+ patch_file = os.path.join(self.source_dir,
+ os.path.basename(patch))
+ utils.system('patch -p1 < %s' % os.path.basename(patch))
+
+
+ def execute(self):
+ '''
+ Performs all steps necessary to download patches and apply them
+ '''
+ self.download()
+ self.patch()
+
+
+class PatchParamHelper(PatchHelper):
+ '''
+ Helps to deal with patches specified in cartersian config files
+
+ This class attempts to make it simple to patch source coude, by using a
+ naming standard that follows this basic syntax:
+
+ [<git_repo>|<local_src>|<local_tar>|<remote_tar>]_<name>_patches
+
+ <prefix> is either a 'local_src' or 'git_repo', that, together with <name>
+ specify a directory containing source code to receive the patches. That is,
+ for source code coming from git repo foo, patches would be specified as:
+
+ git_repo_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']
+
+ And for for patches to be applied on local source code named also foo:
+
+ local_src_foo_patches = ['http://foo/bar.patch', 'http://foo/baz.patch']
+ '''
+ def __init__(self, params, prefix, source_dir):
+ '''
+ Initializes a new PatchParamHelper instance
+ '''
+ self.params = params
+ self.prefix = prefix
+ self.source_dir = source_dir
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to this set of patches
+
+ This method currently does everything that the parent class __init__()
+ method does, that is, sets all instance variables needed by other
+ methods. That means it's not strictly necessary to call parent's
+ __init__().
+ '''
+ logging.debug('Parsing patch parameters for prefix %s' % self.prefix)
+ patches_param_key = '%s_patches' % self.prefix
+
+ self.patches_str = self.params.get(patches_param_key, '[]')
+ logging.debug('Patches config for prefix %s: %s' % (self.prefix,
+ self.patches_str))
+
+ self.patches = eval(self.patches_str)
+ logging.debug('Patches for prefix %s: %s' % (self.prefix,
+ ", ".join(self.patches)))
+
+
+class GnuSourceBuildInvalidSource(Exception):
+ '''
+ Exception raised when build source dir/file is not valid
+ '''
+ pass
+
+
+class GnuSourceBuildHelper(object):
+ '''
+ Handles software installation of GNU-like source code
+
+ This basically means that the build will go though the classic GNU
+ autotools steps: ./configure, make, make install
+ '''
+ def __init__(self, source, build_dir, prefix,
+ configure_options=[]):
+ '''
+ @type source: string
+ @param source: source directory or tarball
+ @type prefix: string
+ @param prefix: installation prefix
+ @type build_dir: string
+ @param build_dir: temporary directory used for building the source code
+ @type configure_options: list
+ @param configure_options: options to pass to configure
+ @throws: GnuSourceBuildInvalidSource
+ '''
+ self.source = source
+ self.build_dir = build_dir
+ self.prefix = prefix
+ self.configure_options = configure_options
+ self.include_pkg_config_path()
+
+
+ def include_pkg_config_path(self):
+ '''
+ Adds the current prefix to the list of paths that pkg-config searches
+
+ This is currently not optional as there is no observed adverse side
+ effects of enabling this. As the "prefix" is usually only valid during
+ a test run, we believe that having other pkg-config files (*.pc) in
+ either '<prefix>/share/pkgconfig' or '<prefix>/lib/pkgconfig' is
+ exactly for the purpose of using them.
+
+ @returns: None
+ '''
+ env_var = 'PKG_CONFIG_PATH'
+
+ include_paths = [os.path.join(self.prefix, 'share', 'pkgconfig'),
+ os.path.join(self.prefix, 'lib', 'pkgconfig')]
+
+ if os.environ.has_key(env_var):
+ paths = os.environ[env_var].split(':')
+ for include_path in include_paths:
+ if include_path not in paths:
+ paths.append(prefix_pkg_config)
+ os.environ[env_var] = ':'.join(paths)
+ else:
+ os.environ[env_var] = ':'.join(include_paths)
+
+ logging.debug('PKG_CONFIG_PATH is: %s' % os.environ['PKG_CONFIG_PATH'])
+
+
+ def get_configure_path(self):
+ '''
+ Checks if 'configure' exists, if not, return 'autogen.sh' as a fallback
+ '''
+ configure_path = os.path.abspath(os.path.join(self.source,
+ "configure"))
+ autogen_path = os.path.abspath(os.path.join(self.source,
+ "autogen.sh"))
+ if os.path.exists(configure_path):
+ return configure_path
+ elif os.path.exists(autogen_path):
+ return autogen_path
+ else:
+ raise GnuSourceBuildInvalidSource('configure script does not exist')
+
+
+ def get_available_configure_options(self):
+ '''
+ Return the list of available options of a GNU like configure script
+
+ This will run the "configure" script at the source directory
+
+ @returns: list of options accepted by configure script
+ '''
+ help_raw = utils.system_output('%s --help' % self.get_configure_path(),
+ ignore_status=True)
+ help_output = help_raw.split("\n")
+ option_list = []
+ for line in help_output:
+ cleaned_line = line.lstrip()
+ if cleaned_line.startswith("--"):
+ option = cleaned_line.split()[0]
+ option = option.split("=")[0]
+ option_list.append(option)
+
+ return option_list
+
+
+ def enable_debug_symbols(self):
+ '''
+ Enables option that leaves debug symbols on compiled software
+
+ This makes debugging a lot easier.
+ '''
+ enable_debug_option = "--disable-strip"
+ if enable_debug_option in self.get_available_configure_options():
+ self.configure_options.append(enable_debug_option)
+ logging.debug('Enabling debug symbols with option: %s' %
+ enable_debug_option)
+
+
+ def get_configure_command(self):
+ '''
+ Formats configure script with all options set
+
+ @returns: string with all configure options, including prefix
+ '''
+ prefix_option = "--prefix=%s" % self.prefix
+ options = self.configure_options
+ options.append(prefix_option)
+ return "%s %s" % (self.get_configure_path(),
+ " ".join(options))
+
+
+ def configure(self):
+ '''
+ Runs the "configure" script passing apropriate command line options
+ '''
+ configure_command = self.get_configure_command()
+ logging.info('Running configure on build dir')
+ os.chdir(self.build_dir)
+ utils.system(configure_command)
+
+
+ def make(self):
+ '''
+ Runs "make" using the correct number of parallel jobs
+ '''
+ parallel_make_jobs = utils.count_cpus()
+ make_command = "make -j %s" % parallel_make_jobs
+ logging.info("Running make on build dir")
+ os.chdir(self.build_dir)
+ utils.system(make_command)
+
+
+ def make_install(self):
+ '''
+ Runs "make install"
+ '''
+ os.chdir(self.build_dir)
+ utils.system("make install")
+
+
+ install = make_install
+
+
+ def execute(self):
+ '''
+ Runs appropriate steps for *building* this source code tree
+ '''
+ self.configure()
+ self.make()
+
+
+class GnuSourceBuildParamHelper(GnuSourceBuildHelper):
+ '''
+ Helps to deal with gnu_autotools build helper in cartersian config files
+
+ This class attempts to make it simple to build source coude, by using a
+ naming standard that follows this basic syntax:
+
+ [<git_repo>|<local_src>]_<name>_<option> = value
+
+ To pass extra options to the configure script, while building foo from a
+ git repo, set the following variable:
+
+ git_repo_foo_configure_options = --enable-feature
+ '''
+ def __init__(self, params, name, destination_dir, install_prefix):
+ '''
+ Instantiates a new GnuSourceBuildParamHelper
+ '''
+ self.params = params
+ self.name = name
+ self.destination_dir = destination_dir
+ self.install_prefix = install_prefix
+ self._parse_params()
+
+
+ def _parse_params(self):
+ '''
+ Parses the params items for entries related to source directory
+
+ This method currently does everything that the parent class __init__()
+ method does, that is, sets all instance variables needed by other
+ methods. That means it's not strictly necessary to call parent's
+ __init__().
+ '''
+ logging.debug('Parsing gnu_autotools build parameters for %s' %
+ self.name)
+
+ configure_opt_key = '%s_configure_options' % self.name
+ configure_options = self.params.get(configure_opt_key, '').split()
+ logging.debug('Configure options for %s: %s' % (self.name,
+ configure_options))
+
+ self.source = self.destination_dir
+ self.build_dir = self.destination_dir
+ self.prefix = self.install_prefix
+ self.configure_options = configure_options
+ self.include_pkg_config_path()
+
+
def install_host_kernel(job, params):
"""
Install a host kernel, given the appropriate params.
--
1.7.6
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
