On Mon, 2010-11-29 at 00:44 -0200, Lucas Meneghel Rodrigues wrote:
> This patch adds a software management abstraction layer
> on top of popular distro package management abstraction
> layers. Frequently we want to install a given distro
> provided software package, but there's no single interface
> to do so.
>
> The software manager API tries to expose conveniently the
> main package operations performed during software testing:
>
> install() - Install a package or list of packages
> remove() - Remove a package or list of packages
> list_all() - List all packages installed
> list_files() - List all files installed by a given package
> add_repo() - Add a software repository
> remove_repo() - Remove a software repository
I forgot to mention
provides() - Return a list of packages that provide '/path/to/file'
install_what_provides() - Install package that provides '/path/to/file'
>
> So test writers don't have to care whether it is yum or
> apt-get managing the system software. A rudimentary
> command line interface for the API is provided, so
> people can try out the methods.
>
> The ideas of use inside autotest are:
> * Replace all client.bin.package usage
> * Usage with build_externals.py to install packages
> * Provide a job.software_manager() attribute so people
> can manipulate software install easily inside control
> files and tests
>
> The idea/very early draft came out more than 18 months
> ago back in IBM (Ramon and Higor helped to write the
> base methods), but only now I really sat to finish it
> and sent it to the mailing list, to break the inertia :)
>
> Tested/developed under the following systems:
> * Fedora 13 (Yum backend)
> * Ubuntu 10.04 (Apt backend)
> * OpenSUSE 11.3 (Zypper backend)
>
> Signed-off-by: Lucas Meneghel Rodrigues <lmr@xxxxxxxxxx>
> Signed-off-by: Higor Vieira Alves <halves@xxxxxxxxxx>
> Signed-off-by: Ramon de Carvalho Valle <rcvalle@xxxxxxxxxx>
> ---
> client/common_lib/software_manager.py | 788 +++++++++++++++++++++++++++++++++
> 1 files changed, 788 insertions(+), 0 deletions(-)
> create mode 100755 client/common_lib/software_manager.py
>
> diff --git a/client/common_lib/software_manager.py b/client/common_lib/software_manager.py
> new file mode 100755
> index 0000000..f67f667
> --- /dev/null
> +++ b/client/common_lib/software_manager.py
> @@ -0,0 +1,788 @@
> +#!/usr/bin/python
> +"""
> +Software package management library.
> +
> +This is an abstraction layer on top of the existing distributions high level
> +package managers. It supports package operations useful for testing purposes,
> +and multiple high level package managers (here called backends). If you want
> +to make this lib to support your particular package manager/distro, please
> +implement the given backend class.
> +
> +@author: Higor Vieira Alves (halves@xxxxxxxxxx)
> +@author: Lucas Meneghel Rodrigues (lmr@xxxxxxxxxx)
> +@author: Ramon de Carvalho Valle (rcvalle@xxxxxxxxxx)
> +
> +@copyright: IBM 2008-2009
> +@copyright: Red Hat 2009-2010
> +"""
> +import os, re, logging, ConfigParser, optparse, random, string
> +try:
> + import yum
> +except:
> + pass
> +import common
> +from autotest_lib.client.bin import os_dep, utils
> +from autotest_lib.client.common_lib import error
> +from autotest_lib.client.common_lib import logging_config, logging_manager
> +
> +
> +def generate_random_string(length):
> + """
> + Return a random string using alphanumeric characters.
> +
> + @length: Length of the string that will be generated.
> + """
> + r = random.SystemRandom()
> + str = ""
> + chars = string.letters + string.digits
> + while length > 0:
> + str += r.choice(chars)
> + length -= 1
> + return str
> +
> +
> +class SoftwareManagerLoggingConfig(logging_config.LoggingConfig):
> + """
> + Used with the sole purpose of providing convenient logging setup
> + for the KVM test auxiliary programs.
> + """
> + def configure_logging(self, results_dir=None, verbose=False):
> + super(SoftwareManagerLoggingConfig, self).configure_logging(
> + use_console=True,
> + verbose=verbose)
> +
> +
> +class SystemInspector(object):
> + """
> + System inspector class.
> +
> + This may grow up to include more complete reports of operating system and
> + machine properties.
> + """
> + def __init__(self):
> + """
> + Probe system, and save information for future reference.
> + """
> + self.distro = utils.get_os_vendor()
> + self.high_level_pms = ['apt-get', 'yum', 'zypper']
> +
> +
> + def get_package_management(self):
> + """
> + Determine the supported package management systems present on the
> + system. If more than one package management system installed, try
> + to find the best supported system.
> + """
> + list_supported = []
> + for high_level_pm in self.high_level_pms:
> + try:
> + os_dep.command(high_level_pm)
> + list_supported.append(high_level_pm)
> + except:
> + pass
> +
> + pm_supported = None
> + if len(list_supported) == 0:
> + pm_supported = None
> + if len(list_supported) == 1:
> + pm_supported = list_supported[0]
> + elif len(list_supported) > 1:
> + if 'apt-get' in list_supported and self.distro in ['Debian', 'Ubuntu']:
> + pm_supported = 'apt-get'
> + elif 'yum' in list_supported and self.distro == 'Fedora':
> + pm_supported = 'yum'
> + else:
> + pm_supported = list_supported[0]
> +
> + logging.debug('Package Manager backend: %s' % pm_supported)
> + return pm_supported
> +
> +
> +class SoftwareManager(object):
> + """
> + Package management abstraction layer.
> +
> + It supports a set of common package operations for testing purposes, and it
> + uses the concept of a backend, a helper class that implements the set of
> + operations of a given package management tool.
> + """
> + def __init__(self):
> + """
> + Class constructor.
> +
> + Determines the best supported package management system for the given
> + operating system running and initializes the appropriate backend.
> + """
> + inspector = SystemInspector()
> + backend_type = inspector.get_package_management()
> + if backend_type == 'yum':
> + self.backend = YumBackend()
> + elif backend_type == 'zypper':
> + self.backend = ZypperBackend()
> + elif backend_type == 'apt-get':
> + self.backend = AptBackend()
> + else:
> + raise NotImplementedError('Unimplemented package management '
> + 'system: %s.' % backend_type)
> +
> +
> + def check_installed(self, name, version=None, arch=None):
> + """
> + Check whether a package is installed on this system.
> +
> + @param name: Package name.
> + @param version: Package version.
> + @param arch: Package architecture.
> + """
> + return self.backend.check_installed(name, version, arch)
> +
> +
> + def list_all(self):
> + """
> + List all installed packages.
> + """
> + return self.backend.list_all()
> +
> +
> + def list_files(self, name):
> + """
> + Get a list of all files installed by package [name].
> +
> + @param name: Package name.
> + """
> + return self.backend.list_files(name)
> +
> +
> + def install(self, name):
> + """
> + Install package [name].
> +
> + @param name: Package name.
> + """
> + return self.backend.install(name)
> +
> +
> + def remove(self, name):
> + """
> + Remove package [name].
> +
> + @param name: Package name.
> + """
> + return self.backend.remove(name)
> +
> +
> + def add_repo(self, url):
> + """
> + Add package repo described by [url].
> +
> + @param name: URL of the package repo.
> + """
> + return self.backend.add_repo(url)
> +
> +
> + def remove_repo(self, url):
> + """
> + Remove package repo described by [url].
> +
> + @param url: URL of the package repo.
> + """
> + return self.backend.remove_repo(url)
> +
> +
> + def upgrade(self):
> + """
> + Upgrade all packages available.
> + """
> + return self.backend.upgrade()
> +
> +
> + def provides(self, file):
> + """
> + Returns a list of packages that provides a given capability to the
> + system (be it a binary, a library).
> +
> + @param file: Path to the file.
> + """
> + return self.backend.provides(file)
> +
> +
> + def install_what_provides(self, file):
> + """
> + Installs package that provides [file].
> +
> + @param file: Path to file.
> + """
> + provides = self.provides(file)
> + if provides is not None:
> + self.install(provides)
> + else:
> + logging.warning('No package seems to provide %s', file)
> +
> +
> +class RpmBackend(object):
> + """
> + This class implements operations executed with the rpm package manager.
> +
> + rpm is a lower level package manager, used by higher level managers such
> + as yum and zypper.
> + """
> + def __init__(self):
> + self.lowlevel_base_cmd = os_dep.command('rpm')
> +
> +
> + def _check_installed_version(self, name, version):
> + """
> + Helper for the check_installed public method.
> +
> + @param name: Package name.
> + @param version: Package version.
> + """
> + cmd = (self.lowlevel_base_cmd + ' -q --qf %{VERSION} ' + name +
> + ' 2> /dev/null')
> + inst_version = utils.system_output(cmd)
> +
> + if inst_version >= version:
> + return True
> + else:
> + return False
> +
> +
> + def check_installed(self, name, version=None, arch=None):
> + """
> + Check if package [name] is installed.
> +
> + @param name: Package name.
> + @param version: Package version.
> + @param arch: Package architecture.
> + """
> + if arch:
> + cmd = (self.lowlevel_base_cmd + ' -q --qf %{ARCH} ' + name +
> + ' 2> /dev/null')
> + inst_archs = utils.system_output(cmd)
> + inst_archs = inst_archs.split('\n')
> +
> + for inst_arch in inst_archs:
> + if inst_arch == arch:
> + return self._check_installed_version(name, version)
> + return False
> +
> + elif version:
> + return self._check_installed_version(name, version)
> + else:
> + cmd = 'rpm -q ' + name + ' 2> /dev/null'
> + return (os.system(cmd) == 0)
> +
> +
> + def list_all(self):
> + """
> + List all installed packages.
> + """
> + installed_packages = utils.system_output('rpm -qa').splitlines()
> + return installed_packages
> +
> +
> + def list_files(self, name):
> + """
> + List files installed on the system by package [name].
> +
> + @param name: Package name.
> + """
> + path = os.path.abspath(name)
> + if os.path.isfile(path):
> + option = '-qlp'
> + name = path
> + else:
> + option = '-ql'
> +
> + l_cmd = 'rpm' + ' ' + option + ' ' + name + ' 2> /dev/null'
> +
> + try:
> + result = utils.system_output(l_cmd)
> + list_files = result.split('\n')
> + return list_files
> + except error.CmdError:
> + return []
> +
> +
> +class DpkgBackend(object):
> + """
> + This class implements operations executed with the dpkg package manager.
> +
> + dpkg is a lower level package manager, used by higher level managers such
> + as apt and aptitude.
> + """
> + def __init__(self):
> + self.lowlevel_base_cmd = os_dep.command('dpkg')
> +
> +
> + def check_installed(self, name):
> + if os.path.isfile(name):
> + n_cmd = (self.lowlevel_base_cmd + ' -f ' + name +
> + ' Package 2>/dev/null')
> + name = utils.system_output(n_cmd)
> + i_cmd = self.lowlevel_base_cmd + ' -s ' + name + ' 2>/dev/null'
> + # Checking if package is installed
> + package_status = utils.system_output(i_cmd, ignore_status=True)
> + not_inst_pattern = re.compile('not-installed', re.IGNORECASE)
> + dpkg_not_installed = re.search(not_inst_pattern, package_status)
> + if dpkg_not_installed:
> + return False
> + return True
> +
> +
> + def list_all(self):
> + """
> + List all packages available in the system.
> + """
> + installed_packages = []
> + raw_list = utils.system_output('dpkg -l').splitlines()[5:]
> + for line in raw_list:
> + parts = line.split()
> + if parts[0] == "ii": # only grab "installed" packages
> + installed_packages.append("%s-%s" % (parts[1], parts[2]))
> +
> +
> + def list_files(self, package):
> + """
> + List files installed by package [package].
> +
> + @param package: Package name.
> + @return: List of paths installed by package.
> + """
> + if os.path.isfile(package):
> + l_cmd = self.lowlevel_base_cmd + ' -c ' + package
> + else:
> + l_cmd = self.lowlevel_base_cmd + ' -l ' + package
> + return utils.system_output(l_cmd).split('\n')
> +
> +
> +class YumBackend(RpmBackend):
> + """
> + Implements the yum backend for software manager.
> +
> + Set of operations for the yum package manager, commonly found on Yellow Dog
> + Linux and Red Hat based distributions, such as Fedora and Red Hat
> + Enterprise Linux.
> + """
> + def __init__(self):
> + """
> + Initializes the base command and the yum package repository.
> + """
> + super(YumBackend, self).__init__()
> + executable = os_dep.command('yum')
> + base_arguments = '-y'
> + self.base_command = executable + ' ' + base_arguments
> + self.repo_file_path = '/etc/yum.repos.d/autotest.repo'
> + self.cfgparser = ConfigParser.ConfigParser()
> + self.cfgparser.read(self.repo_file_path)
> + y_cmd = executable + ' --version | head -1'
> + self.yum_version = utils.system_output(y_cmd, ignore_status=True)
> + logging.debug('Yum backend initialized')
> + logging.debug('Yum version: %s' % self.yum_version)
> + self.yum_base = yum.YumBase()
> +
> +
> + def _cleanup(self):
> + """
> + Clean up the yum cache so new package information can be downloaded.
> + """
> + utils.system("yum clean all")
> +
> +
> + def install(self, name):
> + """
> + Installs package [name]. Handles local installs.
> + """
> + if os.path.isfile(name):
> + name = os.path.abspath(name)
> + command = 'localinstall'
> + else:
> + command = 'install'
> +
> + i_cmd = self.base_command + ' ' + command + ' ' + name
> +
> + try:
> + utils.system(i_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def remove(self, name):
> + """
> + Removes package [name].
> +
> + @param name: Package name (eg. 'ipython').
> + """
> + r_cmd = self.base_command + ' ' + 'erase' + ' ' + name
> + try:
> + utils.system(r_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def add_repo(self, url):
> + """
> + Adds package repository located on [url].
> +
> + @param url: Universal Resource Locator of the repository.
> + """
> + # Check if we URL is already set
> + for section in self.cfgparser.sections():
> + for option, value in self.cfgparser.items(section):
> + if option == 'url' and value == url:
> + return True
> +
> + # Didn't find it, let's set it up
> + while True:
> + section_name = 'software_manager' + '_' + generate_random_string(4)
> + if not self.cfgparser.has_section(section_name):
> + break
> + self.cfgparser.add_section(section_name)
> + self.cfgparser.set(section_name, 'name',
> + 'Repository added by the autotest software manager.')
> + self.cfgparser.set(section_name, 'url', url)
> + self.cfgparser.set(section_name, 'enabled', 1)
> + self.cfgparser.set(section_name, 'gpgcheck', 0)
> + self.cfgparser.write(self.repo_file_path)
> +
> +
> + def remove_repo(self, url):
> + """
> + Removes package repository located on [url].
> +
> + @param url: Universal Resource Locator of the repository.
> + """
> + for section in self.cfgparser.sections():
> + for option, value in self.cfgparser.items(section):
> + if option == 'url' and value == url:
> + self.cfgparser.remove_section(section)
> + self.cfgparser.write(self.repo_file_path)
> +
> +
> + def upgrade(self):
> + """
> + Upgrade all available packages.
> + """
> + r_cmd = self.base_command + ' ' + 'update'
> + try:
> + utils.system(r_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def provides(self, name):
> + """
> + Returns a list of packages that provides a given capability.
> +
> + @param name: Capability name (eg, 'foo').
> + """
> + d_provides = self.yum_base.searchPackageProvides(args=[name])
> + provides_list = [key for key in d_provides]
> + if provides_list:
> + logging.info("Package %s provides %s", provides_list[0], name)
> + return str(provides_list[0])
> + else:
> + return None
> +
> +
> +class ZypperBackend(RpmBackend):
> + """
> + Implements the zypper backend for software manager.
> +
> + Set of operations for the zypper package manager, found on SUSE Linux.
> + """
> + def __init__(self):
> + """
> + Initializes the base command and the yum package repository.
> + """
> + super(ZypperBackend, self).__init__()
> + self.base_command = os_dep.command('zypper') + ' -n'
> + z_cmd = self.base_command + ' --version'
> + self.zypper_version = utils.system_output(z_cmd, ignore_status=True)
> + logging.debug('Zypper backend initialized')
> + logging.debug('Zypper version: %s' % self.zypper_version)
> +
> +
> + def install(self, name):
> + """
> + Installs package [name]. Handles local installs.
> +
> + @param name: Package Name.
> + """
> + path = os.path.abspath(name)
> + i_cmd = self.base_command + ' install -l ' + name
> + try:
> + utils.system(i_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def add_repo(self, url):
> + """
> + Adds repository [url].
> +
> + @param url: URL for the package repository.
> + """
> + ar_cmd = self.base_command + ' addrepo ' + url
> + try:
> + utils.system(ar_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def remove_repo(self, url):
> + """
> + Removes repository [url].
> +
> + @param url: URL for the package repository.
> + """
> + rr_cmd = self.base_command + ' removerepo ' + url
> + try:
> + utils.system(rr_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def remove(self, name):
> + """
> + Removes package [name].
> + """
> + r_cmd = self.base_command + ' ' + 'erase' + ' ' + name
> +
> + try:
> + utils.system(r_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def upgrade(self):
> + """
> + Upgrades all packages of the system.
> + """
> + u_cmd = self.base_command + ' update -l'
> +
> + try:
> + utils.system(u_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def provides(self, name):
> + """
> + Searches for what provides a given file.
> +
> + @param name: File path.
> + """
> + p_cmd = self.base_command + ' what-provides ' + name
> + list_provides = []
> + try:
> + p_output = utils.system_output(p_cmd).split('\n')[4:]
> + for line in p_output:
> + line = [a.strip() for a in line.split('|')]
> + try:
> + state, pname, type, version, arch, repository = line
> + if pname not in list_provides:
> + list_provides.append(pname)
> + except IndexError:
> + pass
> + if len(list_provides) > 1:
> + logging.warning('More than one package found, '
> + 'opting by the first queue result')
> + if list_provides:
> + logging.info("Package %s provides %s", list_provides[0], name)
> + return list_provides[0]
> + return None
> + except:
> + return None
> +
> +
> +class AptBackend(DpkgBackend):
> + """
> + Implements the apt backend for software manager.
> +
> + Set of operations for the apt package manager, commonly found on Debian and
> + Debian based distributions, such as Ubuntu Linux.
> + """
> + def __init__(self):
> + """
> + Initializes the base command and the debian package repository.
> + """
> + super(AptBackend, self).__init__()
> + executable = os_dep.command('apt-get')
> + self.base_command = executable + ' -y'
> + self.repo_file_path = '/etc/apt/sources.list.d/autotest'
> + self.apt_version = utils.system_output('apt-get -v | head -1',
> + ignore_status=True)
> + logging.debug('Apt backend initialized')
> + logging.debug('apt version: %s' % self.apt_version)
> +
> +
> + def install(self, name):
> + """
> + Installs package [name].
> +
> + @param name: Package name.
> + """
> + command = 'install'
> + i_cmd = self.base_command + ' ' + command + ' ' + name
> +
> + try:
> + utils.system(i_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def remove(self, name):
> + """
> + Remove package [name].
> +
> + @param name: Package name.
> + """
> + command = 'remove'
> + flag = '--purge'
> + r_cmd = self.base_command + ' ' + command + ' ' + flag + ' ' + name
> +
> + try:
> + utils.system(r_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def add_repo(self, repo):
> + """
> + Add an apt repository.
> +
> + @param repo: Repository string. Example:
> + 'deb http://archive.ubuntu.com/ubuntu/ maverick universe'
> + """
> + repo_file = open(self.repo_file_path, 'a')
> + repo_file_contents = repo_file.read()
> + if repo not in repo_file_contents:
> + repo_file.write(repo)
> +
> +
> + def remove_repo(self, repo):
> + """
> + Remove an apt repository.
> +
> + @param repo: Repository string. Example:
> + 'deb http://archive.ubuntu.com/ubuntu/ maverick universe'
> + """
> + repo_file = open(self.repo_file_path, 'r')
> + new_file_contents = []
> + for line in repo_file.readlines:
> + if not line == repo:
> + new_file_contents.append(line)
> + repo_file.close()
> + new_file_contents = "\n".join(new_file_contents)
> + repo_file.open(self.repo_file_path, 'w')
> + repo_file.write(new_file_contents)
> + repo_file.close()
> +
> +
> + def upgrade(self):
> + """
> + Upgrade all packages of the system with eventual new versions.
> + """
> + ud_command = 'update'
> + ud_cmd = self.base_command + ' ' + ud_command
> + try:
> + utils.system(ud_cmd)
> + except:
> + logging.error("Apt package update failed")
> + up_command = 'upgrade'
> + up_cmd = self.base_command + ' ' + up_command
> + try:
> + utils.system(up_cmd)
> + return True
> + except:
> + return False
> +
> +
> + def provides(self, file):
> + """
> + Return a list of packages that provide [file].
> +
> + @param file: File path.
> + """
> + if not self.check_installed('apt-file'):
> + self.install('apt-file')
> + command = os_dep.command('apt-file')
> + cache_update_cmd = command + ' update'
> + try:
> + utils.system(cache_update_cmd, ignore_status=True)
> + except:
> + logging.error("Apt file cache update failed")
> + fu_cmd = command + ' search ' + file
> + try:
> + provides = utils.system_output(fu_cmd).split('\n')
> + list_provides = []
> + for line in provides:
> + if line:
> + try:
> + line = line.split(':')
> + package = line[0].strip()
> + path = line[1].strip()
> + if path == file and package not in list_provides:
> + list_provides.append(package)
> + except IndexError:
> + pass
> + if len(list_provides) > 1:
> + logging.warning('More than one package found, '
> + 'opting by the first queue result')
> + if list_provides:
> + logging.info("Package %s provides %s", list_provides[0], file)
> + return list_provides[0]
> + return None
> + except:
> + return None
> +
> +
> +if __name__ == '__main__':
> + parser = optparse.OptionParser(
> + "usage: %prog [install|remove|list-all|list-files|add-repo|remove-repo|"
> + "upgrade|what-provides|install-what-provides] arguments")
> + parser.add_option('--verbose', dest="debug", action='store_true',
> + help='include debug messages in console output')
> +
> + options, args = parser.parse_args()
> + debug = options.debug
> + logging_manager.configure_logging(SoftwareManagerLoggingConfig(),
> + verbose=debug)
> + software_manager = SoftwareManager()
> + if args:
> + action = args[0]
> + args = " ".join(args[1:])
> + else:
> + action = 'show-help'
> +
> + if action == 'install':
> + software_manager.install(args)
> + elif action == 'remove':
> + software_manager.remove(args)
> + if action == 'list-all':
> + software_manager.list_all()
> + elif action == 'list-files':
> + software_manager.list_files(args)
> + elif action == 'add-repo':
> + software_manager.add_repo(args)
> + elif action == 'remove-repo':
> + software_manager.remove_repo(args)
> + elif action == 'upgrade':
> + software_manager.upgrade()
> + elif action == 'what-provides':
> + software_manager.provides(args)
> + elif action == 'install-what-provides':
> + software_manager.install_what_provides(args)
> + elif action == 'show-help':
> + parser.print_help()
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
