This patch adds a software management abstraction layer
on top of popular distro package management abstraction
layers. Frequently we want to install a given distro
provided software package, but there's no single interface
to do so.
The software manager API tries to expose conveniently the
main package operations performed during software testing:
install() - Install a package or list of packages
remove() - Remove a package or list of packages
list_all() - List all packages installed
list_files() - List all files installed by a given package
add_repo() - Add a software repository
remove_repo() - Remove a software repository
So test writers don't have to care whether it is yum or
apt-get managing the system software. A rudimentary
command line interface for the API is provided, so
people can try out the methods.
The ideas of use inside autotest are:
* Replace all client.bin.package usage
* Usage with build_externals.py to install packages
* Provide a job.software_manager() attribute so people
can manipulate software install easily inside control
files and tests
The idea/very early draft came out more than 18 months
ago back in IBM (Ramon and Higor helped to write the
base methods), but only now I really sat to finish it
and sent it to the mailing list, to break the inertia :)
Tested/developed under the following systems:
* Fedora 13 (Yum backend)
* Ubuntu 10.04 (Apt backend)
* OpenSUSE 11.3 (Zypper backend)
Signed-off-by: Lucas Meneghel Rodrigues <lmr@xxxxxxxxxx>
Signed-off-by: Higor Vieira Alves <halves@xxxxxxxxxx>
Signed-off-by: Ramon de Carvalho Valle <rcvalle@xxxxxxxxxx>
---
client/common_lib/software_manager.py | 788 +++++++++++++++++++++++++++++++++
1 files changed, 788 insertions(+), 0 deletions(-)
create mode 100755 client/common_lib/software_manager.py
diff --git a/client/common_lib/software_manager.py b/client/common_lib/software_manager.py
new file mode 100755
index 0000000..f67f667
--- /dev/null
+++ b/client/common_lib/software_manager.py
@@ -0,0 +1,788 @@
+#!/usr/bin/python
+"""
+Software package management library.
+
+This is an abstraction layer on top of the existing distributions high level
+package managers. It supports package operations useful for testing purposes,
+and multiple high level package managers (here called backends). If you want
+to make this lib to support your particular package manager/distro, please
+implement the given backend class.
+
+@author: Higor Vieira Alves (halves@xxxxxxxxxx)
+@author: Lucas Meneghel Rodrigues (lmr@xxxxxxxxxx)
+@author: Ramon de Carvalho Valle (rcvalle@xxxxxxxxxx)
+
+@copyright: IBM 2008-2009
+@copyright: Red Hat 2009-2010
+"""
+import os, re, logging, ConfigParser, optparse, random, string
+try:
+ import yum
+except:
+ pass
+import common
+from autotest_lib.client.bin import os_dep, utils
+from autotest_lib.client.common_lib import error
+from autotest_lib.client.common_lib import logging_config, logging_manager
+
+
+def generate_random_string(length):
+ """
+ Return a random string using alphanumeric characters.
+
+ @length: Length of the string that will be generated.
+ """
+ r = random.SystemRandom()
+ str = ""
+ chars = string.letters + string.digits
+ while length > 0:
+ str += r.choice(chars)
+ length -= 1
+ return str
+
+
+class SoftwareManagerLoggingConfig(logging_config.LoggingConfig):
+ """
+ Used with the sole purpose of providing convenient logging setup
+ for the KVM test auxiliary programs.
+ """
+ def configure_logging(self, results_dir=None, verbose=False):
+ super(SoftwareManagerLoggingConfig, self).configure_logging(
+ use_console=True,
+ verbose=verbose)
+
+
+class SystemInspector(object):
+ """
+ System inspector class.
+
+ This may grow up to include more complete reports of operating system and
+ machine properties.
+ """
+ def __init__(self):
+ """
+ Probe system, and save information for future reference.
+ """
+ self.distro = utils.get_os_vendor()
+ self.high_level_pms = ['apt-get', 'yum', 'zypper']
+
+
+ def get_package_management(self):
+ """
+ Determine the supported package management systems present on the
+ system. If more than one package management system installed, try
+ to find the best supported system.
+ """
+ list_supported = []
+ for high_level_pm in self.high_level_pms:
+ try:
+ os_dep.command(high_level_pm)
+ list_supported.append(high_level_pm)
+ except:
+ pass
+
+ pm_supported = None
+ if len(list_supported) == 0:
+ pm_supported = None
+ if len(list_supported) == 1:
+ pm_supported = list_supported[0]
+ elif len(list_supported) > 1:
+ if 'apt-get' in list_supported and self.distro in ['Debian', 'Ubuntu']:
+ pm_supported = 'apt-get'
+ elif 'yum' in list_supported and self.distro == 'Fedora':
+ pm_supported = 'yum'
+ else:
+ pm_supported = list_supported[0]
+
+ logging.debug('Package Manager backend: %s' % pm_supported)
+ return pm_supported
+
+
+class SoftwareManager(object):
+ """
+ Package management abstraction layer.
+
+ It supports a set of common package operations for testing purposes, and it
+ uses the concept of a backend, a helper class that implements the set of
+ operations of a given package management tool.
+ """
+ def __init__(self):
+ """
+ Class constructor.
+
+ Determines the best supported package management system for the given
+ operating system running and initializes the appropriate backend.
+ """
+ inspector = SystemInspector()
+ backend_type = inspector.get_package_management()
+ if backend_type == 'yum':
+ self.backend = YumBackend()
+ elif backend_type == 'zypper':
+ self.backend = ZypperBackend()
+ elif backend_type == 'apt-get':
+ self.backend = AptBackend()
+ else:
+ raise NotImplementedError('Unimplemented package management '
+ 'system: %s.' % backend_type)
+
+
+ def check_installed(self, name, version=None, arch=None):
+ """
+ Check whether a package is installed on this system.
+
+ @param name: Package name.
+ @param version: Package version.
+ @param arch: Package architecture.
+ """
+ return self.backend.check_installed(name, version, arch)
+
+
+ def list_all(self):
+ """
+ List all installed packages.
+ """
+ return self.backend.list_all()
+
+
+ def list_files(self, name):
+ """
+ Get a list of all files installed by package [name].
+
+ @param name: Package name.
+ """
+ return self.backend.list_files(name)
+
+
+ def install(self, name):
+ """
+ Install package [name].
+
+ @param name: Package name.
+ """
+ return self.backend.install(name)
+
+
+ def remove(self, name):
+ """
+ Remove package [name].
+
+ @param name: Package name.
+ """
+ return self.backend.remove(name)
+
+
+ def add_repo(self, url):
+ """
+ Add package repo described by [url].
+
+ @param name: URL of the package repo.
+ """
+ return self.backend.add_repo(url)
+
+
+ def remove_repo(self, url):
+ """
+ Remove package repo described by [url].
+
+ @param url: URL of the package repo.
+ """
+ return self.backend.remove_repo(url)
+
+
+ def upgrade(self):
+ """
+ Upgrade all packages available.
+ """
+ return self.backend.upgrade()
+
+
+ def provides(self, file):
+ """
+ Returns a list of packages that provides a given capability to the
+ system (be it a binary, a library).
+
+ @param file: Path to the file.
+ """
+ return self.backend.provides(file)
+
+
+ def install_what_provides(self, file):
+ """
+ Installs package that provides [file].
+
+ @param file: Path to file.
+ """
+ provides = self.provides(file)
+ if provides is not None:
+ self.install(provides)
+ else:
+ logging.warning('No package seems to provide %s', file)
+
+
+class RpmBackend(object):
+ """
+ This class implements operations executed with the rpm package manager.
+
+ rpm is a lower level package manager, used by higher level managers such
+ as yum and zypper.
+ """
+ def __init__(self):
+ self.lowlevel_base_cmd = os_dep.command('rpm')
+
+
+ def _check_installed_version(self, name, version):
+ """
+ Helper for the check_installed public method.
+
+ @param name: Package name.
+ @param version: Package version.
+ """
+ cmd = (self.lowlevel_base_cmd + ' -q --qf %{VERSION} ' + name +
+ ' 2> /dev/null')
+ inst_version = utils.system_output(cmd)
+
+ if inst_version >= version:
+ return True
+ else:
+ return False
+
+
+ def check_installed(self, name, version=None, arch=None):
+ """
+ Check if package [name] is installed.
+
+ @param name: Package name.
+ @param version: Package version.
+ @param arch: Package architecture.
+ """
+ if arch:
+ cmd = (self.lowlevel_base_cmd + ' -q --qf %{ARCH} ' + name +
+ ' 2> /dev/null')
+ inst_archs = utils.system_output(cmd)
+ inst_archs = inst_archs.split('\n')
+
+ for inst_arch in inst_archs:
+ if inst_arch == arch:
+ return self._check_installed_version(name, version)
+ return False
+
+ elif version:
+ return self._check_installed_version(name, version)
+ else:
+ cmd = 'rpm -q ' + name + ' 2> /dev/null'
+ return (os.system(cmd) == 0)
+
+
+ def list_all(self):
+ """
+ List all installed packages.
+ """
+ installed_packages = utils.system_output('rpm -qa').splitlines()
+ return installed_packages
+
+
+ def list_files(self, name):
+ """
+ List files installed on the system by package [name].
+
+ @param name: Package name.
+ """
+ path = os.path.abspath(name)
+ if os.path.isfile(path):
+ option = '-qlp'
+ name = path
+ else:
+ option = '-ql'
+
+ l_cmd = 'rpm' + ' ' + option + ' ' + name + ' 2> /dev/null'
+
+ try:
+ result = utils.system_output(l_cmd)
+ list_files = result.split('\n')
+ return list_files
+ except error.CmdError:
+ return []
+
+
+class DpkgBackend(object):
+ """
+ This class implements operations executed with the dpkg package manager.
+
+ dpkg is a lower level package manager, used by higher level managers such
+ as apt and aptitude.
+ """
+ def __init__(self):
+ self.lowlevel_base_cmd = os_dep.command('dpkg')
+
+
+ def check_installed(self, name):
+ if os.path.isfile(name):
+ n_cmd = (self.lowlevel_base_cmd + ' -f ' + name +
+ ' Package 2>/dev/null')
+ name = utils.system_output(n_cmd)
+ i_cmd = self.lowlevel_base_cmd + ' -s ' + name + ' 2>/dev/null'
+ # Checking if package is installed
+ package_status = utils.system_output(i_cmd, ignore_status=True)
+ not_inst_pattern = re.compile('not-installed', re.IGNORECASE)
+ dpkg_not_installed = re.search(not_inst_pattern, package_status)
+ if dpkg_not_installed:
+ return False
+ return True
+
+
+ def list_all(self):
+ """
+ List all packages available in the system.
+ """
+ installed_packages = []
+ raw_list = utils.system_output('dpkg -l').splitlines()[5:]
+ for line in raw_list:
+ parts = line.split()
+ if parts[0] == "ii": # only grab "installed" packages
+ installed_packages.append("%s-%s" % (parts[1], parts[2]))
+
+
+ def list_files(self, package):
+ """
+ List files installed by package [package].
+
+ @param package: Package name.
+ @return: List of paths installed by package.
+ """
+ if os.path.isfile(package):
+ l_cmd = self.lowlevel_base_cmd + ' -c ' + package
+ else:
+ l_cmd = self.lowlevel_base_cmd + ' -l ' + package
+ return utils.system_output(l_cmd).split('\n')
+
+
+class YumBackend(RpmBackend):
+ """
+ Implements the yum backend for software manager.
+
+ Set of operations for the yum package manager, commonly found on Yellow Dog
+ Linux and Red Hat based distributions, such as Fedora and Red Hat
+ Enterprise Linux.
+ """
+ def __init__(self):
+ """
+ Initializes the base command and the yum package repository.
+ """
+ super(YumBackend, self).__init__()
+ executable = os_dep.command('yum')
+ base_arguments = '-y'
+ self.base_command = executable + ' ' + base_arguments
+ self.repo_file_path = '/etc/yum.repos.d/autotest.repo'
+ self.cfgparser = ConfigParser.ConfigParser()
+ self.cfgparser.read(self.repo_file_path)
+ y_cmd = executable + ' --version | head -1'
+ self.yum_version = utils.system_output(y_cmd, ignore_status=True)
+ logging.debug('Yum backend initialized')
+ logging.debug('Yum version: %s' % self.yum_version)
+ self.yum_base = yum.YumBase()
+
+
+ def _cleanup(self):
+ """
+ Clean up the yum cache so new package information can be downloaded.
+ """
+ utils.system("yum clean all")
+
+
+ def install(self, name):
+ """
+ Installs package [name]. Handles local installs.
+ """
+ if os.path.isfile(name):
+ name = os.path.abspath(name)
+ command = 'localinstall'
+ else:
+ command = 'install'
+
+ i_cmd = self.base_command + ' ' + command + ' ' + name
+
+ try:
+ utils.system(i_cmd)
+ return True
+ except:
+ return False
+
+
+ def remove(self, name):
+ """
+ Removes package [name].
+
+ @param name: Package name (eg. 'ipython').
+ """
+ r_cmd = self.base_command + ' ' + 'erase' + ' ' + name
+ try:
+ utils.system(r_cmd)
+ return True
+ except:
+ return False
+
+
+ def add_repo(self, url):
+ """
+ Adds package repository located on [url].
+
+ @param url: Universal Resource Locator of the repository.
+ """
+ # Check if we URL is already set
+ for section in self.cfgparser.sections():
+ for option, value in self.cfgparser.items(section):
+ if option == 'url' and value == url:
+ return True
+
+ # Didn't find it, let's set it up
+ while True:
+ section_name = 'software_manager' + '_' + generate_random_string(4)
+ if not self.cfgparser.has_section(section_name):
+ break
+ self.cfgparser.add_section(section_name)
+ self.cfgparser.set(section_name, 'name',
+ 'Repository added by the autotest software manager.')
+ self.cfgparser.set(section_name, 'url', url)
+ self.cfgparser.set(section_name, 'enabled', 1)
+ self.cfgparser.set(section_name, 'gpgcheck', 0)
+ self.cfgparser.write(self.repo_file_path)
+
+
+ def remove_repo(self, url):
+ """
+ Removes package repository located on [url].
+
+ @param url: Universal Resource Locator of the repository.
+ """
+ for section in self.cfgparser.sections():
+ for option, value in self.cfgparser.items(section):
+ if option == 'url' and value == url:
+ self.cfgparser.remove_section(section)
+ self.cfgparser.write(self.repo_file_path)
+
+
+ def upgrade(self):
+ """
+ Upgrade all available packages.
+ """
+ r_cmd = self.base_command + ' ' + 'update'
+ try:
+ utils.system(r_cmd)
+ return True
+ except:
+ return False
+
+
+ def provides(self, name):
+ """
+ Returns a list of packages that provides a given capability.
+
+ @param name: Capability name (eg, 'foo').
+ """
+ d_provides = self.yum_base.searchPackageProvides(args=[name])
+ provides_list = [key for key in d_provides]
+ if provides_list:
+ logging.info("Package %s provides %s", provides_list[0], name)
+ return str(provides_list[0])
+ else:
+ return None
+
+
+class ZypperBackend(RpmBackend):
+ """
+ Implements the zypper backend for software manager.
+
+ Set of operations for the zypper package manager, found on SUSE Linux.
+ """
+ def __init__(self):
+ """
+ Initializes the base command and the yum package repository.
+ """
+ super(ZypperBackend, self).__init__()
+ self.base_command = os_dep.command('zypper') + ' -n'
+ z_cmd = self.base_command + ' --version'
+ self.zypper_version = utils.system_output(z_cmd, ignore_status=True)
+ logging.debug('Zypper backend initialized')
+ logging.debug('Zypper version: %s' % self.zypper_version)
+
+
+ def install(self, name):
+ """
+ Installs package [name]. Handles local installs.
+
+ @param name: Package Name.
+ """
+ path = os.path.abspath(name)
+ i_cmd = self.base_command + ' install -l ' + name
+ try:
+ utils.system(i_cmd)
+ return True
+ except:
+ return False
+
+
+ def add_repo(self, url):
+ """
+ Adds repository [url].
+
+ @param url: URL for the package repository.
+ """
+ ar_cmd = self.base_command + ' addrepo ' + url
+ try:
+ utils.system(ar_cmd)
+ return True
+ except:
+ return False
+
+
+ def remove_repo(self, url):
+ """
+ Removes repository [url].
+
+ @param url: URL for the package repository.
+ """
+ rr_cmd = self.base_command + ' removerepo ' + url
+ try:
+ utils.system(rr_cmd)
+ return True
+ except:
+ return False
+
+
+ def remove(self, name):
+ """
+ Removes package [name].
+ """
+ r_cmd = self.base_command + ' ' + 'erase' + ' ' + name
+
+ try:
+ utils.system(r_cmd)
+ return True
+ except:
+ return False
+
+
+ def upgrade(self):
+ """
+ Upgrades all packages of the system.
+ """
+ u_cmd = self.base_command + ' update -l'
+
+ try:
+ utils.system(u_cmd)
+ return True
+ except:
+ return False
+
+
+ def provides(self, name):
+ """
+ Searches for what provides a given file.
+
+ @param name: File path.
+ """
+ p_cmd = self.base_command + ' what-provides ' + name
+ list_provides = []
+ try:
+ p_output = utils.system_output(p_cmd).split('\n')[4:]
+ for line in p_output:
+ line = [a.strip() for a in line.split('|')]
+ try:
+ state, pname, type, version, arch, repository = line
+ if pname not in list_provides:
+ list_provides.append(pname)
+ except IndexError:
+ pass
+ if len(list_provides) > 1:
+ logging.warning('More than one package found, '
+ 'opting by the first queue result')
+ if list_provides:
+ logging.info("Package %s provides %s", list_provides[0], name)
+ return list_provides[0]
+ return None
+ except:
+ return None
+
+
+class AptBackend(DpkgBackend):
+ """
+ Implements the apt backend for software manager.
+
+ Set of operations for the apt package manager, commonly found on Debian and
+ Debian based distributions, such as Ubuntu Linux.
+ """
+ def __init__(self):
+ """
+ Initializes the base command and the debian package repository.
+ """
+ super(AptBackend, self).__init__()
+ executable = os_dep.command('apt-get')
+ self.base_command = executable + ' -y'
+ self.repo_file_path = '/etc/apt/sources.list.d/autotest'
+ self.apt_version = utils.system_output('apt-get -v | head -1',
+ ignore_status=True)
+ logging.debug('Apt backend initialized')
+ logging.debug('apt version: %s' % self.apt_version)
+
+
+ def install(self, name):
+ """
+ Installs package [name].
+
+ @param name: Package name.
+ """
+ command = 'install'
+ i_cmd = self.base_command + ' ' + command + ' ' + name
+
+ try:
+ utils.system(i_cmd)
+ return True
+ except:
+ return False
+
+
+ def remove(self, name):
+ """
+ Remove package [name].
+
+ @param name: Package name.
+ """
+ command = 'remove'
+ flag = '--purge'
+ r_cmd = self.base_command + ' ' + command + ' ' + flag + ' ' + name
+
+ try:
+ utils.system(r_cmd)
+ return True
+ except:
+ return False
+
+
+ def add_repo(self, repo):
+ """
+ Add an apt repository.
+
+ @param repo: Repository string. Example:
+ 'deb http://archive.ubuntu.com/ubuntu/ maverick universe'
+ """
+ repo_file = open(self.repo_file_path, 'a')
+ repo_file_contents = repo_file.read()
+ if repo not in repo_file_contents:
+ repo_file.write(repo)
+
+
+ def remove_repo(self, repo):
+ """
+ Remove an apt repository.
+
+ @param repo: Repository string. Example:
+ 'deb http://archive.ubuntu.com/ubuntu/ maverick universe'
+ """
+ repo_file = open(self.repo_file_path, 'r')
+ new_file_contents = []
+ for line in repo_file.readlines:
+ if not line == repo:
+ new_file_contents.append(line)
+ repo_file.close()
+ new_file_contents = "\n".join(new_file_contents)
+ repo_file.open(self.repo_file_path, 'w')
+ repo_file.write(new_file_contents)
+ repo_file.close()
+
+
+ def upgrade(self):
+ """
+ Upgrade all packages of the system with eventual new versions.
+ """
+ ud_command = 'update'
+ ud_cmd = self.base_command + ' ' + ud_command
+ try:
+ utils.system(ud_cmd)
+ except:
+ logging.error("Apt package update failed")
+ up_command = 'upgrade'
+ up_cmd = self.base_command + ' ' + up_command
+ try:
+ utils.system(up_cmd)
+ return True
+ except:
+ return False
+
+
+ def provides(self, file):
+ """
+ Return a list of packages that provide [file].
+
+ @param file: File path.
+ """
+ if not self.check_installed('apt-file'):
+ self.install('apt-file')
+ command = os_dep.command('apt-file')
+ cache_update_cmd = command + ' update'
+ try:
+ utils.system(cache_update_cmd, ignore_status=True)
+ except:
+ logging.error("Apt file cache update failed")
+ fu_cmd = command + ' search ' + file
+ try:
+ provides = utils.system_output(fu_cmd).split('\n')
+ list_provides = []
+ for line in provides:
+ if line:
+ try:
+ line = line.split(':')
+ package = line[0].strip()
+ path = line[1].strip()
+ if path == file and package not in list_provides:
+ list_provides.append(package)
+ except IndexError:
+ pass
+ if len(list_provides) > 1:
+ logging.warning('More than one package found, '
+ 'opting by the first queue result')
+ if list_provides:
+ logging.info("Package %s provides %s", list_provides[0], file)
+ return list_provides[0]
+ return None
+ except:
+ return None
+
+
+if __name__ == '__main__':
+ parser = optparse.OptionParser(
+ "usage: %prog [install|remove|list-all|list-files|add-repo|remove-repo|"
+ "upgrade|what-provides|install-what-provides] arguments")
+ parser.add_option('--verbose', dest="debug", action='store_true',
+ help='include debug messages in console output')
+
+ options, args = parser.parse_args()
+ debug = options.debug
+ logging_manager.configure_logging(SoftwareManagerLoggingConfig(),
+ verbose=debug)
+ software_manager = SoftwareManager()
+ if args:
+ action = args[0]
+ args = " ".join(args[1:])
+ else:
+ action = 'show-help'
+
+ if action == 'install':
+ software_manager.install(args)
+ elif action == 'remove':
+ software_manager.remove(args)
+ if action == 'list-all':
+ software_manager.list_all()
+ elif action == 'list-files':
+ software_manager.list_files(args)
+ elif action == 'add-repo':
+ software_manager.add_repo(args)
+ elif action == 'remove-repo':
+ software_manager.remove_repo(args)
+ elif action == 'upgrade':
+ software_manager.upgrade()
+ elif action == 'what-provides':
+ software_manager.provides(args)
+ elif action == 'install-what-provides':
+ software_manager.install_what_provides(args)
+ elif action == 'show-help':
+ parser.print_help()
--
1.7.2.3
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
