I thought that I had updated the depcheck task on production taskotron a while back but it turns out that never happened and depcheck is hitting failures in production that it shouldn't be hitting. Original issue is: https://phab.qadevel.cloud.fedoraproject.org/T366 The fix was reviewed before updating dev/stg and has been running there more than 2 weeks now without issue. I'd like to update the depcheck task in production to fix some recurring issues that some devs have been complaining about. Diff is at the end of this email. Tim diff --git a/depcheck/__init__.py b/depcheck/__init__.py index ccac8a4..609eb13 100755 --- a/depcheck/__init__.py +++ b/depcheck/__init__.py @@ -36,6 +36,7 @@ import sys import argparse import tempfile import platform +import logging from . import depcheck from . import metadata @@ -91,7 +92,7 @@ def run(rpms, arch, repos, report_format="updates", workdir=None): arch = 'i686' # handle data formatting for repo metadata (list of dicts) - log.logger.debug("Prepairing metadata download") + log.logger.debug("Preparing metadata download") handled_repos = {} for repo, path in repos.items(): log.logger.debug(" - %r: %r, %r", repo, arch, path) @@ -121,7 +122,7 @@ def run(rpms, arch, repos, report_format="updates", workdir=None): # Sometimes, depcheck gets run with no rpms if not run_result: - log.logger.info("No results") + log.logger.warn("No results") output = squash_results.format_output(run_result, arch, report_format) log.logger.info(output) @@ -131,6 +132,7 @@ def run(rpms, arch, repos, report_format="updates", workdir=None): def depcheck_cli(): parser = argparse.ArgumentParser() parser.add_argument("rpm", nargs='+', help="RPMs to be tested") + parser.add_argument("-d", "--debug", action="store_true", help="enable debug logging for depcheck") parser.add_argument("-a", "--arch", choices=["i386", "x86_64", "armhfp"], help=""" Architecture to be tested. If omitted, defaults to the current machine's architecture. @@ -143,9 +145,17 @@ def depcheck_cli(): parser.add_argument("-f", "--format", choices=["rpms", "updates"], help=""" Specify output format to override configuration. """) - + parser.add_argument("-w", "--workdir", help="workdir to use for depcheck") args = parser.parse_args() + + # initialize logging when entering depcheck via cli - this won't be executed + # otherwise but in that case, taskotron is responsible for initialization + logging.basicConfig() + + if args.debug: + log.logger.setLevel(logging.DEBUG) + # autodetect arch if none is passed in arch = args.arch if arch is None: @@ -155,8 +165,8 @@ def depcheck_cli(): repos = dict([(repo, repo) for repo in args.repos]) try: - print run(args.rpm, arch, repos, args.format) - except KeyboardInterrupt, e: + run(args.rpm, arch, repos, args.format, args.workdir) + except KeyboardInterrupt: print >> sys.stderr, ("\n\nExiting on user cancel.") sys.exit(1) diff --git a/depcheck/depcheck.py b/depcheck/depcheck.py index bb0b37a..e5ba15c 100755 --- a/depcheck/depcheck.py +++ b/depcheck/depcheck.py @@ -30,7 +30,6 @@ import solv import tempfile from . import Pass, Fail -from . import exc from . import log @@ -77,6 +76,8 @@ class Depcheck(object): checking dependencies for the supplied repositories """ + log.logger.debug('preparing libsolv repos for depcheck run') + for repo in self.repos: primary = open(self.repos[repo]['primary'], 'rb') filelist = open(self.repos[repo]['filelists'], 'rb') @@ -85,37 +86,36 @@ class Depcheck(object): self.solvrepo.add_rpmmd(filelist, None, solv.Repo.REPO_EXTEND_SOLVABLES) - def _check_rpm(self, rpmfile): + def _check_rpm(self, rpmfile, solvable): """Check a single rpm file against prepared repodata :param rpmfile: filename of rpm file to check :type rpmfile: str + :param solvable: libsolv solvable from solvpool for the rpm to check + :type solvable: libsolv.Solver.Solvable """ - solvable = self.solvrepo.add_rpm(rpmfile) - self.solvpool.addfileprovides() - self.solvpool.createwhatprovides() + # the job setup here is mostly taken from libsolv's installcheck tool + # create a job using the predetermined solvable in the repo job = self.solvpool.Job(solv.Job.SOLVER_INSTALL | solv.Job.SOLVER_SOLVABLE, - solvable.id) + solvable.id) + self.solvpool.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, 1) - # important note - make sure to keep the solver in the same scope as - # the problem analysis code or else python-solv segfults will ensue - solvpool_solver = self.solvpool.Solver() - solv_problems = solvpool_solver.solve([job]) + # solve the deps for the input solvable's rpm and look for problems + solver = self.solvpool.Solver() + solv_problems = solver.solve([job]) if len(solv_problems) == 0: - return Pass, [] - + return {'result': Pass,'details':[]} else: problem_strs = [] for problem in solv_problems: probrules = problem.findallproblemrules() - for rule in probrules: ruleinfos = rule.allinfos() for info in ruleinfos: problem_strs.append(info.problemstr()) + return {'result':Fail, 'details':problem_strs} - return Fail, problem_strs def check_rpm_deps(self, rpmfiles): """Check dependencies of the given rpms against supplied repodata @@ -125,26 +125,33 @@ class Depcheck(object): :returns: dictionary of results {rpm: {'results': Pass/Fail, 'details': []}} """ - self._prepare_libsolv() + log.logger.debug('adding all rpms to libsolv repos and preparing solvpool') - run_result = {} + # add all rpmfiles that we're checking to the libsolv repo and save the + # solvable that is assigned to them. + # this makes libsolv aware of all the incoming rpms and allows it to + # check for any dep breakage between the rpms + self._prepare_libsolv() + solvables = {} for rpmfile in rpmfiles: - log.logger.debug('running depcheck for %s' % rpmfile) - result_state, result_details = self._check_rpm(rpmfile) + rpmname = rpmfile.split('/')[-1] + solvable = self.solvrepo.add_rpm(rpmfile) + solvables[rpmname] = solvable - if rpmfile in run_result: - if result_state != run_result[rpmfile]['result']: - #TODO maybe do something saner, ideas welcomed - raise exc.DepcheckException("RPM %r has multiple conflicting results" % rpmfile) - else: - run_result[rpmfile]['details'].append(result_details) + # now that the rpms have been added, prepare the solvpool + self.solvpool.addfileprovides() + self.solvpool.createwhatprovides() - else: - run_result[rpmfile] = {'result': result_state, - 'details': result_details} - return run_result + # check each rpm using its solvable that was assigned while adding to + # the repo + results = {} + for rpmname in solvables: + log.logger.debug('running depcheck for %s' % rpmname) + run_result = self._check_rpm(rpmname, solvables[rpmname]) + results[rpmname] = run_result + return results diff --git a/depcheck/squash_results.py b/depcheck/squash_results.py index 7376f0d..30330ca 100644 --- a/depcheck/squash_results.py +++ b/depcheck/squash_results.py @@ -1,4 +1,3 @@ -import os import json from . import Pass, Fail @@ -7,7 +6,6 @@ from . import log from libtaskotron.koji_utils import KojiClient from libtaskotron.bodhi_utils import BodhiUtils from libtaskotron import check -from libtaskotron.rpm_utils import rpmformat def _squash_rpms_to_builds(run_result, koji_client = None): """ diff --git a/tests/depcheck_scenarios b/tests/depcheck_scenarios index f1e5d57..3ad160a 160000 --- a/tests/depcheck_scenarios +++ b/tests/depcheck_scenarios @@ -1 +1 @@ -Subproject commit f1e5d57b0e68c53bd05c1b4d6eab3078706d269e +Subproject commit 3ad160ad3145dc4595286291e5e43e2d91737ea9 diff --git a/tests/functest_depcheck_scenarios.py b/tests/functest_depcheck_scenarios.py index dc69e07..3f5733d 100644 --- a/tests/functest_depcheck_scenarios.py +++ b/tests/functest_depcheck_scenarios.py @@ -36,9 +36,12 @@ def get_depcheck_scenarios(): for dircontent in dircontents: if len(dircontent[2]) == 0: continue + if dircontent[1] == 'known_broken': + continue for infile in dircontent[2]: filename = '%s/%s' % (dircontent[0], infile) - yamlfiles.append((scenario_type == 'pass', filename)) + if infile.split('.')[-1] in ['yml', 'yaml']: + yamlfiles.append((scenario_type == 'pass', filename)) return yamlfiles diff --git a/tests/test_depcheck.py b/tests/test_depcheck.py index d27d011..909b2ad 100644 --- a/tests/test_depcheck.py +++ b/tests/test_depcheck.py @@ -36,10 +36,10 @@ class TestDepcheck(object): monkeypatch.setattr(test_depcheck, '_prepare_libsolv', Dingus()) rpmfiles = ['rpm_1', 'rpm_2'] - def fake_check_rpm(rpmfile): + def fake_check_rpm(rpmfile, solvable): results = { - rpmfiles[0]: (Pass, []), - rpmfiles[1]: (Fail, ['message']), + rpmfiles[0]: {'result':Pass, 'details':[]}, + rpmfiles[1]: {'result':Fail, 'details':['message']}, } return results.get(rpmfile, (Pass, [])) @@ -54,25 +54,3 @@ class TestDepcheck(object): assert outcome['rpm_2']['result'] == Fail assert outcome['rpm_2']['details'] == ['message'] - def test_check_rpm_deps_conflicting_results(self, monkeypatch): - """ - Checks whether rpm_deps raises Exceptions when conflicting - results are provided for one RPM. - """ - - test_depcheck = depcheck.Depcheck(None, None, "fakedir") - monkeypatch.setattr(test_depcheck, '_prepare_libsolv', Dingus()) - - self.tmp_result = Pass - def fake_check_rpm(rpmfile): - if self.tmp_result == Pass: - self.tmp_result = Fail - else: - self.tmp_result = Pass - return (self.tmp_result, []) - - monkeypatch.setattr(test_depcheck, '_check_rpm', fake_check_rpm) - - with pytest.raises(exc.DepcheckException): - outcome = test_depcheck.check_rpm_deps(['rpm1', 'rpm1']) -
Attachment:
pgpyo8JS61x7f.pgp
Description: OpenPGP digital signature
_______________________________________________ infrastructure mailing list infrastructure@xxxxxxxxxxxxxxxxxxxxxxx https://admin.fedoraproject.org/mailman/listinfo/infrastructure
