Freeze Break Request: new depcheck on taskotron clients

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



I thought that I had updated the depcheck task on production taskotron
a while back but it turns out that never happened and depcheck is
hitting failures in production that it shouldn't be hitting. Original
issue is:

https://phab.qadevel.cloud.fedoraproject.org/T366

The fix was reviewed before updating dev/stg and has been running
there more than 2 weeks now without issue.

I'd like to update the depcheck task in production to fix some
recurring issues that some devs have been complaining about. Diff is at
the end of this email.

Tim


diff --git a/depcheck/__init__.py b/depcheck/__init__.py
index ccac8a4..609eb13 100755
--- a/depcheck/__init__.py
+++ b/depcheck/__init__.py
@@ -36,6 +36,7 @@ import sys
 import argparse
 import tempfile
 import platform
+import logging
 
 from . import depcheck
 from . import metadata
@@ -91,7 +92,7 @@ def run(rpms, arch, repos, report_format="updates",
workdir=None): arch = 'i686'
 
     # handle data formatting for repo metadata (list of dicts)
-    log.logger.debug("Prepairing metadata download")
+    log.logger.debug("Preparing metadata download")
     handled_repos = {}
     for repo, path in repos.items():
         log.logger.debug("  - %r: %r, %r", repo, arch, path)
@@ -121,7 +122,7 @@ def run(rpms, arch, repos, report_format="updates",
workdir=None): 
     # Sometimes, depcheck gets run with no rpms
     if not run_result:
-        log.logger.info("No results")
+        log.logger.warn("No results")
 
     output = squash_results.format_output(run_result, arch,
report_format) log.logger.info(output)
@@ -131,6 +132,7 @@ def run(rpms, arch, repos, report_format="updates",
workdir=None): def depcheck_cli():
     parser = argparse.ArgumentParser()
     parser.add_argument("rpm", nargs='+', help="RPMs to be tested")
+    parser.add_argument("-d", "--debug", action="store_true",
help="enable debug logging for depcheck") parser.add_argument("-a",
"--arch", choices=["i386", "x86_64", "armhfp"], help=""" Architecture
to be tested. If omitted, defaults to the current machine's
architecture. @@ -143,9 +145,17 @@ def depcheck_cli():
     parser.add_argument("-f", "--format", choices=["rpms", "updates"],
help=""" Specify output format to override configuration.
                         """)
-
+    parser.add_argument("-w", "--workdir", help="workdir to use for
depcheck") 
     args = parser.parse_args()
+
+    # initialize logging when entering depcheck via cli - this won't
be executed
+    # otherwise but in that case, taskotron is responsible for
initialization
+    logging.basicConfig()
+
+    if args.debug:
+        log.logger.setLevel(logging.DEBUG)
+
     # autodetect arch if none is passed in
     arch = args.arch
     if arch is None:
@@ -155,8 +165,8 @@ def depcheck_cli():
     repos = dict([(repo, repo) for repo in args.repos])
 
     try:
-        print run(args.rpm, arch, repos, args.format)
-    except KeyboardInterrupt, e:
+        run(args.rpm, arch, repos, args.format, args.workdir)
+    except KeyboardInterrupt:
         print >> sys.stderr, ("\n\nExiting on user cancel.")
         sys.exit(1)
 
diff --git a/depcheck/depcheck.py b/depcheck/depcheck.py
index bb0b37a..e5ba15c 100755
--- a/depcheck/depcheck.py
+++ b/depcheck/depcheck.py
@@ -30,7 +30,6 @@ import solv
 import tempfile
 
 from . import Pass, Fail
-from . import exc
 from . import log
 
 
@@ -77,6 +76,8 @@ class Depcheck(object):
         checking dependencies for the supplied repositories
         """
 
+        log.logger.debug('preparing libsolv repos for depcheck run')
+
         for repo in self.repos:
             primary = open(self.repos[repo]['primary'], 'rb')
             filelist = open(self.repos[repo]['filelists'], 'rb')
@@ -85,37 +86,36 @@ class Depcheck(object):
             self.solvrepo.add_rpmmd(filelist, None,
                                     solv.Repo.REPO_EXTEND_SOLVABLES)
 
-    def _check_rpm(self, rpmfile):
+    def _check_rpm(self, rpmfile, solvable):
         """Check a single rpm file against prepared repodata
          :param rpmfile: filename of rpm file to check
          :type rpmfile: str
+         :param solvable: libsolv solvable from solvpool for the rpm
to check
+         :type solvable: libsolv.Solver.Solvable
         """
-        solvable = self.solvrepo.add_rpm(rpmfile)
-        self.solvpool.addfileprovides()
-        self.solvpool.createwhatprovides()
 
+        # the job setup here is mostly taken from libsolv's
installcheck tool
+        # create a job using the predetermined solvable in the repo
         job = self.solvpool.Job(solv.Job.SOLVER_INSTALL |
solv.Job.SOLVER_SOLVABLE,
-                                solvable.id)
+                solvable.id)
+
self.solvpool.set_flag(solv.Solver.SOLVER_FLAG_IGNORE_RECOMMENDED, 1) 
-        # important note - make sure to keep the solver in the same
scope as
-        # the problem analysis code or else python-solv segfults will
ensue
-        solvpool_solver = self.solvpool.Solver()
-        solv_problems = solvpool_solver.solve([job])
+        # solve the deps for the input solvable's rpm and look for
problems
+        solver = self.solvpool.Solver()
+        solv_problems = solver.solve([job])
 
         if len(solv_problems) == 0:
-            return Pass, []
-
+             return {'result': Pass,'details':[]}
         else:
             problem_strs = []
             for problem in solv_problems:
                 probrules = problem.findallproblemrules()
-
                 for rule in probrules:
                     ruleinfos = rule.allinfos()
                     for info in ruleinfos:
                         problem_strs.append(info.problemstr())
+            return {'result':Fail, 'details':problem_strs}
 
-            return Fail, problem_strs
 
     def check_rpm_deps(self, rpmfiles):
         """Check dependencies of the given rpms against supplied
repodata @@ -125,26 +125,33 @@ class Depcheck(object):
         :returns: dictionary of results {rpm: {'results': Pass/Fail,
'details': []}} """
 
-        self._prepare_libsolv()
+        log.logger.debug('adding all rpms to libsolv repos and
preparing solvpool') 
-        run_result = {}
 
+        # add all rpmfiles that we're checking to the libsolv repo and
save the
+        # solvable that is assigned to them.
+        # this makes libsolv aware of all the incoming rpms and allows
it to
+        # check for any dep breakage between the rpms
+        self._prepare_libsolv()
+        solvables = {}
         for rpmfile in rpmfiles:
-            log.logger.debug('running depcheck for %s' % rpmfile)
-            result_state, result_details = self._check_rpm(rpmfile)
+            rpmname = rpmfile.split('/')[-1]
+            solvable = self.solvrepo.add_rpm(rpmfile)
+            solvables[rpmname] = solvable
 
-            if rpmfile in run_result:
-                if result_state != run_result[rpmfile]['result']:
-                    #TODO maybe do something saner, ideas welcomed
-                    raise exc.DepcheckException("RPM %r has multiple
conflicting results" % rpmfile)
-                else:
-
run_result[rpmfile]['details'].append(result_details)
+        # now that the rpms have been added, prepare the solvpool
+        self.solvpool.addfileprovides()
+        self.solvpool.createwhatprovides()
 
-            else:
-                run_result[rpmfile] = {'result': result_state,
-                                   'details': result_details}
 
-        return run_result
+        # check each rpm using its solvable that was assigned while
adding to
+        # the repo
+        results = {}
+        for rpmname in solvables:
+            log.logger.debug('running depcheck for %s' % rpmname)
+            run_result = self._check_rpm(rpmname, solvables[rpmname])
+            results[rpmname] = run_result
 
+        return results
 
 
diff --git a/depcheck/squash_results.py b/depcheck/squash_results.py
index 7376f0d..30330ca 100644
--- a/depcheck/squash_results.py
+++ b/depcheck/squash_results.py
@@ -1,4 +1,3 @@
-import os
 import json
 
 from . import Pass, Fail
@@ -7,7 +6,6 @@ from . import log
 from libtaskotron.koji_utils import KojiClient
 from libtaskotron.bodhi_utils import BodhiUtils
 from libtaskotron import check
-from libtaskotron.rpm_utils import rpmformat
 
 def _squash_rpms_to_builds(run_result, koji_client = None):
     """
diff --git a/tests/depcheck_scenarios b/tests/depcheck_scenarios
index f1e5d57..3ad160a 160000
--- a/tests/depcheck_scenarios
+++ b/tests/depcheck_scenarios
@@ -1 +1 @@
-Subproject commit f1e5d57b0e68c53bd05c1b4d6eab3078706d269e
+Subproject commit 3ad160ad3145dc4595286291e5e43e2d91737ea9
diff --git a/tests/functest_depcheck_scenarios.py
b/tests/functest_depcheck_scenarios.py index dc69e07..3f5733d 100644
--- a/tests/functest_depcheck_scenarios.py
+++ b/tests/functest_depcheck_scenarios.py
@@ -36,9 +36,12 @@ def get_depcheck_scenarios():
         for dircontent in dircontents:
             if len(dircontent[2]) == 0:
                 continue
+            if dircontent[1] == 'known_broken':
+                continue
             for infile in dircontent[2]:
                 filename = '%s/%s' % (dircontent[0], infile)
-                yamlfiles.append((scenario_type == 'pass', filename))
+                if infile.split('.')[-1] in ['yml', 'yaml']:
+                    yamlfiles.append((scenario_type == 'pass',
filename)) return yamlfiles
 
 
diff --git a/tests/test_depcheck.py b/tests/test_depcheck.py
index d27d011..909b2ad 100644
--- a/tests/test_depcheck.py
+++ b/tests/test_depcheck.py
@@ -36,10 +36,10 @@ class TestDepcheck(object):
         monkeypatch.setattr(test_depcheck, '_prepare_libsolv',
Dingus()) 
         rpmfiles = ['rpm_1', 'rpm_2']
-        def fake_check_rpm(rpmfile):
+        def fake_check_rpm(rpmfile, solvable):
             results = {
-                    rpmfiles[0]: (Pass, []),
-                    rpmfiles[1]: (Fail, ['message']),
+                    rpmfiles[0]: {'result':Pass, 'details':[]},
+                    rpmfiles[1]: {'result':Fail,
'details':['message']}, }
             return results.get(rpmfile, (Pass, []))
 
@@ -54,25 +54,3 @@ class TestDepcheck(object):
         assert outcome['rpm_2']['result'] == Fail
         assert outcome['rpm_2']['details'] == ['message']
 
-    def test_check_rpm_deps_conflicting_results(self, monkeypatch):
-        """
-        Checks whether rpm_deps raises Exceptions when conflicting
-        results are provided for one RPM.
-        """
-
-        test_depcheck = depcheck.Depcheck(None, None, "fakedir")
-        monkeypatch.setattr(test_depcheck, '_prepare_libsolv',
Dingus()) -
-        self.tmp_result = Pass
-        def fake_check_rpm(rpmfile):
-            if self.tmp_result == Pass:
-                self.tmp_result = Fail
-            else:
-                self.tmp_result = Pass
-            return (self.tmp_result, [])
-
-        monkeypatch.setattr(test_depcheck, '_check_rpm',
fake_check_rpm) -
-        with pytest.raises(exc.DepcheckException):
-            outcome = test_depcheck.check_rpm_deps(['rpm1', 'rpm1'])
-

Attachment: pgpyo8JS61x7f.pgp
Description: OpenPGP digital signature

_______________________________________________
infrastructure mailing list
infrastructure@xxxxxxxxxxxxxxxxxxxxxxx
https://admin.fedoraproject.org/mailman/listinfo/infrastructure

[Index of Archives]     [Fedora Development]     [Fedora Users]     [Fedora Desktop]     [Fedora SELinux]     [Yosemite News]     [KDE Users]

  Powered by Linux