> -----Original Message----- > From: Brendan Higgins > Sent: Friday, August 7, 2020 7:17 PM > > From: Heidi Fahim <heidifahim@xxxxxxxxxx> > > Add a --json flag, which when specified when kunit_tool is run, > generates JSON formatted test results conforming to the KernelCI API > test_group spec[1]. The user can the new flag to specify a filename as Seems to be missing a word. "The user can ? the new flag" > the value to json in order to store the JSON results under linux/. > > Link[1]: https://api.kernelci.org/schema-test-group.html#post > Signed-off-by: Heidi Fahim <heidifahim@xxxxxxxxxx> > Signed-off-by: Brendan Higgins <brendanhiggins@xxxxxxxxxx> > --- > tools/testing/kunit/kunit.py | 35 +++++++++++--- > tools/testing/kunit/kunit_json.py | 63 ++++++++++++++++++++++++++ > tools/testing/kunit/kunit_tool_test.py | 33 ++++++++++++++ > 3 files changed, 125 insertions(+), 6 deletions(-) > create mode 100644 tools/testing/kunit/kunit_json.py > > diff --git a/tools/testing/kunit/kunit.py b/tools/testing/kunit/kunit.py > index 96344a11ff1f..485b7c63b967 100755 > --- a/tools/testing/kunit/kunit.py > +++ b/tools/testing/kunit/kunit.py > @@ -17,6 +17,7 @@ from collections import namedtuple > from enum import Enum, auto > > import kunit_config > +import kunit_json > import kunit_kernel > import kunit_parser > > @@ -30,9 +31,9 @@ KunitBuildRequest = namedtuple('KunitBuildRequest', > KunitExecRequest = namedtuple('KunitExecRequest', > ['timeout', 'build_dir', 'alltests']) > KunitParseRequest = namedtuple('KunitParseRequest', > - ['raw_output', 'input_data']) > + ['raw_output', 'input_data', 'build_dir', 'json']) > KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', > - 'build_dir', 'alltests', > + 'build_dir', 'alltests', 'json', > 'make_options']) > > KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] > @@ -113,12 +114,22 @@ def parse_tests(request: KunitParseRequest) -> KunitResult: > test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, > [], > 'Tests not Parsed.') > + > if request.raw_output: > kunit_parser.raw_output(request.input_data) > else: > test_result = kunit_parser.parse_run_tests(request.input_data) > parse_end = time.time() > > + if request.json: > + json_obj = kunit_json.get_json_result( > + test_result=test_result, > + def_config='kunit_defconfig', > + build_dir=request.build_dir, > + json_path=request.json) > + if request.json == 'stdout': > + print(json_obj) > + > if test_result.status != kunit_parser.TestStatus.SUCCESS: > return KunitResult(KunitStatus.TEST_FAILURE, test_result, > parse_end - parse_start) > @@ -151,7 +162,9 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree, > return exec_result > > parse_request = KunitParseRequest(request.raw_output, > - exec_result.result) > + exec_result.result, > + request.build_dir, > + request.json) > parse_result = parse_tests(parse_request) > > run_end = time.time() > @@ -195,7 +208,12 @@ def add_exec_opts(parser): > def add_parse_opts(parser): > parser.add_argument('--raw_output', help='don\'t format output from kernel', > action='store_true') > - > + parser.add_argument('--json', > + nargs='?', > + help='Stores test results in a JSON, and either ' > + 'prints to stdout or saves to file if a ' > + 'filename is specified', > + type=str, const='stdout', default=None) > > def main(argv, linux=None): > parser = argparse.ArgumentParser( > @@ -254,6 +272,7 @@ def main(argv, linux=None): > cli_args.jobs, > cli_args.build_dir, > cli_args.alltests, > + cli_args.json, > cli_args.make_options) > result = run_tests(linux, request) > if result.status != KunitStatus.SUCCESS: > @@ -308,7 +327,9 @@ def main(argv, linux=None): > cli_args.alltests) > exec_result = exec_tests(linux, exec_request) > parse_request = KunitParseRequest(cli_args.raw_output, > - exec_result.result) > + exec_result.result, > + cli_args.build_dir, > + cli_args.json) > result = parse_tests(parse_request) > kunit_parser.print_with_timestamp(( > 'Elapsed time: %.3fs\n') % ( > @@ -322,7 +343,9 @@ def main(argv, linux=None): > with open(cli_args.file, 'r') as f: > kunit_output = f.read().splitlines() > request = KunitParseRequest(cli_args.raw_output, > - kunit_output) > + kunit_output, > + cli_args.build_dir, > + cli_args.json) > result = parse_tests(request) > if result.status != KunitStatus.SUCCESS: > sys.exit(1) > diff --git a/tools/testing/kunit/kunit_json.py b/tools/testing/kunit/kunit_json.py > new file mode 100644 > index 000000000000..624b31b2dbd6 > --- /dev/null > +++ b/tools/testing/kunit/kunit_json.py > @@ -0,0 +1,63 @@ > +# SPDX-License-Identifier: GPL-2.0 > +# > +# Generates JSON from KUnit results according to > +# KernelCI spec: https://github.com/kernelci/kernelci-doc/wiki/Test-API > +# > +# Copyright (C) 2020, Google LLC. > +# Author: Heidi Fahim <heidifahim@xxxxxxxxxx> > + > +import json > +import os > + > +import kunit_parser > + > +from kunit_parser import TestStatus > + > +def get_json_result(test_result, def_config, build_dir, json_path): > + sub_groups = [] > + > + # Each test suite is mapped to a KernelCI sub_group > + for test_suite in test_result.suites: > + sub_group = { > + "name": test_suite.name, > + "arch": "UM", > + "defconfig": def_config, > + "build_environment": build_dir, > + "test_cases": [], > + "lab_name": None, > + "kernel": None, > + "job": None, > + "git_branch": "kselftest", > + } > + test_cases = [] > + # TODO: Add attachments attribute in test_case with detailed > + # failure message, see https://api.kernelci.org/schema-test-case.html#get > + for case in test_suite.cases: > + test_case = {"name": case.name, "status": "FAIL"} > + if case.status == TestStatus.SUCCESS: > + test_case["status"] = "PASS" > + elif case.status == TestStatus.TEST_CRASHED: > + test_case["status"] = "ERROR" > + test_cases.append(test_case) > + sub_group["test_cases"] = test_cases > + sub_groups.append(sub_group) > + test_group = { > + "name": "KUnit Test Group", > + "arch": "UM", > + "defconfig": def_config, > + "build_environment": build_dir, > + "sub_groups": sub_groups, > + "lab_name": None, > + "kernel": None, > + "job": None, > + "git_branch": "kselftest", > + } > + json_obj = json.dumps(test_group, indent=4) > + if json_path != 'stdout': > + with open(json_path, 'w') as result_path: > + result_path.write(json_obj) > + root = __file__.split('tools/testing/kunit/')[0] > + kunit_parser.print_with_timestamp( > + "Test results stored in %s" % > + os.path.join(root, result_path.name)) > + return json_obj > diff --git a/tools/testing/kunit/kunit_tool_test.py b/tools/testing/kunit/kunit_tool_test.py > index 287c74d821c3..99c3c5671ea4 100755 > --- a/tools/testing/kunit/kunit_tool_test.py > +++ b/tools/testing/kunit/kunit_tool_test.py > @@ -11,11 +11,13 @@ from unittest import mock > > import tempfile, shutil # Handling test_tmpdir > > +import json > import os > > import kunit_config > import kunit_parser > import kunit_kernel > +import kunit_json > import kunit > > test_tmpdir = '' > @@ -230,6 +232,37 @@ class KUnitParserTest(unittest.TestCase): > result = kunit_parser.parse_run_tests(file.readlines()) > self.assertEqual('kunit-resource-test', result.suites[0].name) > > +class KUnitJsonTest(unittest.TestCase): > + > + def _json_for(self, log_file): > + with(open(get_absolute_path(log_file))) as file: > + test_result = kunit_parser.parse_run_tests(file) > + json_obj = kunit_json.get_json_result( > + test_result=test_result, > + def_config='kunit_defconfig', > + build_dir=None, > + json_path='stdout') > + return json.loads(json_obj) > + > + def test_failed_test_json(self): > + result = self._json_for( > + 'test_data/test_is_test_passed-failure.log') > + self.assertEqual( > + {'name': 'example_simple_test', 'status': 'FAIL'}, > + result["sub_groups"][1]["test_cases"][0]) > + > + def test_crashed_test_json(self): > + result = self._json_for( > + 'test_data/test_is_test_passed-crash.log') > + self.assertEqual( > + {'name': 'example_simple_test', 'status': 'ERROR'}, > + result["sub_groups"][1]["test_cases"][0]) > + > + def test_no_tests_json(self): > + result = self._json_for( > + 'test_data/test_is_test_passed-no_tests_run.log') > + self.assertEqual(0, len(result['sub_groups'])) > + > class StrContains(str): > def __eq__(self, other): > return self in other > -- > 2.28.0.236.gb10cc79966-goog
