Let's embed a copy of the piglit test runner, so we don't this external
dependency, removing an excuse to not run a complete series of tests
Signed-off-by: Damien Lespiau <damien.lespiau@xxxxxxxxx>
---
.gitignore | 2 +
piglit/framework/__init__.py | 21 ++
piglit/framework/core.py | 696 +++++++++++++++++++++++++++++++++++++
piglit/framework/exectest.py | 304 ++++++++++++++++
piglit/framework/gleantest.py | 49 +++
piglit/framework/junit.py | 377 ++++++++++++++++++++
piglit/framework/log.py | 53 +++
piglit/framework/patterns.py | 90 +++++
piglit/framework/status.py | 226 ++++++++++++
piglit/framework/summary.py | 525 ++++++++++++++++++++++++++++
piglit/framework/threadpool.py | 67 ++++
piglit/framework/threads.py | 43 +++
piglit/piglit-framework-tests.py | 47 +++
piglit/piglit-merge-results.py | 53 +++
piglit/piglit-print-commands.py | 86 +++++
piglit/piglit-run.py | 184 ++++++++++
piglit/piglit-summary-html.py | 98 ++++++
piglit/piglit-summary-junit.py | 128 +++++++
piglit/piglit-summary.py | 80 +++++
piglit/templates/empty_status.mako | 27 ++
piglit/templates/index.css | 78 +++++
piglit/templates/index.mako | 81 +++++
piglit/templates/result.css | 37 ++
piglit/templates/test_result.mako | 66 ++++
piglit/templates/testrun_info.mako | 49 +++
25 files changed, 3467 insertions(+)
create mode 100644 piglit/framework/__init__.py
create mode 100644 piglit/framework/core.py
create mode 100644 piglit/framework/exectest.py
create mode 100644 piglit/framework/gleantest.py
create mode 100644 piglit/framework/junit.py
create mode 100644 piglit/framework/log.py
create mode 100644 piglit/framework/patterns.py
create mode 100644 piglit/framework/status.py
create mode 100644 piglit/framework/summary.py
create mode 100644 piglit/framework/threadpool.py
create mode 100644 piglit/framework/threads.py
create mode 100755 piglit/piglit-framework-tests.py
create mode 100755 piglit/piglit-merge-results.py
create mode 100755 piglit/piglit-print-commands.py
create mode 100755 piglit/piglit-run.py
create mode 100755 piglit/piglit-summary-html.py
create mode 100755 piglit/piglit-summary-junit.py
create mode 100755 piglit/piglit-summary.py
create mode 100644 piglit/templates/empty_status.mako
create mode 100644 piglit/templates/index.css
create mode 100644 piglit/templates/index.mako
create mode 100644 piglit/templates/result.css
create mode 100644 piglit/templates/test_result.mako
create mode 100644 piglit/templates/testrun_info.mako
diff --git a/.gitignore b/.gitignore
index f5b326e..c9a5302 100644
--- a/.gitignore
+++ b/.gitignore
@@ -78,6 +78,8 @@ core
#
*.swo
*.swp
+*.pyc
+/.makotmp
cscope.*
TAGS
build-aux/
diff --git a/piglit/framework/__init__.py b/piglit/framework/__init__.py
new file mode 100644
index 0000000..3cf6d82
--- /dev/null
+++ b/piglit/framework/__init__.py
@@ -0,0 +1,21 @@
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
diff --git a/piglit/framework/core.py b/piglit/framework/core.py
new file mode 100644
index 0000000..e7767c2
--- /dev/null
+++ b/piglit/framework/core.py
@@ -0,0 +1,696 @@
+
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# Piglit core
+
+import errno
+import os
+import platform
+import re
+import stat
+import subprocess
+import string
+import sys
+import time
+import traceback
+from log import log
+from cStringIO import StringIO
+from textwrap import dedent
+from threads import synchronized_self
+import threading
+import multiprocessing
+try:
+ import simplejson as json
+except ImportError:
+ import json
+
+from threadpool import ThreadPool
+
+import status
+
+__all__ = ['Environment',
+ 'checkDir',
+ 'loadTestProfile',
+ 'TestrunResult',
+ 'GroupResult',
+ 'TestResult',
+ 'TestProfile',
+ 'Group',
+ 'Test',
+ 'testBinDir']
+
+
+class JSONWriter:
+ '''
+ Writes to a JSON file stream
+
+ JSONWriter is threadsafe.
+
+ Example
+ -------
+
+ This call to ``json.dump``::
+ json.dump(
+ {
+ 'a': [1, 2, 3],
+ 'b': 4,
+ 'c': {
+ 'x': 100,
+ },
+ }
+ file,
+ indent=JSONWriter.INDENT)
+
+ is equivalent to::
+ w = JSONWriter(file)
+ w.open_dict()
+ w.write_dict_item('a', [1, 2, 3])
+ w.write_dict_item('b', 4)
+ w.write_dict_item('c', {'x': 100})
+ w.close_dict()
+
+ which is also equivalent to::
+ w = JSONWriter(file)
+ w.open_dict()
+ w.write_dict_item('a', [1, 2, 3])
+ w.write_dict_item('b', 4)
+
+ w.write_dict_key('c')
+ w.open_dict()
+ w.write_dict_item('x', 100)
+ w.close_dict()
+
+ w.close_dict()
+ '''
+
+ INDENT = 4
+
+ def __init__(self, file):
+ self.file = file
+ self.__indent_level = 0
+ self.__inhibit_next_indent = False
+ self.__encoder = json.JSONEncoder(indent=self.INDENT)
+
+ # self.__is_collection_empty
+ #
+ # A stack that indicates if the currect collection is empty
+ #
+ # When open_dict is called, True is pushed onto the
+ # stack. When the first element is written to the newly
+ # opened dict, the top of the stack is set to False.
+ # When the close_dict is called, the stack is popped.
+ #
+ # The top of the stack is element -1.
+ #
+ # XXX: How does one attach docstrings to member variables?
+ #
+ self.__is_collection_empty = []
+
+ @synchronized_self
+ def __write_indent(self):
+ if self.__inhibit_next_indent:
+ self.__inhibit_next_indent = False
+ return
+ else:
+ i = ' ' * self.__indent_level * self.INDENT
+ self.file.write(i)
+
+ @synchronized_self
+ def __write(self, obj):
+ lines = list(self.__encoder.encode(obj).split('\n'))
+ n = len(lines)
+ for i in range(n):
+ self.__write_indent()
+ self.file.write(lines[i])
+ if i != n - 1:
+ self.file.write('\n')
+
+ @synchronized_self
+ def open_dict(self):
+ self.__write_indent()
+ self.file.write('{')
+
+ self.__indent_level += 1
+ self.__is_collection_empty.append(True)
+
+ @synchronized_self
+ def close_dict(self, comma=True):
+ self.__indent_level -= 1
+ self.__is_collection_empty.pop()
+
+ self.file.write('\n')
+ self.__write_indent()
+ self.file.write('}')
+
+ @synchronized_self
+ def write_dict_item(self, key, value):
+ # Write key.
+ self.write_dict_key(key)
+
+ # Write value.
+ self.__write(value)
+
+ @synchronized_self
+ def write_dict_key(self, key):
+ # Write comma if this is not the initial item in the dict.
+ if self.__is_collection_empty[-1]:
+ self.__is_collection_empty[-1] = False
+ else:
+ self.file.write(',')
+
+ self.file.write('\n')
+ self.__write(key)
+ self.file.write(': ')
+
+ self.__inhibit_next_indent = True
+
+
+# Ensure the given directory exists
+def checkDir(dirname, failifexists):
+ exists = True
+ try:
+ os.stat(dirname)
+ except OSError as e:
+ if e.errno == errno.ENOENT or e.errno == errno.ENOTDIR:
+ exists = False
+
+ if exists and failifexists:
+ print >>sys.stderr, "%(dirname)s exists already.\nUse --overwrite if" \
+ "you want to overwrite it.\n" % locals()
+ exit(1)
+
+ try:
+ os.makedirs(dirname)
+ except OSError as e:
+ if e.errno != errno.EEXIST:
+ raise
+
+if 'PIGLIT_BUILD_DIR' in os.environ:
+ testBinDir = os.path.join(os.environ['PIGLIT_BUILD_DIR'], 'bin')
+else:
+ testBinDir = os.path.normpath(os.path.join(os.path.dirname(__file__),
+ '../bin'))
+
+if 'PIGLIT_SOURCE_DIR' not in os.environ:
+ p = os.path
+ os.environ['PIGLIT_SOURCE_DIR'] = p.abspath(p.join(p.dirname(__file__),
+ '..'))
+
+# In debug builds, Mesa will by default log GL API errors to stderr.
+# This is useful for application developers or driver developers
+# trying to debug applications that should execute correctly. But for
+# piglit we expect to generate errors regularly as part of testing,
+# and for exhaustive error-generation tests (particularly some in
+# khronos's conformance suite), it can end up ooming your system
+# trying to parse the strings.
+if 'MESA_DEBUG' not in os.environ:
+ os.environ['MESA_DEBUG'] = 'silent'
+
+class TestResult(dict):
+ def __init__(self, *args):
+ dict.__init__(self, *args)
+
+ # Replace the result with a status object
+ try:
+ self['result'] = status.status_lookup(self['result'])
+ except KeyError:
+ # If there isn't a result (like when used by piglit-run), go on
+ # normally
+ pass
+
+
+class GroupResult(dict):
+ def get_subgroup(self, path, create=True):
+ '''
+ Retrieve subgroup specified by path
+
+ For example, ``self.get_subgroup('a/b/c')`` will attempt to
+ return ``self['a']['b']['c']``. If any subgroup along ``path``
+ does not exist, then it will be created if ``create`` is true;
+ otherwise, ``None`` is returned.
+ '''
+ group = self
+ for subname in path.split('/'):
+ if subname not in group:
+ if create:
+ group[subname] = GroupResult()
+ else:
+ return None
+ group = group[subname]
+ assert(isinstance(group, GroupResult))
+ return group
+
+ @staticmethod
+ def make_tree(tests):
+ '''
+ Convert a flat dict of test results to a hierarchical tree
+
+ ``tests`` is a dict whose items have form ``(path, TestResult)``,
+ where path is a string with form ``group1/group2/.../test_name``.
+
+ Return a tree whose leaves are the values of ``tests`` and
+ whose nodes, which have type ``GroupResult``, reflect the
+ paths in ``tests``.
+ '''
+ root = GroupResult()
+
+ for (path, result) in tests.items():
+ group_path = os.path.dirname(path)
+ test_name = os.path.basename(path)
+
+ group = root.get_subgroup(group_path)
+ group[test_name] = TestResult(result)
+
+ return root
+
+
+class TestrunResult:
+ def __init__(self, resultfile=None):
+ self.serialized_keys = ['options',
+ 'name',
+ 'tests',
+ 'wglinfo',
+ 'glxinfo',
+ 'lspci',
+ 'time_elapsed']
+ self.name = None
+ self.glxinfo = None
+ self.lspci = None
+ self.time_elapsed = None
+ self.tests = {}
+
+ if resultfile:
+ # Attempt to open the json file normally, if it fails then attempt
+ # to repair it.
+ try:
+ raw_dict = json.load(resultfile)
+ except ValueError:
+ raw_dict = json.load(self.__repairFile(resultfile))
+
+ # Check that only expected keys were unserialized.
+ for key in raw_dict:
+ if key not in self.serialized_keys:
+ raise Exception('unexpected key in results file: ', str(key))
+
+ self.__dict__.update(raw_dict)
+
+ # Replace each raw dict in self.tests with a TestResult.
+ for (path, result) in self.tests.items():
+ self.tests[path] = TestResult(result)
+
+ def __repairFile(self, file):
+ '''
+ Reapair JSON file if necessary
+
+ If the JSON file is not closed properly, perhaps due a system
+ crash during a test run, then the JSON is repaired by
+ discarding the trailing, incomplete item and appending braces
+ to the file to close the JSON object.
+
+ The repair is performed on a string buffer, and the given file
+ is never written to. This allows the file to be safely read
+ during a test run.
+
+ :return: If no repair occured, then ``file`` is returned.
+ Otherwise, a new file object containing the repaired JSON
+ is returned.
+ '''
+
+ file.seek(0)
+ lines = file.readlines()
+
+ # JSON object was not closed properly.
+ #
+ # To repair the file, we execute these steps:
+ # 1. Find the closing brace of the last, properly written
+ # test result.
+ # 2. Discard all subsequent lines.
+ # 3. Remove the trailing comma of that test result.
+ # 4. Append enough closing braces to close the json object.
+ # 5. Return a file object containing the repaired JSON.
+
+ # Each non-terminal test result ends with this line:
+ safe_line = 2 * JSONWriter.INDENT * ' ' + '},\n'
+
+ # Search for the last occurence of safe_line.
+ safe_line_num = None
+ for i in range(-1, - len(lines), -1):
+ if lines[i] == safe_line:
+ safe_line_num = i
+ break
+
+ if safe_line_num is None:
+ raise Exception('failed to repair corrupt result file: ' +
+ file.name)
+
+ # Remove corrupt lines.
+ lines = lines[0:(safe_line_num + 1)]
+
+ # Remove trailing comma.
+ lines[-1] = 2 * JSONWriter.INDENT * ' ' + '}\n'
+
+ # Close json object.
+ lines.append(JSONWriter.INDENT * ' ' + '}\n')
+ lines.append('}')
+
+ # Return new file object containing the repaired JSON.
+ new_file = StringIO()
+ new_file.writelines(lines)
+ new_file.flush()
+ new_file.seek(0)
+ return new_file
+
+ def write(self, file):
+ # Serialize only the keys in serialized_keys.
+ keys = set(self.__dict__.keys()).intersection(self.serialized_keys)
+ raw_dict = dict([(k, self.__dict__[k]) for k in keys])
+ json.dump(raw_dict, file, indent=JSONWriter.INDENT)
+
+
+class Environment:
+ def __init__(self, concurrent=True, execute=True, include_filter=[],
+ exclude_filter=[], valgrind=False, dmesg=False):
+ self.concurrent = concurrent
+ self.execute = execute
+ self.filter = []
+ self.exclude_filter = []
+ self.exclude_tests = set()
+ self.valgrind = valgrind
+ self.dmesg = dmesg
+
+ """
+ The filter lists that are read in should be a list of string objects,
+ however, the filters need to be a list or regex object.
+
+ This code uses re.compile to rebuild the lists and set self.filter
+ """
+ for each in include_filter:
+ self.filter.append(re.compile(each))
+ for each in exclude_filter:
+ self.exclude_filter.append(re.compile(each))
+
+ def run(self, command):
+ try:
+ p = subprocess.Popen(command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ universal_newlines=True)
+ (stdout, stderr) = p.communicate()
+ except:
+ return "Failed to run " + command
+ return stderr+stdout
+
+ def collectData(self):
+ result = {}
+ system = platform.system()
+ if (system == 'Windows' or system.find("CYGWIN_NT") == 0):
+ result['wglinfo'] = self.run('wglinfo')
+ else:
+ result['glxinfo'] = self.run('glxinfo')
+ if system == 'Linux':
+ result['lspci'] = self.run('lspci')
+ return result
+
+
+class Test:
+ ignoreErrors = []
+
+ def __init__(self, runConcurrent=False):
+ '''
+ 'runConcurrent' controls whether this test will
+ execute it's work (i.e. __doRunWork) on the calling thread
+ (i.e. the main thread) or from the ConcurrentTestPool threads.
+ '''
+ self.runConcurrent = runConcurrent
+ self.skip_test = False
+
+ def run(self):
+ raise NotImplementedError
+
+ def execute(self, env, path, json_writer):
+ '''
+ Run the test.
+
+ :path:
+ Fully qualified test name as a string. For example,
+ ``spec/glsl-1.30/preprocessor/compiler/keywords/void.frag``.
+ '''
+ def status(msg):
+ log(msg=msg, channel=path)
+
+ # Run the test
+ if env.execute:
+ try:
+ status("running")
+ time_start = time.time()
+ result = self.run(env)
+ time_end = time.time()
+ if 'time' not in result:
+ result['time'] = time_end - time_start
+ if 'result' not in result:
+ result['result'] = 'fail'
+ if not isinstance(result, TestResult):
+ result = TestResult(result)
+ result['result'] = 'warn'
+ result['note'] = 'Result not returned as an instance ' \
+ 'of TestResult'
+ except:
+ result = TestResult()
+ result['result'] = 'fail'
+ result['exception'] = str(sys.exc_info()[0]) + \
+ str(sys.exc_info()[1])
+ result['traceback'] = \
+ "".join(traceback.format_tb(sys.exc_info()[2]))
+
+ status(result['result'])
+
+ json_writer.write_dict_item(path, result)
+ else:
+ status("dry-run")
+
+ # Returns True iff the given error message should be ignored
+ def isIgnored(self, error):
+ for pattern in Test.ignoreErrors:
+ if pattern.search(error):
+ return True
+
+ return False
+
+ # Default handling for stderr messages
+ def handleErr(self, results, err):
+ errors = filter(lambda s: len(s) > 0,
+ map(lambda s: s.strip(), err.split('\n')))
+
+ ignored = [s for s in errors if self.isIgnored(s)]
+ errors = [s for s in errors if s not in ignored]
+
+ if len(errors) > 0:
+ results['errors'] = errors
+
+ if results['result'] == 'pass':
+ results['result'] = 'warn'
+
+ if len(ignored) > 0:
+ results['errors_ignored'] = ignored
+
+
+class Group(dict):
+ pass
+
+
+class TestProfile:
+ def __init__(self):
+ self.tests = Group()
+ self.test_list = {}
+
+ def flatten_group_hierarchy(self):
+ '''
+ Convert Piglit's old hierarchical Group() structure into a flat
+ dictionary mapping from fully qualified test names to "Test" objects.
+
+ For example,
+ tests['spec']['glsl-1.30']['preprocessor']['compiler']['void.frag']
+ would become:
+ test_list['spec/glsl-1.30/preprocessor/compiler/void.frag']
+ '''
+
+ def f(prefix, group, test_dict):
+ for key in group:
+ fullkey = key if prefix == '' else os.path.join(prefix, key)
+ if isinstance(group[key], dict):
+ f(fullkey, group[key], test_dict)
+ else:
+ test_dict[fullkey] = group[key]
+ f('', self.tests, self.test_list)
+ # Clear out the old Group()
+ self.tests = Group()
+
+ def prepare_test_list(self, env):
+ self.flatten_group_hierarchy()
+
+ def matches_any_regexp(x, re_list):
+ return True in map(lambda r: r.search(x) is not None, re_list)
+
+ def test_matches(item):
+ path, test = item
+ return ((not env.filter or matches_any_regexp(path, env.filter))
+ and not path in env.exclude_tests and
+ not matches_any_regexp(path, env.exclude_filter))
+
+ # Filter out unwanted tests
+ self.test_list = dict(filter(test_matches, self.test_list.items()))
+
+ def run(self, env, json_writer):
+ '''
+ Schedule all tests in profile for execution.
+
+ See ``Test.schedule`` and ``Test.run``.
+ '''
+
+ self.prepare_test_list(env)
+
+ # If using concurrency, add all the concurrent tests to the pool and
+ # execute that pool
+ if env.concurrent:
+ pool = ThreadPool(multiprocessing.cpu_count())
+ for (path, test) in self.test_list.items():
+ if test.runConcurrent:
+ pool.add(test.execute, (env, path, json_writer))
+ pool.join()
+
+ # Run any remaining tests serially from a single thread pool after the
+ # concurrent tests have finished
+ pool = ThreadPool(1)
+ for (path, test) in self.test_list.items():
+ if not env.concurrent or not test.runConcurrent:
+ pool.add(test.execute, (env, path, json_writer))
+ pool.join()
+
+ def remove_test(self, test_path):
+ """Remove a fully qualified test from the profile.
+
+ ``test_path`` is a string with slash ('/') separated
+ components. It has no leading slash. For example::
+ test_path = 'spec/glsl-1.30/linker/do-stuff'
+ """
+
+ l = test_path.split('/')
+ group = self.tests[l[0]]
+ for group_name in l[1:-2]:
+ group = group[group_name]
+ del group[l[-1]]
+
+
+def loadTestProfile(filename):
+ ns = {'__file__': filename}
+ try:
+ execfile(filename, ns)
+ except:
+ traceback.print_exc()
+ raise Exception('Could not read tests profile')
+ return ns['profile']
+
+
+def load_results(filename):
+ """ Loader function for TestrunResult class
+
+ This function takes a single argument of a results file.
+
+ It makes quite a few assumptions, first it assumes that it has been passed
+ a folder, if that fails then it looks for a plain text json file called
+ "main"
+
+ """
+ filename = os.path.realpath(filename)
+
+ try:
+ with open(filename, 'r') as resultsfile:
+ testrun = TestrunResult(resultsfile)
+ except IOError:
+ with open(os.path.join(filename, "main"), 'r') as resultsfile:
+ testrun = TestrunResult(resultsfile)
+
+ assert(testrun.name is not None)
+ return testrun
+
+
+# Error messages to be ignored
+Test.ignoreErrors = map(re.compile,
+ ["couldn't open libtxc_dxtn.so",
+ "compression/decompression available",
+ "Mesa: .*build",
+ "Mesa: CPU.*",
+ "Mesa: .*cpu detected.",
+ "Mesa: Test.*",
+ "Mesa: Yes.*",
+ "libGL: XF86DRIGetClientDriverName.*",
+ "libGL: OpenDriver: trying.*",
+ "libGL: Warning in.*drirc*",
+ "ATTENTION.*value of option.*",
+ "drmOpen.*",
+ "Mesa: Not testing OS support.*",
+ "Mesa: User error:.*",
+ "Mesa: Initializing .* optimizations",
+ "debug_get_.*",
+ "util_cpu_caps.*",
+ "Mesa: 3Dnow! detected",
+ "r300:.*",
+ "radeon:.*",
+ "Warning:.*",
+ "0 errors, .*",
+ "Mesa.*",
+ "no rrb",
+ "; ModuleID.*",
+ "%.*",
+ ".*failed to translate tgsi opcode.*to SSE",
+ ".*falling back to interpreter",
+ "GLSL version is .*, but requested version .* is "
+ "required",
+ "kCGErrorIllegalArgument: CGSOrderWindowList",
+ "kCGErrorFailure: Set a breakpoint @ "
+ "CGErrorBreakpoint\(\) to catch errors as they are "
+ "logged.",
+ "stw_(init|cleanup).*",
+ "OpenGLInfo..*",
+ "AdapterInfo..*",
+ "frameThrottleRate.*",
+ ".*DeviceName.*",
+ "No memory leaks detected.",
+ "libGL: Can't open configuration file.*"])
+
+
+def parse_listfile(filename):
+ """
+ Parses a newline-seperated list in a text file and returns a python list
+ object. It will expand tildes on Unix-like system to the users home
+ directory.
+
+ ex file.txt:
+ ~/tests1
+ ~/tests2/main
+ /tmp/test3
+
+ returns:
+ ['/home/user/tests1', '/home/users/tests2/main', '/tmp/test3']
+ """
+ with open(filename, 'r') as file:
+ return [path.expanduser(i.rstrip('\n')) for i in file.readlines()]
diff --git a/piglit/framework/exectest.py b/piglit/framework/exectest.py
new file mode 100644
index 0000000..e239940
--- /dev/null
+++ b/piglit/framework/exectest.py
@@ -0,0 +1,304 @@
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import errno
+import os
+import subprocess
+import threading
+import shlex
+import types
+import re
+
+from core import Test, testBinDir, TestResult
+
+
+# Platform global variables
+if 'PIGLIT_PLATFORM' in os.environ:
+ PIGLIT_PLATFORM = os.environ['PIGLIT_PLATFORM']
+else:
+ PIGLIT_PLATFORM = ''
+
+
+def read_dmesg():
+ proc = subprocess.Popen(['dmesg', '-l', 'emerg,alert,crit,err,warn,notice'], stdout=subprocess.PIPE)
+ return proc.communicate()[0].rstrip('\n')
+
+def get_dmesg_diff(old, new):
+ # Note that dmesg is a ring buffer, i.e. lines at the beginning may
+ # be removed when new lines are added.
+
+ # Get the last dmesg timestamp from the old dmesg as string.
+ last = old.split('\n')[-1]
+ ts = last[:last.find(']')+1]
+ if ts == '':
+ return ''
+
+ # Find the last occurence of the timestamp.
+ pos = new.find(ts)
+ if pos == -1:
+ return new # dmesg was completely overwritten by new messages
+
+ while pos != -1:
+ start = pos
+ pos = new.find(ts, pos+len(ts))
+
+ # Find the next line and return the rest of the string.
+ nl = new.find('\n', start+len(ts))
+ return new[nl+1:] if nl != -1 else ''
+
+
+# ExecTest: A shared base class for tests that simply runs an executable.
+class ExecTest(Test):
+ def __init__(self, command):
+ Test.__init__(self)
+ self.command = command
+ self.split_command = os.path.split(self.command[0])[1]
+ self.env = {}
+ self.timeout = None
+
+ if isinstance(self.command, basestring):
+ self.command = shlex.split(str(self.command))
+
+ self.skip_test = self.check_for_skip_scenario(command)
+
+ def interpretResult(self, out, returncode, results, dmesg):
+ raise NotImplementedError
+ return out
+
+ def run(self, env):
+ """
+ Run a test. The return value will be a dictionary with keys
+ including 'result', 'info', 'returncode' and 'command'.
+ * For 'result', the value may be one of 'pass', 'fail', 'skip',
+ 'crash', or 'warn'.
+ * For 'info', the value will include stderr/out text.
+ * For 'returncode', the value will be the numeric exit code/value.
+ * For 'command', the value will be command line program and arguments.
+ """
+ fullenv = os.environ.copy()
+ for e in self.env:
+ fullenv[e] = str(self.env[e])
+
+ if self.command is not None:
+ command = self.command
+
+ if env.valgrind:
+ command[:0] = ['valgrind', '--quiet', '--error-exitcode=1',
+ '--tool=memcheck']
+
+ i = 0
+ dmesg_diff = ''
+ while True:
+ if self.skip_test:
+ out = "PIGLIT: {'result': 'skip'}\n"
+ err = ""
+ returncode = None
+ else:
+ if env.dmesg:
+ old_dmesg = read_dmesg()
+ (out, err, returncode, timeout) = \
+ self.get_command_result(command, fullenv)
+ if env.dmesg:
+ dmesg_diff = get_dmesg_diff(old_dmesg, read_dmesg())
+
+ # https://bugzilla.gnome.org/show_bug.cgi?id=680214 is
+ # affecting many developers. If we catch it
+ # happening, try just re-running the test.
+ if out.find("Got spurious window resize") >= 0:
+ i = i + 1
+ if i >= 5:
+ break
+ else:
+ break
+
+ # proc.communicate() returns 8-bit strings, but we need
+ # unicode strings. In Python 2.x, this is because we
+ # will eventually be serializing the strings as JSON,
+ # and the JSON library expects unicode. In Python 3.x,
+ # this is because all string operations require
+ # unicode. So translate the strings into unicode,
+ # assuming they are using UTF-8 encoding.
+ #
+ # If the subprocess output wasn't properly UTF-8
+ # encoded, we don't want to raise an exception, so
+ # translate the strings using 'replace' mode, which
+ # replaces erroneous charcters with the Unicode
+ # "replacement character" (a white question mark inside
+ # a black diamond).
+ out = out.decode('utf-8', 'replace')
+ err = err.decode('utf-8', 'replace')
+
+ results = TestResult()
+
+ if self.skip_test:
+ results['result'] = 'skip'
+ else:
+ results['result'] = 'fail'
+ out = self.interpretResult(out, returncode, results, dmesg_diff)
+
+ crash_codes = [
+ # Unix: terminated by a signal
+ -5, # SIGTRAP
+ -6, # SIGABRT
+ -8, # SIGFPE (Floating point exception)
+ -10, # SIGUSR1
+ -11, # SIGSEGV (Segmentation fault)
+ # Windows:
+ # EXCEPTION_ACCESS_VIOLATION (0xc0000005):
+ -1073741819,
+ # EXCEPTION_INT_DIVIDE_BY_ZERO (0xc0000094):
+ -1073741676
+ ]
+
+ if returncode in crash_codes:
+ results['result'] = 'crash'
+ elif returncode != 0:
+ results['note'] = 'Returncode was {0}'.format(returncode)
+
+ if timeout:
+ results['result'] = 'timeout'
+
+ if env.valgrind:
+ # If the underlying test failed, simply report
+ # 'skip' for this valgrind test.
+ if results['result'] != 'pass':
+ results['result'] = 'skip'
+ elif returncode == 0:
+ # Test passes and is valgrind clean.
+ results['result'] = 'pass'
+ else:
+ # Test passed but has valgrind errors.
+ results['result'] = 'fail'
+
+ env = ''
+ for key in self.env:
+ env = env + key + '="' + self.env[key] + '" '
+ if env:
+ results['environment'] = env
+
+ results['info'] = unicode("Returncode: {0}\n\nErrors:\n{1}\n\n"
+ "Output:\n{2}").format(returncode,
+ err, out)
+ results['returncode'] = returncode
+ results['command'] = ' '.join(self.command)
+ results['dmesg'] = dmesg_diff
+ results['timeout'] = timeout
+
+ self.handleErr(results, err)
+
+ else:
+ results = TestResult()
+ if 'result' not in results:
+ results['result'] = 'skip'
+
+ return results
+
+ def check_for_skip_scenario(self, command):
+ global PIGLIT_PLATFORM
+ if PIGLIT_PLATFORM == 'gbm':
+ if 'glean' == self.split_command:
+ return True
+ if self.split_command.startswith('glx-'):
+ return True
+ return False
+
+ def get_command_result(self, command, fullenv):
+ try:
+ timeout = False
+ proc = subprocess.Popen(command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=fullenv,
+ universal_newlines=True)
+ output = ['', '']
+
+ def thread_fn():
+ output[0], output[1] = proc.communicate()
+
+ thread = threading.Thread(target=thread_fn)
+ thread.start()
+
+ thread.join(self.timeout)
+
+ if thread.is_alive():
+ proc.terminate()
+ thread.join()
+ timeout = True
+
+ returncode = proc.returncode
+ out, err = output
+ except OSError as e:
+ # Different sets of tests get built under
+ # different build configurations. If
+ # a developer chooses to not build a test,
+ # Piglit should not report that test as having
+ # failed.
+ if e.errno == errno.ENOENT:
+ out = "PIGLIT: {'result': 'skip'}\n" \
+ + "Test executable not found.\n"
+ err = ""
+ returncode = None
+ else:
+ raise e
+ return out, err, returncode, timeout
+
+
+class PlainExecTest(ExecTest):
+ """
+ PlainExecTest: Run a "native" piglit test executable
+
+ Expect one line prefixed PIGLIT: in the output, which contains a result
+ dictionary. The plain output is appended to this dictionary
+ """
+ def __init__(self, command):
+ ExecTest.__init__(self, command)
+ # Prepend testBinDir to the path.
+ self.command[0] = os.path.join(testBinDir, self.command[0])
+
+ def interpretResult(self, out, returncode, results, dmesg):
+ outlines = out.split('\n')
+ outpiglit = map(lambda s: s[7:],
+ filter(lambda s: s.startswith('PIGLIT:'), outlines))
+
+ if dmesg != '':
+ outpiglit = map(lambda s: s.replace("'pass'", "'dmesg-warn'"), outpiglit)
+ outpiglit = map(lambda s: s.replace("'warn'", "'dmesg-warn'"), outpiglit)
+ outpiglit = map(lambda s: s.replace("'fail'", "'dmesg-fail'"), outpiglit)
+
+ if len(outpiglit) > 0:
+ try:
+ for piglit in outpiglit:
+ if piglit.startswith('subtest'):
+ if not 'subtest' in results:
+ results['subtest'] = {}
+ results['subtest'].update(eval(piglit[7:]))
+ else:
+ results.update(eval(piglit))
+ out = '\n'.join(filter(lambda s: not s.startswith('PIGLIT:'),
+ outlines))
+ except:
+ results['result'] = 'fail'
+ results['note'] = 'Failed to parse result string'
+
+ if 'result' not in results:
+ results['result'] = 'fail'
+ return out
diff --git a/piglit/framework/gleantest.py b/piglit/framework/gleantest.py
new file mode 100644
index 0000000..88432e0
--- /dev/null
+++ b/piglit/framework/gleantest.py
@@ -0,0 +1,49 @@
+#
+# Permission is hereby granted, free of charge, to any person
+
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import os
+import subprocess
+
+from core import checkDir, testBinDir, Test, TestResult
+from exectest import ExecTest
+
+glean_executable = os.path.join(testBinDir, "glean")
+
+# GleanTest: Execute a sub-test of Glean
+class GleanTest(ExecTest):
+ globalParams = []
+
+ def __init__(self, name):
+ ExecTest.__init__(self, [glean_executable,
+ "-o", "-v", "-v", "-v", "-t",
+ "+"+name] + GleanTest.globalParams)
+ self.name = name
+
+ def interpretResult(self, out, returncode, results, dmesg):
+ if "{'result': 'skip'}" in out:
+ results['result'] = 'skip'
+ elif out.find('FAIL') >= 0:
+ results['result'] = 'dmesg-fail' if dmesg != '' else 'fail'
+ else:
+ results['result'] = 'dmesg-warn' if dmesg != '' else 'pass'
+ return out
diff --git a/piglit/framework/junit.py b/piglit/framework/junit.py
new file mode 100644
index 0000000..7916731
--- /dev/null
+++ b/piglit/framework/junit.py
@@ -0,0 +1,377 @@
+###########################################################################
+#
+# Copyright 2010-2011 VMware, Inc.
+# All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sub license, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice (including the
+# next paragraph) shall be included in all copies or substantial portions
+# of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR
+# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+###########################################################################
+
+"""Testing framework that assists invoking external programs and outputing
+results in ANT's junit XML format, used by Jenkins-CI."""
+
+
+import locale
+import optparse
+import os.path
+import shutil
+import string
+import sys
+import time
+
+
+__all__ = [
+ 'Error',
+ 'Failure',
+ 'Main',
+ 'Report',
+ 'Test',
+ 'TestSuite',
+]
+
+
+class Failure(Exception):
+ pass
+
+
+class Error(Exception):
+ pass
+
+
+# Not all valid Unicode characters are valid XML.
+# See http://www.w3.org/TR/xml/#charsets
+_validXmlAscii = ''.join([((_c >= 0x20 and _c < 0x80) or _c in (0x9, 0xA, 0xD)) and chr(_c) or '?' for _c in range(256)])
+_validXmlUnicode = {}
+for _c in range(0x20):
+ if _c not in (0x9, 0xA, 0xD):
+ _validXmlUnicode[_c] = ord('?')
+del _c
+
+
+def escape(s):
+ '''Escape and encode a XML string.'''
+ if isinstance(s, unicode):
+ s = s.translate(_validXmlUnicode)
+ else:
+ #s = s.decode(locale.getpreferredencoding(), 'replace')
+ s = s.translate(_validXmlAscii)
+ s = s.decode('ascii', 'ignore')
+ s = s.replace('&', '&')
+ s = s.replace('<', '<')
+ s = s.replace('>', '>')
+ s = s.replace('"', '"')
+ s = s.replace("'", ''')
+ s = s.encode('UTF-8')
+ return s
+
+
+# same as string.printable, but without '\v\f'
+_printable = string.digits + string.letters + string.punctuation + ' \t\n\r'
+_printable = ''.join([chr(_c) in _printable and chr(_c) or '?' for _c in range(256)])
+del _c
+
+
+class Report:
+ """Write test results in ANT's junit XML format.
+
+ See also:
+ - https://github.com/jenkinsci/jenkins/tree/master/test/src/test/resources/hudson/tasks/junit
+ - http://www.junit.org/node/399
+ - http://wiki.apache.org/ant/Proposals/EnhancedTestReports
+ """
+
+ def __init__(self, filename, time = True):
+ self.path = os.path.dirname(os.path.abspath(filename))
+ if not os.path.exists(self.path):
+ os.makedirs(self.path)
+
+ self.stream = open(filename, 'wt')
+ self.testsuites = []
+ self.inside_testsuite = False
+ self.inside_testcase = False
+ self.time = time
+
+ def start(self):
+ self.stream.write('<?xml version="1.0" encoding="UTF-8" ?>\n')
+ self.stream.write('<testsuites>\n')
+
+ def stop(self):
+ if self.inside_testcase:
+ self.stream.write('</testcase>\n')
+ self.inside_testcase = False
+ if self.inside_testsuite:
+ self.stream.write('</testsuite>\n')
+ self.inside_testsuite = False
+ self.stream.write('</testsuites>\n')
+ self.stream.flush()
+ self.stream.close()
+
+ def escapeName(self, name):
+ '''Dots are special for junit, so escape them with underscores.'''
+ name = name.replace('.', '_')
+ return name
+
+ def startSuite(self, name):
+ self.testsuites.append(self.escapeName(name))
+
+ def stopSuite(self):
+ if self.inside_testsuite:
+ self.stream.write('</testsuite>\n')
+ self.inside_testsuite = False
+ self.testsuites.pop(-1)
+
+ def startCase(self, name):
+ assert not self.inside_testcase
+ self.inside_testcase = True
+
+ if not self.inside_testsuite:
+ self.stream.write('<testsuite name="%s">\n' % escape('.'.join(self.testsuites[:1])))
+ self.inside_testsuite = True
+
+ self.case_name = name
+ self.buffer = []
+ self.stdout = []
+ self.stderr = []
+ self.start_time = time.time()
+
+ def stopCase(self, duration = None):
+ assert self.inside_testcase
+ self.inside_testcase = False
+
+ if len(self.testsuites) == 1:
+ classname = self.testsuites[0] + '.' + self.testsuites[0]
+ else:
+ classname = '.'.join(self.testsuites)
+ name = self.case_name
+
+ self.stream.write('<testcase classname="%s" name="%s"' % (escape(classname), escape(name)))
+ if duration is None:
+ if self.time:
+ stop_time = time.time()
+ duration = stop_time - self.start_time
+ if duration is not None:
+ self.stream.write(' time="%f"' % duration)
+
+ if not self.buffer and not self.stdout and not self.stderr:
+ self.stream.write('/>\n')
+ else:
+ self.stream.write('>')
+
+ for entry in self.buffer:
+ self.stream.write(entry)
+ if self.stdout:
+ self.stream.write('<system-out>')
+ for text in self.stdout:
+ self.stream.write(escape(text))
+ self.stream.write('</system-out>')
+ if self.stderr:
+ self.stream.write('<system-err>')
+ for text in self.stderr:
+ self.stream.write(escape(text))
+ self.stream.write('</system-err>')
+
+ self.stream.write('</testcase>\n')
+
+ self.stream.flush()
+
+ def addStdout(self, text):
+ if isinstance(text, str):
+ text = text.translate(_printable)
+ self.stdout.append(text)
+
+ def addStderr(self, text):
+ if isinstance(text, str):
+ text = text.translate(_printable)
+ self.stderr.append(text)
+
+ def addSkipped(self):
+ self.buffer.append('<skipped/>\n')
+
+ def addError(self, message, stacktrace=""):
+ self.buffer.append('<error message="%s"' % escape(message))
+ if not stacktrace:
+ self.buffer.append('/>')
+ else:
+ self.buffer.append('>')
+ self.buffer.append(escape(stacktrace))
+ self.buffer.append('</error>')
+
+ def addFailure(self, message, stacktrace=""):
+ self.buffer.append('<failure message="%s"' % escape(message))
+ if not stacktrace:
+ self.buffer.append('/>')
+ else:
+ self.buffer.append('>')
+ self.buffer.append(escape(stacktrace))
+ self.buffer.append('</failure>')
+
+ def addMeasurement(self, name, value):
+ '''Embedded a measurement in the standard output.
+
+ https://wiki.jenkins-ci.org/display/JENKINS/Measurement+Plots+Plugin
+ '''
+
+ if value is not None:
+ message = '<measurement><name>%s</name><value>%f</value></measurement>\n' % (name, value)
+ self.addStdout(message)
+
+ def addAttachment(self, path):
+ '''Attach a file.
+
+ https://wiki.jenkins-ci.org/display/JENKINS/JUnit+Attachments+Plugin
+ '''
+
+ attach_dir = os.path.join(self.path, '.'.join(self.testsuites + [self.case_name]))
+ if not os.path.exists(attach_dir):
+ os.makedirs(attach_dir)
+ shutil.copy2(path, attach_dir)
+
+ def addWorkspaceURL(self, path):
+ import urlparse
+ try:
+ workspace_path = os.environ['WORKSPACE']
+ job_url = os.environ['JOB_URL']
+ except KeyError:
+ self.addStdout(path + '\n')
+ else:
+ rel_path = os.path.relpath(path, workspace_path)
+ workspace_url = urlparse.urljoin(job_url, 'ws/')
+ url = urlparse.urljoin(workspace_url, rel_path)
+ if os.path.isdir(path):
+ url += '/'
+ self.addStdout(url + '\n')
+
+
+class BaseTest:
+
+ def _visit(self, report):
+ raise NotImplementedError
+
+ def fail(self, *args):
+ raise Failure(*args)
+
+ def error(self, *args):
+ raise Error(*args)
+
+
+
+class TestSuite(BaseTest):
+
+ def __init__(self, name, tests=()):
+ self.name = name
+ self.tests = []
+ self.addTests(tests)
+
+ def addTest(self, test):
+ self.tests.append(test)
+
+ def addTests(self, tests):
+ for test in tests:
+ self.addTest(test)
+
+ def run(self, filename = None, report = None):
+ if report is None:
+ if filename is None:
+ filename = self.name + '.xml'
+ report = Report(filename)
+ report.start()
+ try:
+ self._visit(report)
+ finally:
+ report.stop()
+
+ def _visit(self, report):
+ report.startSuite(self.name)
+ try:
+ self.test(report)
+ finally:
+ report.stopSuite()
+
+ def test(self, report):
+ for test in self.tests:
+ test._visit(report)
+
+
+class Test(BaseTest):
+
+ def __init__(self, name):
+ self.name = name
+
+ def _visit(self, report):
+ report.startCase(self.name)
+ try:
+ try:
+ return self.test(report)
+ except Failure as ex:
+ report.addFailure(*ex.args)
+ except Error as ex:
+ report.addError(*ex.args)
+ except KeyboardInterrupt:
+ raise
+ except:
+ report.addError(str(sys.exc_value))
+ finally:
+ report.stopCase()
+
+ def test(self, report):
+ raise NotImplementedError
+
+
+class Main:
+
+ default_timeout = 5*60
+
+ def __init__(self, name):
+ self.name = name
+
+ def optparser(self):
+ optparser = optparse.OptionParser(usage="\n\t%prog [options] ...")
+ optparser.add_option(
+ '-n', '--dry-run',
+ action="store_true",
+ dest="dry_run", default=False,
+ help="perform a trial run without executing")
+ optparser.add_option(
+ '-t', '--timeout', metavar='SECONDS',
+ type="float", dest="timeout", default = self.default_timeout,
+ help="timeout in seconds [default: %default]")
+ #optparser.add_option(
+ # '-f', '--filter',
+ # action='append',
+ # type="choice", metevar='GLOB',
+ # dest="filters", default=[],
+ # help="filter")
+ return optparser
+
+ def create_suite(self):
+ raise NotImplementedError
+
+ def run_suite(self, suite):
+ filename = self.name + '.xml'
+ report = Report(filename)
+ suite.run()
+
+ def main(self):
+ optparser = self.optparser()
+ (self.options, self.args) = optparser.parse_args(sys.argv[1:])
+
+ suite = self.create_suite()
+ self.run_suite(suite)
diff --git a/piglit/framework/log.py b/piglit/framework/log.py
new file mode 100644
index 0000000..310c552
--- /dev/null
+++ b/piglit/framework/log.py
@@ -0,0 +1,53 @@
+#
+# Copyright (c) 2010 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+import logging
+
+from threads import synchronized_self
+from patterns import Singleton
+
+
+class Logger(Singleton):
+ @synchronized_self
+ def __logMessage(self, logfunc, message, **kwargs):
+ [logfunc(line, **kwargs) for line in message.split('\n')]
+
+ @synchronized_self
+ def getLogger(self, channel=None):
+ if 0 == len(logging.root.handlers):
+ logging.basicConfig(format="[%(asctime)s] :: %(message)+8s "
+ ":: %(name)s",
+ datefmt="%c",
+ level=logging.INFO)
+ if channel is None:
+ channel = "base"
+ logger = logging.getLogger(channel)
+ return logger
+
+ def log(self, type=logging.INFO, msg="", channel=None):
+ self.__logMessage(lambda m,
+ **kwargs: self.getLogger(channel).log(type,
+ m,
+ **kwargs), msg)
+
+log = Logger().log
diff --git a/piglit/framework/patterns.py b/piglit/framework/patterns.py
new file mode 100644
index 0000000..bcf4e7e
--- /dev/null
+++ b/piglit/framework/patterns.py
@@ -0,0 +1,90 @@
+#
+# Copyright (c) 2010 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+import threading
+
+
+class Singleton(object):
+ '''
+ Modeled after
+ http://www.python.org/download/releases/2.2.3/descrintro/*__new__
+
+ A thread-safe (mostly -- see NOTE) Singleton class pattern.
+
+ NOTE: deleting a singleton instance (i.e. Singleton::delInstance) does not
+ guarantee that nothing else is currently using it. To reduce this risk, a
+ program should not hold a reference to the instance. Rather, use the
+ create/construct syntax (see example below) to access the instance. Yet,
+ this still does not guarantee that this type of usage will result in a
+ desired effect in a multithreaded program.
+ You've been warned so use the singleton pattern wisely!
+
+ Example:
+
+ class MySingletonClass(Singleton):
+ def init(self):
+ print "in MySingletonClass::init()", self
+
+ def foo(self):
+ print "in MySingletonClass::foo()", self
+
+ MySingletonClass().foo()
+ MySingletonClass().foo()
+ MySingletonClass().foo()
+
+ ---> output will look something like this:
+ in MySingletonClass::init() <__main__.MySingletonClass object at 0x7ff5b322f3d0>
+ in MySingletonClass::foo() <__main__.MySingletonClass object at 0x7ff5b322f3d0>
+ in MySingletonClass::foo() <__main__.MySingletonClass object at 0x7ff5b322f3d0>
+ in MySingletonClass::foo() <__main__.MySingletonClass object at 0x7ff5b322f3d0>
+ '''
+
+ lock = threading.RLock()
+
+ def __new__(cls, *args, **kwargs):
+ try:
+ cls.lock.acquire()
+ it = cls.__dict__.get('__it__')
+ if it is not None:
+ return it
+ cls.__it__ = it = object.__new__(cls)
+ it.init(*args, **kwargs)
+ return it
+ finally:
+ cls.lock.release()
+
+ def init(self, *args, **kwargs):
+ '''
+ Derived classes should override this method to do its initializations
+ The derived class should not implement a '__init__' method.
+ '''
+ pass
+
+ @classmethod
+ def delInstance(cls):
+ cls.lock.acquire()
+ try:
+ if cls.__dict__.get('__it__') is not None:
+ del cls.__it__
+ finally:
+ cls.lock.release()
diff --git a/piglit/framework/status.py b/piglit/framework/status.py
new file mode 100644
index 0000000..3a9e2d3
--- /dev/null
+++ b/piglit/framework/status.py
@@ -0,0 +1,226 @@
+# Copyright (c) 2013 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+""" Status ordering from best to worst:
+
+pass
+dmesg-warn
+warn
+dmesg-fail
+fail
+crash
+timeout
+skip
+
+
+The following are regressions:
+
+pass|warn|dmesg-warn|fail|dmesg-fail|crash|timeout -> skip
+pass|warn|dmesg-warn|fail|dmesg-fail|crash -> timeout|skip
+pass|warn|dmesg-warn|fail|dmesg-fail -> crash|timeout|skip
+pass|warn|dmesg-warn|fail -> dmesg-fail|crash|timeout|skip
+pass|warn|dmesg-warn -> fail|dmesg-fail|crash|timeout|skip
+pass|warn -> dmesg-warn|fail|dmesg-fail|crash|timeout|skip
+pass -> warn|dmesg-warn|fail|dmesg-fail|crash|timeout|skip
+
+
+The following are fixes:
+
+skip -> pass|warn|dmesg-warn|fail|dmesg-fail|crash|timeout
+timeout|skip -> pass|warn|dmesg-warn|fail|dmesg-fail|crash
+crash|timeout|skip - >pass|warn|dmesg-warn|fail|dmesg-fail
+dmesg-fail|crash|timeout|skip -> pass|warn|dmesg-warn|fail
+fail|dmesg-fail|crash|timeout|skip -> pass|warn|dmesg-warn
+dmesg-warn|fail|dmesg-fail|crash|timeout|skip -> pass|warn
+warn|dmesg-warn|fail|dmesg-fail|crash|timeout|skip -> pass
+
+
+NotRun -> * and * -> NotRun is a change, but not a fix or a regression. This is
+because NotRun is not a status, but a representation of an unknown status.
+
+"""
+
+
+def status_lookup(status):
+ """ Provided a string return a status object instance
+
+ When provided a string that corresponds to a key in it's status_dict
+ variable, this function returns a status object instance. If the string
+ does not correspond to a key it will raise an exception
+
+ """
+ status_dict = {'skip': Skip,
+ 'pass': Pass,
+ 'warn': Warn,
+ 'fail': Fail,
+ 'crash': Crash,
+ 'dmesg-warn': DmesgWarn,
+ 'dmesg-fail': DmesgFail,
+ 'timeout' : Timeout,
+ 'notrun': NotRun}
+
+ try:
+ return status_dict[status]()
+ except KeyError:
+ # Raise a StatusException rather than a key error
+ raise StatusException
+
+
+class StatusException(LookupError):
+ """ Raise this exception when a string is passed to status_lookup that
+ doesn't exists
+
+ The primary reason to have a special exception is that otherwise
+ status_lookup returns a KeyError, but there are many cases where it is
+ desireable to except a KeyError and have an exception path. Using a custom
+ Error class here allows for more fine-grained control.
+
+ """
+ pass
+
+
+class Status(object):
+ """
+ A simple class for representing the output values of tests.
+
+ This is a base class, and should not be directly called. Instead a child
+ class should be created and called. This module provides 8 of them: Skip,
+ Pass, Warn, Fail, Crash, NotRun, DmesgWarn, and DmesgFail.
+ """
+
+ # Using __slots__ allows us to implement the flyweight pattern, limiting
+ # the memory consumed for creating tens of thousands of these objects.
+ __slots__ = ['name', 'value', 'fraction']
+
+ name = None
+ value = None
+ fraction = (0, 1)
+
+ def __init__(self):
+ raise NotImplementedError
+
+ def split(self, spliton):
+ return (self.name.split(spliton))
+
+ def __repr__(self):
+ return self.name
+
+ def __str__(self):
+ return str(self.name)
+
+ def __unicode__(self):
+ return unicode(self.name)
+
+ def __lt__(self, other):
+ return int(self) < int(other)
+
+ def __le__(self, other):
+ return int(self) <= int(other)
+
+ def __eq__(self, other):
+ return int(self) == int(other)
+
+ def __ne__(self, other):
+ return int(self) != int(other)
+
+ def __ge__(self, other):
+ return int(self) >= int(other)
+
+ def __gt__(self, other):
+ return int(self) > int(other)
+
+ def __int__(self):
+ return self.value
+
+
+class NotRun(Status):
+ name = 'Not Run'
+ value = 0
+ fraction = (0, 0)
+
+ def __init__(self):
+ pass
+
+
+class Pass(Status):
+ name = 'pass'
+ value = 10
+ fraction = (1, 1)
+
+ def __init__(self):
+ pass
+
+
+class DmesgWarn(Status):
+ name = 'dmesg-warn'
+ value = 20
+
+ def __init__(self):
+ pass
+
+
+class Warn(Status):
+ name = 'warn'
+ value = 25
+
+ def __init__(self):
+ pass
+
+
+class DmesgFail(Status):
+ name = 'dmesg-fail'
+ value = 30
+
+ def __init__(self):
+ pass
+
+
+class Fail(Status):
+ name = 'fail'
+ value = 35
+
+ def __init__(self):
+ pass
+
+
+class Crash(Status):
+ name = 'crash'
+ value = 40
+
+ def __init__(self):
+ pass
+
+
+class Timeout(Status):
+ name = 'timeout'
+ value = 50
+
+ def __init__(self):
+ pass
+
+
+class Skip(Status):
+ name = 'skip'
+ value = 60
+ fraction = (0, 0)
+
+ def __init__(self):
+ pass
diff --git a/piglit/framework/summary.py b/piglit/framework/summary.py
new file mode 100644
index 0000000..8fbe2a8
--- /dev/null
+++ b/piglit/framework/summary.py
@@ -0,0 +1,525 @@
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import os
+import os.path as path
+import itertools
+import shutil
+import collections
+import tempfile
+from mako.template import Template
+
+# a local variable status exists, prevent accidental overloading by renaming
+# the module
+import status as so
+import core
+
+
+__all__ = [
+ 'Summary',
+]
+
+
+class HTMLIndex(list):
+ """
+ Builds HTML output to be passed to the index mako template, which will be
+ rendered into HTML pages. It does this by parsing the lists provided by the
+ Summary object, and returns itself, an object with one accessor, a list of
+ html strings that will be printed by the mako template.
+ """
+
+ def __init__(self, summary, page):
+ """
+ Steps through the list of groups and tests from all of the results and
+ generates a list of dicts that are passed to mako and turned into HTML
+ """
+
+ def returnList(open, close):
+ """
+ As HTMLIndex iterates through the groups and tests it uses this
+ function to determine which groups to close (and thus reduce the
+ depth of the next write) and which ones to open (thus increasing
+ the depth)
+
+ To that end one of two things happens, the path to the previous
+ group (close) and the next group (open) are equal, in that event we
+ don't want to open and close, becasue that will result in a
+ sawtooth pattern of a group with one test followed by the same
+ group with one test, over and over. Instead we simply return two
+ empty lists, which will result in writing a long list of test
+ results. The second option is that the paths are different, and
+ the function determines any commonality between the paths, and
+ returns the differences as close (the groups which are completly
+ written) and open (the new groups to write).
+ """
+ common = []
+
+ # Open and close are lists, representing the group hierarchy, open
+ # being the groups that need are soon to be written, and close
+ # representing the groups that have finished writing.
+ if open == close:
+ return [], []
+ else:
+ for i, j in itertools.izip_longest(open, close):
+ if i != j:
+ for k in common:
+ open.remove(k)
+ close.remove(k)
+ return open, close
+ else:
+ common.append(i)
+
+ # set a starting depth of 1, 0 is used for 'all' so 1 is the
+ # next available group
+ depth = 1
+
+ # Current dir is a list representing the groups currently being
+ # written.
+ currentDir = []
+
+ # Add a new 'tab' for each result
+ self._newRow()
+ self.append({'type': 'other', 'text': '<td />'})
+ for each in summary.results:
+ self.append({'type': 'other',
+ 'text': '<td class="head"><b>%(name)s</b><br />'
+ '(<a href="%(href)s">info</a>)'
+ '</td>' % {'name': each.name,
+ 'href': path.join(each.name,
+ "index.html")}})
+ self._endRow()
+
+ # Add the toplevel 'all' group
+ self._newRow()
+ self._groupRow("head", 0, 'all')
+ for each in summary.results:
+ self._groupResult(summary.fractions[each.name]['all'],
+ summary.status[each.name]['all'])
+ self._endRow()
+
+ # Add the groups and tests to the out list
+ for key in sorted(page):
+
+ # Split the group names and test names, then determine
+ # which groups to close and which to open
+ openList = key.split('/')
+ test = openList.pop()
+ openList, closeList = returnList(openList, list(currentDir))
+
+ # Close any groups in the close list
+ # for each group closed, reduce the depth by one
+ for i in reversed(closeList):
+ currentDir.remove(i)
+ depth -= 1
+
+ # Open new groups
+ for localGroup in openList:
+ self._newRow()
+
+ # Add the left-most column: the name of the group
+ self._groupRow("head", depth, localGroup)
+
+ # Add the group that we just opened to the currentDir, which
+ # will then be used to add that group to the HTML list. If
+ # there is a KeyError (the group doesn't exist), use (0, 0)
+ # which will get skip. This sets the group coloring correctly
+ currentDir.append(localGroup)
+ for each in summary.results:
+ # Decide which fields need to be updated
+ self._groupResult(
+ summary.fractions[each.name][path.join(*currentDir)],
+ summary.status[each.name][path.join(*currentDir)])
+
+ # After each group increase the depth by one
+ depth += 1
+ self._endRow()
+
+ # Add the tests for the current group
+ self._newRow()
+
+ # Add the left-most column: the name of the test
+ self._testRow("group", depth, test)
+
+ # Add the result from each test result to the HTML summary If there
+ # is a KeyError (a result doesn't contain a particular test),
+ # return Not Run, with clas skip for highlighting
+ for each in summary.results:
+ # If the "group" at the top of the key heirachy contains
+ # 'subtest' then it is really not a group, link to that page
+ try:
+ if each.tests[path.dirname(key)]['subtest']:
+ href = path.dirname(key)
+ except KeyError:
+ href = key
+
+ try:
+ self._testResult(each.name, href,
+ summary.status[each.name][key])
+ except KeyError:
+ self.append({'type': 'other',
+ 'text': '<td class="skip">Not Run</td>'})
+ self._endRow()
+
+ def _newRow(self):
+ self.append({'type': 'newRow'})
+
+ def _endRow(self):
+ self.append({'type': 'endRow'})
+
+ def _groupRow(self, cssclass, depth, groupname):
+ """
+ Helper function for appending new groups to be written out
+ in HTML.
+
+ This particular function is used to write the left most
+ column of the summary. (the one with the indents)
+ """
+ self.append({'type': "groupRow",
+ 'class': cssclass,
+ 'indent': (1.75 * depth),
+ 'text': groupname})
+
+ def _groupResult(self, value, css):
+ """
+ Helper function for appending the results of groups to the
+ HTML summary file.
+ """
+ # "Not Run" is not a valid css class replace it with skip
+ if isinstance(css, so.NotRun):
+ css = 'skip'
+
+ self.append({'type': "groupResult",
+ 'class': css,
+ 'text': "%s/%s" % (value[0], value[1])})
+
+ def _testRow(self, cssclass, depth, groupname):
+ """
+ Helper function for appending new tests to be written out
+ in HTML.
+
+ This particular function is used to write the left most
+ column of the summary. (the one with the indents)
+ """
+ self.append({'type': "testRow",
+ 'class': cssclass,
+ 'indent': (1.75 * depth),
+ 'text': groupname})
+
+ def _testResult(self, group, href, text):
+ """
+ Helper function for writing the results of tests
+
+ This function writes the cells other than the left-most cell,
+ displaying pass/fail/crash/etc and formatting the cell to the
+ correct color.
+ """
+ # "Not Run" is not a valid class, if it apears set the class to skip
+ if isinstance(text, so.NotRun):
+ css = 'skip'
+ href = None
+ else:
+ css = text
+ href = path.join(group, href + ".html")
+
+ self.append({'type': 'testResult',
+ 'class': css,
+ 'href': href,
+ 'text': text})
+
+
+class Summary:
+ """
+ This Summary class creates an initial object containing lists of tests
+ including all, changes, problems, skips, regressions, and fixes. It then
+ uses methods to generate various kinds of output. The reference
+ implementation is HTML output through mako, aptly named generateHTML().
+ """
+ TEMP_DIR = path.join(tempfile.gettempdir(), "piglit/html-summary")
+ TEMPLATE_DIR = path.join(os.environ['PIGLIT_SOURCE_DIR'], 'templates')
+
+ def __init__(self, resultfiles):
+ """
+ Create an initial object with all of the result information rolled up
+ in an easy to process form.
+
+ The constructor of the summary class has an attribute for each HTML
+ summary page, which are fed into the index.mako file to produce HTML
+ files. resultfiles is a list of paths to JSON results generated by
+ piglit-run.
+ """
+
+ # Create a Result object for each piglit result and append it to the
+ # results list
+ self.results = [core.load_results(i) for i in resultfiles]
+
+ self.status = {}
+ self.fractions = {}
+ self.totals = {}
+ self.tests = {'all': set(), 'changes': set(), 'problems': set(),
+ 'skipped': set(), 'regressions': set(), 'fixes': set()}
+
+ def fgh(test, result):
+ """ Helper for updating the fractions and status lists """
+ fraction[test] = tuple(
+ [sum(i) for i in zip(fraction[test], result.fraction)])
+ if result != so.Skip() and status[test] < result:
+ status[test] = result
+
+ for results in self.results:
+ # Create a set of all of the tset names across all of the runs
+ self.tests['all'] = set(self.tests['all'] | set(results.tests))
+
+ # Create two dictionaries that have a default factory: they return
+ # a default value instead of a key error.
+ # This default key must be callable
+ self.fractions[results.name] = collections.defaultdict(lambda: (0, 0))
+ self.status[results.name] = collections.defaultdict(so.NotRun)
+
+ # short names
+ fraction = self.fractions[results.name]
+ status = self.status[results.name]
+
+ # store the results to be appeneded to results. Adding them in the
+ # loop will cause a RuntimeError
+ temp_results = {}
+
+ for key, value in results.tests.iteritems():
+ # Treat a test with subtests as if it is a group, assign the
+ # subtests' statuses and fractions down to the test, and then
+ # proceed like normal.
+ try:
+ for (subt, subv) in value['subtest'].iteritems():
+ subt = path.join(key, subt)
+ subv = so.status_lookup(subv)
+
+ # Add the subtest to the fractions and status lists
+ fraction[subt] = subv.fraction
+ status[subt] = subv
+ temp_results.update({subt: {'result': subv}})
+
+ self.tests['all'].add(subt)
+ while subt != '':
+ fgh(subt, subv)
+ subt = path.dirname(subt)
+ fgh('all', subv)
+
+ # remove the test from the 'all' list, this will cause to
+ # be treated as a group
+ self.tests['all'].discard(key)
+ except KeyError:
+ # Walk the test name as if it was a path, at each level update
+ # the tests passed over the total number of tests (fractions),
+ # and update the status of the current level if the status of
+ # the previous level was worse, but is not skip
+ while key != '':
+ fgh(key, value['result'])
+ key = path.dirname(key)
+
+ # when we hit the root update the 'all' group and stop
+ fgh('all', value['result'])
+
+ # Update the the results.tests dictionary with the subtests so that
+ # they are entered into the appropriate pages other than all.
+ # Updating it in the loop will raise a RuntimeError
+ results.tests.update({k:v for k,v in temp_results.iteritems()})
+
+ # Create the lists of statuses like problems, regressions, fixes,
+ # changes and skips
+ for test in self.tests['all']:
+ status = []
+ for each in self.results:
+ try:
+ status.append(each.tests[test]['result'])
+ except KeyError:
+ status.append(so.NotRun())
+
+ # Problems include: warn, dmesg-warn, fail, dmesg-fail, and crash.
+ # Skip does not go on this page, it has the 'skipped' page
+ if so.Skip() > max(status) > so.Pass():
+ self.tests['problems'].add(test)
+
+ # Find all tests with a status of skip
+ if so.Skip() in status:
+ self.tests['skipped'].add(test)
+
+ # find fixes, regressions, and changes
+ for i in xrange(len(status) - 1):
+ first = status[i]
+ last = status[i + 1]
+ if first < last and so.NotRun() not in (first, last):
+ self.tests['regressions'].add(test)
+ if first > last and so.NotRun() not in (first, last):
+ self.tests['fixes'].add(test)
+ # Changes cannot be added in the fixes and regressions passes
+ # becasue NotRun is a change, but not a regression or fix
+ if first != last:
+ self.tests['changes'].add(test)
+
+ def __find_totals(self):
+ """
+ Private: Find the total number of pass, fail, crash, skip, and warn in
+ the *last* set of results stored in self.results.
+ """
+ self.totals = {'pass': 0, 'fail': 0, 'crash': 0, 'skip': 0, 'warn': 0,
+ 'dmesg-warn': 0, 'dmesg-fail': 0}
+
+ for test in self.results[-1].tests.values():
+ self.totals[str(test['result'])] += 1
+
+ def generate_html(self, destination, exclude):
+ """
+ Produce HTML summaries.
+
+ Basically all this does is takes the information provided by the
+ constructor, and passes it to mako templates to generate HTML files.
+ The beauty of this approach is that mako is leveraged to do the
+ heavy lifting, this method just passes it a bunch of dicts and lists
+ of dicts, which mako turns into pretty HTML.
+ """
+
+ # Copy static files
+ shutil.copy(path.join(self.TEMPLATE_DIR, "index.css"),
+ path.join(destination, "index.css"))
+ shutil.copy(path.join(self.TEMPLATE_DIR, "result.css"),
+ path.join(destination, "result.css"))
+
+ # Create the mako object for creating the test/index.html file
+ testindex = Template(filename=path.join(self.TEMPLATE_DIR, "testrun_info.mako"),
+ output_encoding="utf-8",
+ module_directory=self.TEMP_DIR)
+
+ # Create the mako object for the individual result files
+ testfile = Template(filename=path.join(self.TEMPLATE_DIR, "test_result.mako"),
+ output_encoding="utf-8",
+ module_directory=self.TEMP_DIR)
+
+ result_css = path.join(destination, "result.css")
+ index = path.join(destination, "index.html")
+
+ # Iterate across the tests creating the various test specific files
+ for each in self.results:
+ os.mkdir(path.join(destination, each.name))
+
+ with open(path.join(destination, each.name, "index.html"), 'w') as out:
+ out.write(testindex.render(name=each.name,
+ time=each.time_elapsed,
+ options=each.options,
+ glxinfo=each.glxinfo,
+ lspci=each.lspci))
+
+ # Then build the individual test results
+ for key, value in each.tests.iteritems():
+ temp_path = path.join(destination, each.name, path.dirname(key))
+
+ if value['result'] not in exclude:
+ # os.makedirs is very annoying, it throws an OSError if
+ # the path requested already exists, so do this check to
+ # ensure that it doesn't
+ if not path.exists(temp_path):
+ os.makedirs(temp_path)
+
+ with open(path.join(destination, each.name, key + ".html"),
+ 'w') as out:
+ out.write(testfile.render(
+ testname=key,
+ status=value.get('result', 'None'),
+ # Returning a NoneType (instaed of 'None') prevents
+ # this field from being generated.setting the
+ # environment to run tests is ugly, and should
+ # disapear at somepoint
+ env=value.get('environment', None),
+ returncode=value.get('returncode', 'None'),
+ time=value.get('time', 'None'),
+ info=value.get('info', 'None'),
+ traceback=value.get('traceback', 'None'),
+ command=value.get('command', 'None'),
+ dmesg=value.get('dmesg', 'None'),
+ css=path.relpath(result_css, temp_path),
+ index=path.relpath(index, temp_path)))
+
+ # Finally build the root html files: index, regressions, etc
+ index = Template(filename=path.join(self.TEMPLATE_DIR, "index.mako"),
+ output_encoding="utf-8",
+ module_directory=self.TEMP_DIR)
+
+ empty_status = Template(filename=path.join(self.TEMPLATE_DIR, "empty_status.mako"),
+ output_encoding="utf-8",
+ module_directory=self.TEMP_DIR)
+
+ pages = ('changes', 'problems', 'skipped', 'fixes', 'regressions')
+
+ # Index.html is a bit of a special case since there is index, all, and
+ # alltests, where the other pages all use the same name. ie,
+ # changes.html, self.changes, and page=changes.
+ with open(path.join(destination, "index.html"), 'w') as out:
+ out.write(index.render(results=HTMLIndex(self, self.tests['all']),
+ page='all',
+ pages=pages,
+ colnum=len(self.results),
+ exclude=exclude))
+
+ # Generate the rest of the pages
+ for page in pages:
+ with open(path.join(destination, page + '.html'), 'w') as out:
+ # If there is information to display display it
+ if self.tests[page]:
+ out.write(index.render(results=HTMLIndex(self,
+ self.tests[page]),
+ pages=pages,
+ page=page,
+ colnum=len(self.results),
+ exclude=exclude))
+ # otherwise provide an empty page
+ else:
+ out.write(empty_status.render(page=page, pages=pages))
+
+ def generate_text(self, diff, summary):
+ """ Write summary information to the console """
+ self.__find_totals()
+
+ # Print the name of the test and the status from each test run
+ if not summary:
+ if diff:
+ for test in self.tests['changes']:
+ print "%(test)s: %(statuses)s" % {'test': test, 'statuses':
+ ' '.join([str(i.tests.get(test, {'result': so.Skip()})
+ ['result']) for i in self.results])}
+ else:
+ for test in self.tests['all']:
+ print "%(test)s: %(statuses)s" % {'test': test, 'statuses':
+ ' '.join([str(i.tests.get(test, {'result': so.Skip()})
+ ['result']) for i in self.results])}
+
+ # Print the summary
+ print "summary:"
+ print " pass: %d" % self.totals['pass']
+ print " fail: %d" % self.totals['fail']
+ print " crash: %d" % self.totals['crash']
+ print " skip: %d" % self.totals['skip']
+ print " warn: %d" % self.totals['warn']
+ print " dmesg-warn: %d" % self.totals['dmesg-warn']
+ print " dmesg-fail: %d" % self.totals['dmesg-fail']
+ if self.tests['changes']:
+ print " changes: %d" % len(self.tests['changes'])
+ print " fixes: %d" % len(self.tests['fixes'])
+ print "regressions: %d" % len(self.tests['regressions'])
+
+ print " total: %d" % sum(self.totals.values())
diff --git a/piglit/framework/threadpool.py b/piglit/framework/threadpool.py
new file mode 100644
index 0000000..5d1fc56
--- /dev/null
+++ b/piglit/framework/threadpool.py
@@ -0,0 +1,67 @@
+# Copyright (c) 2013 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# This code is based on the MIT licensed code by Emilio Monti found here:
+# http://code.activestate.com/recipes/577187-python-thread-pool/
+
+from Queue import Queue
+from threading import Thread
+
+
+class Worker(Thread):
+ """
+ Simple worker thread
+
+ This worker simply consumes tasks off of the queue until it is empty and
+ then waits for more tasks.
+ """
+
+ def __init__(self, queue):
+ Thread.__init__(self)
+ self.queue = queue
+ self.daemon = True
+ self.start()
+
+ def run(self):
+ """ This method is called in the constructor by self.start() """
+ while True:
+ func, args = self.queue.get()
+ func(*args) # XXX: Does this need to be try/except-ed?
+ self.queue.task_done()
+
+
+class ThreadPool(object):
+ """
+ A simple ThreadPool class that maintains a Queue object and a set of Worker
+ threads.
+ """
+
+ def __init__(self, thread_count):
+ self.queue = Queue(thread_count)
+ self.threads = [Worker(self.queue) for _ in xrange(thread_count)]
+
+ def add(self, func, args):
+ """ Add a function and it's arguments to the queue as a tuple """
+ self.queue.put((func, args))
+
+ def join(self):
+ """ Block until self.queue is empty """
+ self.queue.join()
diff --git a/piglit/framework/threads.py b/piglit/framework/threads.py
new file mode 100644
index 0000000..ec7dfcc
--- /dev/null
+++ b/piglit/framework/threads.py
@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2010 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+
+from weakref import WeakKeyDictionary
+from threading import RLock
+
+
+def synchronized_self(function):
+ '''
+ A decorator function for providing multithreaded, synchronized access
+ amongst one or more functions within a class instance.
+ '''
+ def wrapper(self, *args, **kwargs):
+ synchronized_self.locks.setdefault(self, RLock()).acquire()
+ try:
+ return function(self, *args, **kwargs)
+ finally:
+ synchronized_self.locks[self].release()
+ return wrapper
+
+
+# track the locks for each instance
+synchronized_self.locks = WeakKeyDictionary()
diff --git a/piglit/piglit-framework-tests.py b/piglit/piglit-framework-tests.py
new file mode 100755
index 0000000..796b1e1
--- /dev/null
+++ b/piglit/piglit-framework-tests.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2013 Intel Corporation
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import argparse
+import unittest
+
+import framework.tests.summary
+
+# Create a dictionary of all of tests. Do this before the parser so we can use
+# it as a list of optional arguments for the parser
+tests = {"summary": unittest.TestLoader().loadTestsFromModule(framework.tests.summary)}
+
+parser = argparse.ArgumentParser()
+parser.add_argument("tests",
+ action="append",
+ choices=tests.keys(),
+ help="Testing profiles for the framework")
+parser.add_argument("-v", "--verbose",
+ action="store",
+ choices=['0', '1', '2'],
+ default='1',
+ help="Set the level of verbosity to run tests at")
+args = parser.parse_args()
+
+# Run the tests
+map(unittest.TextTestRunner(verbosity=int(args.verbose)).run,
+ [tests[x] for x in args.tests])
diff --git a/piglit/piglit-merge-results.py b/piglit/piglit-merge-results.py
new file mode 100755
index 0000000..e78a5d0
--- /dev/null
+++ b/piglit/piglit-merge-results.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+
+import argparse
+import sys
+import os.path
+
+sys.path.append(os.path.dirname(os.path.realpath(sys.argv[0])))
+import framework.core as core
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("results",
+ metavar="<First Results File>",
+ nargs="*",
+ help="Space seperated list of results files")
+ args = parser.parse_args()
+
+ combined = core.load_results(args.results.pop(0))
+
+ for resultsDir in args.results:
+ results = core.load_results(resultsDir)
+
+ for testname, result in results.tests.items():
+ combined.tests[testname] = result
+
+ combined.write(sys.stdout)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/piglit/piglit-print-commands.py b/piglit/piglit-print-commands.py
new file mode 100755
index 0000000..c42ea6d
--- /dev/null
+++ b/piglit/piglit-print-commands.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+
+import argparse
+import sys
+import os
+import os.path as path
+import time
+import traceback
+
+sys.path.append(path.dirname(path.realpath(sys.argv[0])))
+import framework.core as core
+from framework.exectest import ExecTest
+from framework.gleantest import GleanTest
+
+
+def main():
+ parser = argparse.ArgumentParser(sys.argv)
+ parser.add_argument("-t", "--include-tests",
+ default = [],
+ action = "append",
+ metavar = "<regex>",
+ help = "Run only matching tests (can be used more than once)")
+ parser.add_argument("-x", "--exclude-tests",
+ default=[],
+ action="append",
+ metavar="<regex>",
+ help="Exclude matching tests (can be used more than "
+ "once)")
+ parser.add_argument("testProfile",
+ metavar="<Path to testfile>",
+ help="Path to results folder")
+ args = parser.parse_args()
+
+ # Set the environment, pass in the included and excluded tests
+ env = core.Environment(exclude_filter=args.exclude_tests,
+ include_filter=args.include_tests)
+
+ # Change to the piglit's path
+ piglit_dir = path.dirname(path.realpath(sys.argv[0]))
+ os.chdir(piglit_dir)
+
+ profile = core.loadTestProfile(args.testProfile)
+
+ def getCommand(test):
+ command = ''
+ if isinstance(test, GleanTest):
+ for var, val in test.env.items():
+ command += var + "='" + val + "' "
+
+ # Make the test command relative to the piglit_dir
+ testCommand = test.command[:]
+ testCommand[0] = os.path.relpath(testCommand[0], piglit_dir)
+
+ command += ' '.join(testCommand)
+ return command
+
+ profile.prepare_test_list(env)
+ for name, test in profile.test_list.items():
+ assert(isinstance(test, ExecTest))
+ print name, ':::', getCommand(test)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/piglit/piglit-run.py b/piglit/piglit-run.py
new file mode 100755
index 0000000..1d63cd4
--- /dev/null
+++ b/piglit/piglit-run.py
@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+
+import argparse
+import sys
+import os
+import os.path as path
+import time
+import traceback
+
+sys.path.append(path.dirname(path.realpath(sys.argv[0])))
+import framework.core as core
+from framework.threads import synchronized_self
+
+
+def main():
+ parser = argparse.ArgumentParser(sys.argv)
+ # Either require that a name for the test is passed or that
+ # resume is requested
+ excGroup1 = parser.add_mutually_exclusive_group()
+ excGroup1.add_argument("-n", "--name",
+ metavar="<test name>",
+ default=None,
+ help="Name of this test run")
+ excGroup1.add_argument("-r", "--resume",
+ action="store_true",
+ help="Resume an interupted test run")
+ # Setting the --dry-run flag is equivalent to env.execute=false
+ parser.add_argument("-d", "--dry-run",
+ action="store_false",
+ dest="execute",
+ help="Do not execute the tests")
+ parser.add_argument("-t", "--include-tests",
+ default=[],
+ action="append",
+ metavar="<regex>",
+ help="Run only matching tests "
+ "(can be used more than once)")
+ parser.add_argument("-x", "--exclude-tests",
+ default=[],
+ action="append",
+ metavar="<regex>",
+ help="Exclude matching tests "
+ "(can be used more than once)")
+ parser.add_argument("-1", "--no-concurrency",
+ action="store_false",
+ dest="concurrency",
+ help="Disable concurrent test runs")
+ parser.add_argument("-p", "--platform",
+ choices=["glx", "x11_egl", "wayland", "gbm"],
+ help="Name of windows system passed to waffle")
+ parser.add_argument("--valgrind",
+ action="store_true",
+ help="Run tests in valgrind's memcheck")
+ parser.add_argument("--dmesg",
+ action="store_true",
+ help="Capture a difference in dmesg before and "
+ "after each test")
+ parser.add_argument("testProfile",
+ metavar="<Path to test profile>",
+ help="Path to testfile to run")
+ parser.add_argument("resultsPath",
+ metavar="<Results Path>",
+ help="Path to results folder")
+ args = parser.parse_args()
+
+ # Set the platform to pass to waffle
+ if args.platform is not None:
+ os.environ['PIGLIT_PLATFORM'] = args.platform
+
+ # Always Convert Results Path from Relative path to Actual Path.
+ resultsDir = path.realpath(args.resultsPath)
+
+ # If resume is requested attempt to load the results file
+ # in the specified path
+ if args.resume is True:
+ # Load settings from the old results JSON
+ old_results = core.load_results(resultsDir)
+ profileFilename = old_results.options['profile']
+
+ # Changing the args to the old args allows us to set them
+ # all in one places down the way
+ args.exclude_tests = old_results.options['exclude_filter']
+ args.include_tests = old_results.options['filter']
+
+ # Otherwise parse additional settings from the command line
+ else:
+ profileFilename = args.testProfile
+
+ # Pass arguments into Environment
+ env = core.Environment(concurrent=args.concurrency,
+ exclude_filter=args.exclude_tests,
+ include_filter=args.include_tests,
+ execute=args.execute,
+ valgrind=args.valgrind,
+ dmesg=args.dmesg)
+
+ # Change working directory to the root of the piglit directory
+ piglit_dir = path.dirname(path.realpath(sys.argv[0]))
+ os.chdir(piglit_dir)
+
+ core.checkDir(resultsDir, False)
+
+ results = core.TestrunResult()
+
+ # Set results.name
+ if args.name is not None:
+ results.name = args.name
+ else:
+ results.name = path.basename(resultsDir)
+
+ # Begin json.
+ result_filepath = os.path.join(resultsDir, 'main')
+ result_file = open(result_filepath, 'w')
+ json_writer = core.JSONWriter(result_file)
+ json_writer.open_dict()
+
+ # Write out command line options for use in resuming.
+ json_writer.write_dict_key('options')
+ json_writer.open_dict()
+ json_writer.write_dict_item('profile', profileFilename)
+ json_writer.write_dict_item('filter', args.include_tests)
+ json_writer.write_dict_item('exclude_filter', args.exclude_tests)
+ json_writer.close_dict()
+
+ json_writer.write_dict_item('name', results.name)
+ for (key, value) in env.collectData().items():
+ json_writer.write_dict_item(key, value)
+
+ profile = core.loadTestProfile(profileFilename)
+
+ json_writer.write_dict_key('tests')
+ json_writer.open_dict()
+ # If resuming an interrupted test run, re-write all of the existing
+ # results since we clobbered the results file. Also, exclude them
+ # from being run again.
+ if args.resume is True:
+ for (key, value) in old_results.tests.items():
+ if os.path.sep != '/':
+ key = key.replace(os.path.sep, '/', -1)
+ json_writer.write_dict_item(key, value)
+ env.exclude_tests.add(key)
+
+ time_start = time.time()
+ profile.run(env, json_writer)
+ time_end = time.time()
+
+ json_writer.close_dict()
+
+ results.time_elapsed = time_end - time_start
+ json_writer.write_dict_item('time_elapsed', results.time_elapsed)
+
+ # End json.
+ json_writer.close_dict()
+ json_writer.file.close()
+
+ print
+ print 'Thank you for running Piglit!'
+ print 'Results have been written to ' + result_filepath
+
+
+if __name__ == "__main__":
+ main()
diff --git a/piglit/piglit-summary-html.py b/piglit/piglit-summary-html.py
new file mode 100755
index 0000000..a37b337
--- /dev/null
+++ b/piglit/piglit-summary-html.py
@@ -0,0 +1,98 @@
+#!/usr/bin/env python
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+import argparse
+import sys
+import shutil
+import os.path as path
+
+import framework.summary as summary
+import framework.status as status
+from framework.core import checkDir, parse_listfile
+
+sys.path.append(path.dirname(path.realpath(sys.argv[0])))
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-o", "--overwrite",
+ action="store_true",
+ help="Overwrite existing directories")
+ parser.add_argument("-l", "--list",
+ action="store",
+ help="Load a newline seperated list of results. These "
+ "results will be prepended to any Results "
+ "specified on the command line")
+ parser.add_argument("-e", "--exclude-details",
+ default=[],
+ action="append",
+ choices=['skip', 'pass', 'warn', 'crash' 'fail',
+ 'all'],
+ help="Optionally exclude the generation of HTML pages "
+ "for individual test pages with the status(es) "
+ "given as arguments. This speeds up HTML "
+ "generation, but reduces the info in the HTML "
+ "pages. May be used multiple times")
+ parser.add_argument("summaryDir",
+ metavar="<Summary Directory>",
+ help="Directory to put HTML files in")
+ parser.add_argument("resultsFiles",
+ metavar="<Results Files>",
+ nargs="*",
+ help="Results files to include in HTML")
+ args = parser.parse_args()
+
+ # If args.list and args.resultsFiles are empty, then raise an error
+ if not args.list and not args.resultsFiles:
+ raise parser.error("Missing required option -l or <resultsFiles>")
+
+ # Convert the exclude_details list to status objects, without this using
+ # the -e option will except
+ if args.exclude_details:
+ # If exclude-results has all, then change it to be all
+ if 'all' in args.exclude_details:
+ args.exclude_details = [status.Skip(), status.Pass(), status.Warn(),
+ status.Crash(), status.Fail()]
+ else:
+ args.exclude_details = [status.status_lookup(i) for i in
+ args.exclude_details]
+
+
+ # if overwrite is requested delete the output directory
+ if path.exists(args.summaryDir) and args.overwrite:
+ shutil.rmtree(args.summaryDir)
+
+ # If the requested directory doesn't exist, create it or throw an error
+ checkDir(args.summaryDir, not args.overwrite)
+
+ # Merge args.list and args.resultsFiles
+ if args.list:
+ args.resultsFiles.extend(parse_listfile(args.list))
+
+ # Create the HTML output
+ output = summary.Summary(args.resultsFiles)
+ output.generate_html(args.summaryDir, args.exclude_details)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/piglit/piglit-summary-junit.py b/piglit/piglit-summary-junit.py
new file mode 100755
index 0000000..a6f7aae
--- /dev/null
+++ b/piglit/piglit-summary-junit.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+#
+# Copyright 2010-2011 VMware, Inc.
+# All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sub license, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice (including the
+# next paragraph) shall be included in all copies or substantial portions
+# of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR
+# ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+import argparse
+import os
+import sys
+
+sys.path.append(os.path.dirname(os.path.realpath(sys.argv[0])))
+import framework.core as core
+import framework.status as status
+from framework import junit
+
+
+class Writer:
+
+ def __init__(self, filename):
+ self.report = junit.Report(filename)
+ self.path = []
+
+ def write(self, arg):
+ testrun = core.load_results(arg)
+
+ self.report.start()
+ self.report.startSuite('piglit')
+ try:
+ for (path, result) in testrun.tests.items():
+ self.write_test(testrun, path, result)
+ finally:
+ self.enter_path([])
+ self.report.stopSuite()
+ self.report.stop()
+
+ def write_test(self, testrun, test_path, result):
+ test_path = test_path.split('/')
+ test_name = test_path.pop()
+ self.enter_path(test_path)
+
+ self.report.startCase(test_name)
+ duration = None
+ try:
+ try:
+ self.report.addStdout(result['command'] + '\n')
+ except KeyError:
+ pass
+
+ try:
+ self.report.addStderr(result['info'])
+ except KeyError:
+ pass
+
+ success = result.get('result')
+ if success in (status.Pass(), status.Warn()):
+ pass
+ elif success == status.Skip():
+ self.report.addSkipped()
+ else:
+ self.report.addFailure(success.name)
+
+ try:
+ duration = float(result['time'])
+ except KeyError:
+ pass
+ finally:
+ self.report.stopCase(duration)
+
+ def enter_path(self, path):
+ ancestor = 0
+ try:
+ while self.path[ancestor] == path[ancestor]:
+ ancestor += 1
+ except IndexError:
+ pass
+
+ for dirname in self.path[ancestor:]:
+ self.report.stopSuite()
+
+ for dirname in path[ancestor:]:
+ self.report.startSuite(dirname)
+
+ self.path = path
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-o", "--output",
+ metavar = "<Output File>",
+ action = "store",
+ dest = "output",
+ default = "piglit.xml",
+ help = "Output filename")
+ parser.add_argument("testResults",
+ metavar = "<Input Files>",
+ help = "JSON results file to be converted")
+ args = parser.parse_args()
+
+
+ writer = Writer(args.output)
+ writer.write(args.testResults)
+
+
+if __name__ == "__main__":
+ main()
+
+
+# vim:set sw=4 ts=4 noet:
diff --git a/piglit/piglit-summary.py b/piglit/piglit-summary.py
new file mode 100755
index 0000000..c3b7ea6
--- /dev/null
+++ b/piglit/piglit-summary.py
@@ -0,0 +1,80 @@
+#!/usr/bin/env python
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or
+# sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# This permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+# PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHOR(S) BE
+# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
+# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+
+# Print a very simple summary of piglit results file(s).
+# When multiple result files are specified, compare the results
+# of each test run to look for differences/regressions.
+#
+# Brian Paul
+# April 2013
+
+
+import argparse
+import os.path
+import sys
+
+sys.path.append(os.path.dirname(os.path.realpath(sys.argv[0])))
+import framework.summary as summary
+from framework.core import parse_listfile
+
+def main():
+ parser = argparse.ArgumentParser()
+
+ # Set the -d and -s options as exclusive, since it's silly to call for diff
+ # and then call for only summary
+ excGroup1 = parser.add_mutually_exclusive_group()
+ excGroup1.add_argument("-d", "--diff",
+ action="store_true",
+ help="Only display the differences between multiple "
+ "result files")
+ excGroup1.add_argument("-s", "--summary",
+ action="store_true",
+ help="Only display the summary, not the individual "
+ "test results")
+ parser.add_argument("-l", "--list",
+ action="store",
+ help="Use test results from a list file")
+ parser.add_argument("results",
+ metavar="<Results Path(s)>",
+ nargs="+",
+ help="Space seperated paths to at least one results "
+ "file")
+ args = parser.parse_args()
+
+ # Throw an error if -d/--diff is called, but only one results file is
+ # provided
+ if args.diff and len(args.results) < 2:
+ parser.error('-d/--diff cannot be specified unless two or more '
+ 'results files are specified')
+
+ # make list of results
+ if args.list:
+ args.results.extend(parse_listfile(args.list))
+
+ # Generate the output
+ output = summary.Summary(args.results)
+ output.generate_text(args.diff, args.summary)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/piglit/templates/empty_status.mako b/piglit/templates/empty_status.mako
new file mode 100644
index 0000000..8ee6fba
--- /dev/null
+++ b/piglit/templates/empty_status.mako
@@ -0,0 +1,27 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+ <title>Result summary</title>
+ <link rel="stylesheet" href="status.css" type="text/css" />
+ </head>
+ <body>
+ <h1>Result summary</h1>
+ <p>Currently showing: ${page}</p>
+ <p>Show:
+ ## Index is a logical choice to put first, it will always be a link
+ ## and we don't want in preceeded by a |
+ <a href="index.html">index</a>
+ % for i in pages:
+ % if i == page:
+ | ${i}
+ % else:
+ | <a href="${i}.html">${i}</a>
+ % endif
+ % endfor
+ </p>
+ <h1>No ${page}</h1>
+ </body>
+</html>
diff --git a/piglit/templates/index.css b/piglit/templates/index.css
new file mode 100644
index 0000000..3389738
--- /dev/null
+++ b/piglit/templates/index.css
@@ -0,0 +1,78 @@
+
+table {
+ border: 0pt;
+ border-collapse: collapse;
+ padding-left: 1.75em;
+ padding-right: 1.75em;
+ min-width: 100%;
+ table-layout: fixed;
+}
+
+col:not(:first-child) {
+ width: 70pt;
+}
+
+tr {
+ padding: 4pt;
+}
+
+td {
+ padding: 4pt;
+}
+
+td:first-child {
+ padding: 0;
+}
+
+td:first-child > div {
+ padding: 4pt;
+}
+
+.title {
+ background-color: #c8c838;
+}
+
+.head {
+ background-color: #c8c838
+}
+
+td.skip, td.warn, td.fail, td.pass, td.trap, td.abort, td.crash, td.dmesg-warn, td.dmesg-fail, td.timeout {
+ text-align: right;
+}
+
+td.trap, td.abort, td.crash {
+ color: #ffffff;
+}
+
+td.trap a, td.abort a, td.crash a {
+ color: #ffffff;
+}
+
+tr:nth-child(odd) > td > div.group { background-color: #ffff95 }
+tr:nth-child(even) > td > div.group { background-color: #e1e183 }
+
+tr:nth-child(odd) td.pass { background-color: #20ff20; }
+tr:nth-child(even) td.pass { background-color: #15e015; }
+
+tr:nth-child(odd) td.skip { background-color: #b0b0b0; }
+tr:nth-child(even) td.skip { background-color: #a0a0a0; }
+
+tr:nth-child(odd) td.warn { background-color: #ff9020; }
+tr:nth-child(even) td.warn { background-color: #ef8010; }
+tr:nth-child(odd) td.dmesg-warn { background-color: #ff9020; }
+tr:nth-child(even) td.dmesg-warn { background-color: #ef8010; }
+
+tr:nth-child(odd) td.fail { background-color: #ff2020; }
+tr:nth-child(even) td.fail { background-color: #e00505; }
+tr:nth-child(odd) td.dmesg-fail { background-color: #ff2020; }
+tr:nth-child(even) td.dmesg-fail { background-color: #e00505; }
+
+tr:nth-child(odd) td.timeout { background-color: #83bdf6; }
+tr:nth-child(even) td.timeout { background-color: #4a9ef2; }
+
+tr:nth-child(odd) td.trap { background-color: #111111; }
+tr:nth-child(even) td.trap { background-color: #000000; }
+tr:nth-child(odd) td.abort { background-color: #111111; }
+tr:nth-child(even) td.abort { background-color: #000000; }
+tr:nth-child(odd) td.crash { background-color: #111111; }
+tr:nth-child(even) td.crash { background-color: #000000; }
diff --git a/piglit/templates/index.mako b/piglit/templates/index.mako
new file mode 100644
index 0000000..1ca46d3
--- /dev/null
+++ b/piglit/templates/index.mako
@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+ <title>Result summary</title>
+ <link rel="stylesheet" href="index.css" type="text/css" />
+ </head>
+ <body>
+ <h1>Result summary</h1>
+ <p>Currently showing: ${page}</p>
+ <p>Show:
+ % if page == 'all':
+ all
+ % else:
+ <a href="index.html">all</a>
+ % endif
+ % for i in pages:
+ % if i == page:
+ | ${i}
+ % else:
+ | <a href="${i}.html">${i}</a>
+ % endif
+ % endfor
+ </p>
+ <table>
+ <colgroup>
+ ## Name Column
+ <col />
+
+ ## Status columns
+ ## Create an additional column for each summary
+ % for _ in xrange(colnum):
+ <col />
+ % endfor
+ </colgroup>
+ % for line in results:
+ % if line['type'] == "newRow":
+ <tr>
+ % elif line['type'] == "endRow":
+ </tr>
+ % elif line['type'] == "groupRow":
+ <td>
+ <div class="${line['class']}" style="margin-left: ${line['indent']}em">
+ <b>${line['text']}</b>
+ </div>
+ </td>
+ % elif line['type'] == "testRow":
+ <td>
+ <div class="${line['class']}" style="margin-left: ${line['indent']}em">
+ ${line['text']}
+ </div>
+ </td>
+ % elif line['type'] == "groupResult":
+ <td class="${line['class']}">
+ <b>${line['text']}</b>
+ </td>
+ % elif line['type'] == "testResult":
+ <td class="${line['class']}">
+ ## If the result is in the excluded results page list from
+ ## argparse, just print the text, otherwise add the link
+ % if line['class'] not in exclude and line['href'] is not None:
+ <a href="${line['href']}">
+ ${line['text']}
+ </a>
+ % else:
+ ${line['text']}
+ % endif
+ </td>
+ % elif line['type'] == "subtestResult":
+ <td class="${line['class']}">
+ ${line['text']}
+ </td>
+ % elif line['type'] == "other":
+ ${line['text']}
+ % endif
+ % endfor
+ </table>
+ </body>
+</html>
diff --git a/piglit/templates/result.css b/piglit/templates/result.css
new file mode 100644
index 0000000..19bfedc
--- /dev/null
+++ b/piglit/templates/result.css
@@ -0,0 +1,37 @@
+
+td {
+ padding: 4pt;
+}
+
+th {
+ padding: 4pt;
+}
+
+table {
+ border: 0pt;
+ border-collapse: collapse;
+}
+
+th {
+ background-color: #c8c838
+}
+
+/* Second column (details) */
+tr:nth-child(even) > td {
+ background-color: #ffff95
+}
+
+tr:nth-child(odd) > td {
+ background-color: #e1e183
+}
+
+/* First column (labels) */
+tr:nth-child(even) > td:first-child {
+ vertical-align: top;
+ background-color: #ffff85;
+}
+
+tr:nth-child(odd) > td:first-child {
+ vertical-align: top;
+ background-color: #d1d173;
+}
diff --git a/piglit/templates/test_result.mako b/piglit/templates/test_result.mako
new file mode 100644
index 0000000..490c009
--- /dev/null
+++ b/piglit/templates/test_result.mako
@@ -0,0 +1,66 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//END"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+ <title>${testname} - Details</title>
+ <link rel="stylesheet" href="${css}" type="text/css" />
+ </head>
+ <body>
+ <h1>Results for ${testname}</h1>
+ <h2>Overview</h2>
+ <div>
+ <p><b>Result:</b> ${status}</p>
+ </div>
+ <p><a href="${index}">Back to summary</a></p>
+ <h2>Details</h2>
+ <table>
+ <tr>
+ <th>Detail</th>
+ <th>Value</th>
+ </tr>
+ <tr>
+ <td>Returncode</td>
+ <td>${returncode}</td>
+ </tr>
+ <tr>
+ <td>Time</td>
+ <td>${time}</b>
+ </tr>
+ <tr>
+ <td>Info</td>
+ <td>
+ <pre>${info | h}</pre>
+ </td>
+ </tr>
+ % if env:
+ <tr>
+ <td>Environment</td>
+ <td>
+ <pre>${env | h}</pre>
+ </td>
+ </tr>
+ % endif
+ <tr>
+ <td>Command</td>
+ <td>
+ </pre>${command}</pre>
+ </td>
+ </tr>
+ <tr>
+ <td>Traceback</td>
+ <td>
+ <pre>${traceback | h}</pre>
+ </td>
+ </tr>
+ <tr>
+ <td>dmesg</td>
+ <td>
+ <pre>${dmesg | h}</pre>
+ </td>
+ </tr>
+ </table>
+ <p><a href="${index}">Back to summary</a></p>
+ </body>
+</html>
diff --git a/piglit/templates/testrun_info.mako b/piglit/templates/testrun_info.mako
new file mode 100644
index 0000000..e6e00b3
--- /dev/null
+++ b/piglit/templates/testrun_info.mako
@@ -0,0 +1,49 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//END"
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";>
+<html xmlns="http://www.w3.org/1999/xhtml";>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+ <title>${name} - System info</title>
+ <link rel="stylesheet" href="../result.css" type="text/css" />
+ </head>
+ <body>
+ <h1>System info for ${name}</h1>
+ <p>
+ <a href="../index.html">Back to summary</a>
+ </p>
+ <table>
+ <tr>
+ <th>Detail</th>
+ <th>Value</th>
+ </tr>
+ <tr>
+ <td>time_elapsed</td>
+ <td>${time}</td>
+ </tr>
+ <tr>
+ <td>name</td>
+ <td>${name}</td>
+ </tr>
+ <tr>
+ <td>options</td>
+ <td>${options}</td>
+ </tr>
+ <tr>
+ <td>lspci</td>
+ <td>
+ <pre>${lspci}</pre>
+ </td>
+ </tr>
+ <tr>
+ <td>glxinfo</td>
+ <td>
+ <pre>${glxinfo}</pre>
+ </td>
+ </tr>
+ </table>
+ <p>
+ <a href="../index.html">Back to summary</a>
+ </p>
+ </body>
+</html>
--
1.8.3.1
_______________________________________________
Intel-gfx mailing list
Intel-gfx@xxxxxxxxxxxxxxxxxxxxx
http://lists.freedesktop.org/mailman/listinfo/intel-gfx
