Add tests/remote direcotry and add test runner and configuration file. config.py - handle devices/setup_params tables run-tests.py - will run tests cases You can add own configuration file, by default this is cfg.py and put there devices and setup_params definition in format you can find in config.py file. By default hwsim and localhost devices will be used. Print available devices/test_cases: ./run-tests.py Check devices (ssh connection, interfaces): ./run-test.py -t devices Run sanity tests (test_sanit_*): ./run-test.py -d <dut_name> -t sanity Run all tests: ./run-tests.py -d <dut_name> -t all Run test_A and test_B: ./run-tests.py -d <dut_name> -t "test_A, test_B" Set reference device, and run sanity tests: ./run-tests.py -d <dut_name> -r <ref_name> -t sanity ./run-test.py don't start/terminate wpa_supplicant nor hostpad, test wcases are resposible for that, while we don't know what test case requirements. As a parameters each test case get: - devices - table of available devices - setup_params - dut_name - name of DUT should be tested - ref_name - name of reference device should be used Each tests should return result, append_text. Signed-off-by: Janusz Dziedzic <janusz.dziedzic@xxxxxxxxx> --- tests/remote/config.py | 81 +++++++++++++++++ tests/remote/run-tests.py | 223 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 tests/remote/config.py create mode 100755 tests/remote/run-tests.py diff --git a/tests/remote/config.py b/tests/remote/config.py new file mode 100644 index 0000000..9b620bc --- /dev/null +++ b/tests/remote/config.py @@ -0,0 +1,81 @@ +# +# Environtment configuration +# Copyright (c) 2016, Tieto Corporation +# +# This software may be distributed under the terms of the BSD license. +# See README for more details. + +# +# Currently static definition, in the future this could be config file, +# or even common database with host management. +# + +import logging +logger = logging.getLogger() + +# +# You can put your settings in cfg.py file with setup_params, devices definitions +# in the format as below. In other case HWSIM cfg will be used. +# +setup_params = {"setup_hw" : "./tests/setup_hw.sh", + "hostapd" : "./tests/hostapd", + "wpa_supplicant" : "./tests/wpa_supplicant", + "iperf" : "iperf", + "country" : "PL", + "log_dir" : "/tmp/", + "ipv4_test_net" : "192.168.12.0"} + +# +#devices = [{"hostname": "192.168.254.58", "ifname" : "wlan0", "port": "9877", "name" : "t2-ath9k", "flags" : "AP_HT40 STA_HT40"}, +# {"hostname": "192.168.254.58", "ifname" : "wlan1", "port": "9877", "name" : "t2-ath10k", "flags" : "AP_VHT80"}, +# {"hostname": "192.168.254.58", "ifname" : "wlan3", "port": "9877", "name" : "t2-intel7260", "flags" : "STA_VHT80"}, +# {"hostname": "192.168.254.50", "ifname" : "wlan0", "port": "9877", "name" : "t1-ath9k"}, +# {"hostname": "192.168.254.50", "ifname" : "wlan1", "port": "9877", "name" : "t1-ath10k"}] + +# +# HWSIM - ifaces available after modprobe mac80211_hwsim +# +devices = [{"hostname": "localhost", "ifname": "wlan0", "port": "9878", "name": "hwsim0", "flags": "AP_VHT80 STA_VHT80"}, + {"hostname": "localhost", "ifname": "wlan1", "port": "9879", "name": "hwsim1", "flags": "AP_VHT80 STA_VHT80"}, + {"hostname": "localhost", "ifname": "wlan2", "port": "9880", "name": "hwsim2", "flags": "AP_VHT80 STA_VHT80"}, + {"hostname": "localhost", "ifname": "wlan3", "port": "9881", "name": "hwsim3", "flags": "AP_VHT80 STA_VHT80"}] + + +def get_setup_params(filename="cfg.py"): + try: + mod = __import__(filename.split(".")[0]) + return mod.setup_params + except: + logger.debug("__import__(" + filename + ") failed, using static settings") + pass + return setup_params + +def get_devices(filename="cfg.py"): + try: + mod = __import__(filename.split(".")[0]) + return mod.devices + except: + logger.debug("__import__(" + filename + ") failed, using static settings") + pass + return devices + +def get_device(devices, name=None, flags=None): + if name is None and flags is None: + return devices[0] + for device in devices: + if device['name'] == name: + return device + for device in devices: + try: + device_flags = device['flags'] + if device_flags.find(flags) != -1: + return device + except: + pass + return None + +def get_dut(duts, name=None): + return get_device(duts, name) + +def put_device(devices, name): + pass diff --git a/tests/remote/run-tests.py b/tests/remote/run-tests.py new file mode 100755 index 0000000..61e7f3f --- /dev/null +++ b/tests/remote/run-tests.py @@ -0,0 +1,223 @@ +#!/usr/bin/python +# +# Remote test case executor +# Copyright (c) 2016, Tieto Corporation +# +# This software may be distributed under the terms of the BSD license. +# See README for more details. +import os +import re +import sys +import time +import traceback +import getopt +from datetime import datetime + +import logging +logger = logging.getLogger() + +scriptsdir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__)) +sys.path.append(os.path.join(scriptsdir, '..', '..', 'wpaspy')) +sys.path.append(os.path.join(scriptsdir, '..', 'hwsim')) + +import wpaspy +import config + +def usage(): + print "USAGE: " + sys.argv[0] + " -d <dut_name> -t <all|sanity|tests_to_run> [-r <ref_name>] [-v]" + print "USAGE: " + sys.argv[0] + " -t <help|devices> [-v]" + print "USAGE: " + sys.argv[0] + +def show_devices(devices, setup_params): + print "Devices:" + for device in devices: + dev = config.get_device(devices, device['name']) + host = wpaspy.Host(host = dev['hostname'], ifname = dev['ifname'], + port = dev['port'], name = dev['name']) + # simple check if authorized_keys works correctly + status, buf = host.execute("id") + if status != 0: + print "\t[" + host.name + "] - ssh communication: FAILED" + config.put_device(devices, host.name) + continue + else: + print "\t[" + host.name + "] - ssh communication: OK" + # check setup_hw works correctly + try: + setup_hw = setup_params['setup_hw'] + try: + restart_device = setup_params['restart_device'] + except: + restart_device = "0" + host.execute(setup_hw + " -I " + dev['ifname'] + " -R " + restart_device) + except: + pass + # show uname + status, buf = host.execute("uname -s -n -r -m -o") + print "\t\t" + buf + # show ifconfig + status, buf = host.execute("ifconfig " + dev['ifname']) + if status != 0: + print "\t\t" + host.ifname + " failed\n" + config.put_device(devices, host.name) + continue + lines = buf.splitlines() + for line in lines: + print "\t\t" + line + config.put_device(devices, host.name) + + +def main(): + dut = None + ref = None + requested_tests = ["help"] + log_dir = "./logs/" + verbose = False + + # get env configuration + setup_params = config.get_setup_params() + devices = config.get_devices() + + # parse input parameters + try: + opts, args = getopt.getopt(sys.argv[1:], "d:r:t:l:v", ["dut=", "ref=", "tests=", "log_dir="]) + except getopt.GetoptError as err: + print(err) + usage() + sys.exit(2) + + for option, argument in opts: + if option == "-v": + verbose = True + elif option in ("-d", "--dut"): + dut = argument + elif option in ("-r", "--ref"): + ref = argument + elif option in ("-t", "--tests"): + requested_tests = re.split('; | |, ', argument) + elif option in ("-l", "--log_dir"): + log_dir = argument + else: + assert False, "unhandled option" + + # put logs in log_dir + symlink = os.path.join(log_dir, "current"); + if os.path.exists(symlink): + os.unlink(symlink) + log_dir = os.path.join(log_dir, time.strftime("%Y_%m_%d_%H_%M_%S")) + if not os.path.exists(log_dir): + os.makedirs(log_dir) + os.symlink(os.path.join("../", log_dir), symlink) + + setup_params['local_log_dir'] = log_dir + + # configure logger + logger.setLevel(logging.DEBUG) + + stdout_handler = logging.StreamHandler() + stdout_handler.setLevel(logging.WARNING) + if verbose: + stdout_handler.setLevel(logging.DEBUG) + logger.addHandler(stdout_handler) + + formatter = logging.Formatter('%(asctime)s - %(message)s') + file_name = os.path.join(log_dir, 'run-tests.log') + log_handler = logging.FileHandler(file_name) + log_handler.setLevel(logging.DEBUG) + log_handler.setFormatter(formatter) + logger.addHandler(log_handler) + + # import available tests + tests = [] + failed = [] + test_modules = [] + files = os.listdir(scriptsdir) + for t in files: + m = re.match(r'(test_.*)\.py$', t) + if m: + mod = __import__(m.group(1)) + test_modules.append(mod.__name__.replace('test_', '', 1)) + for key,val in mod.__dict__.iteritems(): + if key.startswith("test_"): + tests.append(val) + test_names = list(set([t.__name__.replace('test_', '', 1) for t in tests])) + + # sort the list + test_names.sort() + tests.sort() + + # print help + if requested_tests[0] == "help": + usage() + print "\nAvailable Devices:" + for device in devices: + print "\t", device['name'] + print "\nAvailable tests:" + for test in test_names: + print "\t", test + return + + if requested_tests[0] == "devices": + show_devices(devices, setup_params) + return + + # setup test we should run + tests_to_run = [] + if requested_tests[0] == "all": + tests_to_run = tests + elif requested_tests[0] == "sanity": + for test in tests: + if test.__name__.startswith("test_sanity_"): + tests_to_run.append(test) + else: + for test in requested_tests: + t = None + for tt in tests: + name = tt.__name__.replace('test_', '', 1) + if name == test: + t = tt + break + if not t: + logger.warning("test case: " + test + " NOT-FOUND") + continue + tests_to_run.append(t) + + + # now run test cases + logger.warning("DUT: " + str(dut)) + if ref: + logger.warning("REF: " + str(ref)) + else: + logger.warning("REF: <auto>") + + test_no = 1 + for test in tests_to_run: + try: + logger.warning("START - " + test.__doc__ + " (" + str(test_no) + "/" + str(len(tests_to_run)) + ")") + setup_params['tc_name'] = test.__name__.replace('test_', '', 1) + start = datetime.now() + res, append = test(devices, setup_params, ref, dut) + if res == 0: + end = datetime.now() + logger.warning("PASS (" + append + ") - " + str((end - start).total_seconds()) + "s") + else: + end = datetime.now() + logger.warning("FAILED - " + str((end - start).total_seconds()) + "s") + failed.append(test.__name__.replace('test_', '', 1)) + except KeyboardInterrupt: + raise + except: + end = datetime.now() + logger.warning("FAILED - " + str((end - start).total_seconds()) + "s") + logger.info(traceback.format_exc()) + failed.append(test.__name__.replace('test_', '', 1)) + test_no = test_no + 1 + + if len(failed) > 0: + logger.warning("Failed test cases:") + for test in failed: + logger.warning("\t" + test) + + +if __name__ == "__main__": + main() -- 1.9.1 _______________________________________________ Hostap mailing list Hostap@xxxxxxxxxxxxxxxxxxx http://lists.infradead.org/mailman/listinfo/hostap