[PATCH v4 26/29] tests: Adde remote directory to tests

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Add tests/remote direcotry and add test
runner and configuration file.

config.py - handle devices/setup_params tables
run-tests.py - will run tests cases

You can add own configuration file, by default
this is cfg.py and put there devices and
setup_params definition in format you can
find in config.py file.
By default hwsim and localhost devices will
be used.

Print available devices/test_cases:
./run-tests.py

Check devices (ssh connection, interfaces):
./run-test.py -t devices

Run sanity tests (test_sanit_*):
./run-test.py -d <dut_name> -t sanity

Run all tests:
./run-tests.py -d <dut_name> -t all

Run test_A and test_B:
./run-tests.py -d <dut_name> -t "test_A, test_B"

Set reference device, and run sanity tests:
./run-tests.py -d <dut_name> -r <ref_name> -t sanity

./run-test.py don't start/terminate wpa_supplicant nor hostpad,
test wcases are resposible for that, while we don't know what
test case requirements.

As a parameters each test case get:
- devices - table of available devices
- setup_params
- dut_name - name of DUT should be tested
- ref_name - name of reference device should be used

Each tests should return result, append_text.

Signed-off-by: Janusz Dziedzic <janusz.dziedzic@xxxxxxxxx>
---
 tests/remote/config.py    |  81 +++++++++++++++++
 tests/remote/run-tests.py | 223 ++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 304 insertions(+)
 create mode 100644 tests/remote/config.py
 create mode 100755 tests/remote/run-tests.py

diff --git a/tests/remote/config.py b/tests/remote/config.py
new file mode 100644
index 0000000..9b620bc
--- /dev/null
+++ b/tests/remote/config.py
@@ -0,0 +1,81 @@
+#
+# Environtment configuration
+# Copyright (c) 2016, Tieto Corporation
+#
+# This software may be distributed under the terms of the BSD license.
+# See README for more details.
+
+#
+# Currently static definition, in the future this could be config file,
+# or even common database with host management.
+#
+
+import logging
+logger = logging.getLogger()
+
+#
+# You can put your settings in cfg.py file with setup_params, devices definitions
+# in the format as below. In other case HWSIM cfg will be used.
+#
+setup_params = {"setup_hw" : "./tests/setup_hw.sh",
+                "hostapd" : "./tests/hostapd",
+                "wpa_supplicant" : "./tests/wpa_supplicant",
+                "iperf" : "iperf",
+                "country" : "PL",
+                "log_dir" : "/tmp/",
+                "ipv4_test_net" : "192.168.12.0"}
+
+#
+#devices = [{"hostname": "192.168.254.58", "ifname" : "wlan0", "port": "9877", "name" : "t2-ath9k", "flags" : "AP_HT40 STA_HT40"},
+#           {"hostname": "192.168.254.58", "ifname" : "wlan1", "port": "9877", "name" : "t2-ath10k", "flags" : "AP_VHT80"},
+#           {"hostname": "192.168.254.58", "ifname" : "wlan3", "port": "9877", "name" : "t2-intel7260", "flags" : "STA_VHT80"},
+#           {"hostname": "192.168.254.50", "ifname" : "wlan0", "port": "9877", "name" : "t1-ath9k"},
+#           {"hostname": "192.168.254.50", "ifname" : "wlan1", "port": "9877", "name" : "t1-ath10k"}]
+
+#
+# HWSIM - ifaces available after modprobe mac80211_hwsim
+#
+devices = [{"hostname": "localhost", "ifname": "wlan0", "port": "9878", "name": "hwsim0", "flags": "AP_VHT80 STA_VHT80"},
+           {"hostname": "localhost", "ifname": "wlan1", "port": "9879", "name": "hwsim1", "flags": "AP_VHT80 STA_VHT80"},
+           {"hostname": "localhost", "ifname": "wlan2", "port": "9880", "name": "hwsim2", "flags": "AP_VHT80 STA_VHT80"},
+           {"hostname": "localhost", "ifname": "wlan3", "port": "9881", "name": "hwsim3", "flags": "AP_VHT80 STA_VHT80"}]
+
+
+def get_setup_params(filename="cfg.py"):
+    try:
+       mod = __import__(filename.split(".")[0])
+       return mod.setup_params
+    except:
+       logger.debug("__import__(" + filename + ") failed, using static settings")
+       pass
+    return setup_params
+
+def get_devices(filename="cfg.py"):
+    try:
+       mod = __import__(filename.split(".")[0])
+       return mod.devices
+    except:
+       logger.debug("__import__(" + filename + ") failed, using static settings")
+       pass
+    return devices
+
+def get_device(devices, name=None, flags=None):
+    if name is None and flags is None:
+        return devices[0]
+    for device in devices:
+        if device['name'] == name:
+            return device
+    for device in devices:
+        try:
+            device_flags = device['flags']
+            if device_flags.find(flags) != -1:
+                return device
+        except:
+            pass
+    return None
+
+def get_dut(duts, name=None):
+    return get_device(duts, name)
+
+def put_device(devices, name):
+    pass
diff --git a/tests/remote/run-tests.py b/tests/remote/run-tests.py
new file mode 100755
index 0000000..61e7f3f
--- /dev/null
+++ b/tests/remote/run-tests.py
@@ -0,0 +1,223 @@
+#!/usr/bin/python
+#
+# Remote test case executor
+# Copyright (c) 2016, Tieto Corporation
+#
+# This software may be distributed under the terms of the BSD license.
+# See README for more details.
+import os
+import re
+import sys
+import time
+import traceback
+import getopt
+from datetime import datetime
+
+import logging
+logger = logging.getLogger()
+
+scriptsdir = os.path.dirname(os.path.realpath(sys.modules[__name__].__file__))
+sys.path.append(os.path.join(scriptsdir, '..', '..', 'wpaspy'))
+sys.path.append(os.path.join(scriptsdir, '..', 'hwsim'))
+
+import wpaspy
+import config
+
+def usage():
+    print "USAGE: " + sys.argv[0] + " -d <dut_name> -t <all|sanity|tests_to_run> [-r <ref_name>] [-v]"
+    print "USAGE: " + sys.argv[0] + " -t <help|devices> [-v]"
+    print "USAGE: " + sys.argv[0]
+
+def show_devices(devices, setup_params):
+    print "Devices:"
+    for device in devices:
+        dev = config.get_device(devices, device['name'])
+        host = wpaspy.Host(host = dev['hostname'], ifname = dev['ifname'],
+                           port = dev['port'], name = dev['name'])
+        # simple check if authorized_keys works correctly
+        status, buf = host.execute("id")
+        if status != 0:
+            print "\t[" + host.name + "] - ssh communication:  FAILED"
+            config.put_device(devices, host.name)
+            continue
+        else:
+            print "\t[" + host.name + "] - ssh communication: OK"
+        # check setup_hw works correctly
+        try:
+            setup_hw = setup_params['setup_hw']
+            try:
+                restart_device = setup_params['restart_device']
+            except:
+                restart_device = "0"
+            host.execute(setup_hw + " -I " + dev['ifname'] + " -R " + restart_device)
+        except:
+            pass
+        # show uname
+        status, buf = host.execute("uname -s -n -r -m -o")
+        print "\t\t" + buf
+        # show ifconfig
+        status, buf = host.execute("ifconfig " + dev['ifname'])
+        if status != 0:
+            print "\t\t" + host.ifname + " failed\n"
+            config.put_device(devices, host.name)
+            continue
+        lines = buf.splitlines()
+        for line in lines:
+            print "\t\t" + line
+        config.put_device(devices, host.name)
+
+
+def main():
+    dut = None
+    ref = None
+    requested_tests = ["help"]
+    log_dir = "./logs/"
+    verbose = False
+
+    # get env configuration
+    setup_params = config.get_setup_params()
+    devices = config.get_devices()
+
+    # parse input parameters
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "d:r:t:l:v", ["dut=", "ref=", "tests=", "log_dir="])
+    except getopt.GetoptError as err:
+        print(err)
+        usage()
+        sys.exit(2)
+
+    for option, argument in opts:
+        if option == "-v":
+            verbose = True
+        elif option in ("-d", "--dut"):
+            dut = argument
+        elif option in ("-r", "--ref"):
+            ref = argument
+        elif option in ("-t", "--tests"):
+            requested_tests = re.split('; | |, ', argument)
+        elif option in ("-l", "--log_dir"):
+            log_dir = argument
+        else:
+            assert False, "unhandled option"
+
+    # put logs in log_dir
+    symlink = os.path.join(log_dir, "current");
+    if os.path.exists(symlink):
+        os.unlink(symlink)
+    log_dir = os.path.join(log_dir, time.strftime("%Y_%m_%d_%H_%M_%S"))
+    if not os.path.exists(log_dir):
+        os.makedirs(log_dir)
+    os.symlink(os.path.join("../", log_dir), symlink)
+
+    setup_params['local_log_dir'] = log_dir
+
+    # configure logger
+    logger.setLevel(logging.DEBUG)
+
+    stdout_handler = logging.StreamHandler()
+    stdout_handler.setLevel(logging.WARNING)
+    if verbose:
+        stdout_handler.setLevel(logging.DEBUG)
+    logger.addHandler(stdout_handler)
+
+    formatter = logging.Formatter('%(asctime)s - %(message)s')
+    file_name = os.path.join(log_dir, 'run-tests.log')
+    log_handler = logging.FileHandler(file_name)
+    log_handler.setLevel(logging.DEBUG)
+    log_handler.setFormatter(formatter)
+    logger.addHandler(log_handler)
+
+    # import available tests
+    tests = []
+    failed = []
+    test_modules = []
+    files = os.listdir(scriptsdir)
+    for t in files:
+        m = re.match(r'(test_.*)\.py$', t)
+        if m:
+            mod = __import__(m.group(1))
+            test_modules.append(mod.__name__.replace('test_', '', 1))
+            for key,val in mod.__dict__.iteritems():
+                if key.startswith("test_"):
+                    tests.append(val)
+    test_names = list(set([t.__name__.replace('test_', '', 1) for t in tests]))
+
+    # sort the list
+    test_names.sort()
+    tests.sort()
+
+    # print help
+    if requested_tests[0] == "help":
+        usage()
+        print "\nAvailable Devices:"
+        for device in devices:
+            print "\t", device['name']
+        print "\nAvailable tests:"
+        for test in test_names:
+            print "\t", test
+        return
+
+    if requested_tests[0] == "devices":
+        show_devices(devices, setup_params)
+        return
+
+    # setup test we should run
+    tests_to_run = []
+    if requested_tests[0] == "all":
+        tests_to_run = tests
+    elif requested_tests[0] == "sanity":
+        for test in tests:
+            if test.__name__.startswith("test_sanity_"):
+                tests_to_run.append(test)
+    else:
+        for test in requested_tests:
+            t = None
+            for tt in tests:
+                name = tt.__name__.replace('test_', '', 1)
+                if name == test:
+                    t = tt
+                    break
+            if not t:
+                logger.warning("test case: " + test + " NOT-FOUND")
+                continue
+            tests_to_run.append(t)
+
+
+    # now run test cases
+    logger.warning("DUT: " + str(dut))
+    if ref:
+        logger.warning("REF: " + str(ref))
+    else:
+        logger.warning("REF: <auto>")
+
+    test_no = 1
+    for test in tests_to_run:
+        try:
+            logger.warning("START - " + test.__doc__ + " (" + str(test_no) + "/" + str(len(tests_to_run)) + ")")
+            setup_params['tc_name'] = test.__name__.replace('test_', '', 1)
+            start = datetime.now()
+            res, append = test(devices, setup_params, ref, dut)
+            if res == 0:
+                end = datetime.now()
+                logger.warning("PASS (" + append + ") - " + str((end - start).total_seconds()) + "s")
+            else:
+                end = datetime.now()
+                logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
+                failed.append(test.__name__.replace('test_', '', 1))
+        except KeyboardInterrupt:
+            raise
+        except:
+            end = datetime.now()
+            logger.warning("FAILED - " + str((end - start).total_seconds()) + "s")
+            logger.info(traceback.format_exc())
+            failed.append(test.__name__.replace('test_', '', 1))
+        test_no = test_no + 1
+
+    if len(failed) > 0:
+        logger.warning("Failed test cases:")
+        for test in failed:
+            logger.warning("\t" + test)
+
+
+if __name__ == "__main__":
+        main()
-- 
1.9.1


_______________________________________________
Hostap mailing list
Hostap@xxxxxxxxxxxxxxxxxxx
http://lists.infradead.org/mailman/listinfo/hostap



[Index of Archives]     [Linux Wireless]     [Linux Kernel]     [ATH6KL]     [Linux Bluetooth]     [Linux Netdev]     [Kernel Newbies]     [IDE]     [Security]     [Git]     [Netfilter]     [Bugtraq]     [Yosemite News]     [MIPS Linux]     [ARM Linux]     [Linux Security]     [Linux RAID]     [Linux ATA RAID]     [Samba]     [Device Mapper]

  Powered by Linux