[KVM-AUTOTEST PATCH 21/28] KVM test: refactor whql_submission_15.cs and whql_submission.py

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



- deps/whql_submission_15.cs:
    * Handle multiple machines instead of one.
    * Split the code into smaller functions.
    * Get job parameters from the user: each parameter is a regex that should
      match a device name in a machine, and is set using GetParameterByName().
      Note that job parameters differ from DeviceData objects and descriptors.
    * Get machine dimensions from the user and set them for each machine.
    * Set the WDKSubmissionId dimension for all machines (it seems to be
      required by some or all submissions).
    * Instead of AddDeviceJob() use submission.ProcessJobs() and set the .Device
      attribute of the ScheduleItem objects returned.  This is required for
      NDISTest to work.
    * Don't set status to 'Unsafe' before resetting machines (unnecessary).
    * If the machine pool already exists, delete it and create a new one.

- tests/whql_submission.py:
    * Use all vms instead of just main_vm.
    * Pass job parameters for each machine to the automation program.
    * Pass machine dimensions for each machine.
    * Write result summary to a file named 'summary' in addition to the regular
      logs.
    * Sort results in the result summary by failures and total test count in
      descending order.
    * Before starting the test, delete the machines from the data store, reboot
      them and wait for them to reappear in the data store.
    * Don't restart any services on the client machines (not necessary).

- Set 'restart_vm' to 'yes' in tests_base.cfg.sample to force a restart of the
  VMs before each test.

Signed-off-by: Michael Goldish <mgoldish@xxxxxxxxxx>
---
 client/tests/kvm/deps/whql_submission_15.cs  |  662 +++++++++++++++-----------
 client/tests/kvm/deps/whql_submission_15.exe |  Bin 10240 -> 12288 bytes
 client/tests/kvm/tests/whql_submission.py    |  182 +++++---
 client/tests/kvm/tests_base.cfg.sample       |    5 +-
 4 files changed, 497 insertions(+), 352 deletions(-)

diff --git a/client/tests/kvm/deps/whql_submission_15.cs b/client/tests/kvm/deps/whql_submission_15.cs
index 8fa6856..0928548 100644
--- a/client/tests/kvm/deps/whql_submission_15.cs
+++ b/client/tests/kvm/deps/whql_submission_15.cs
@@ -1,289 +1,373 @@
-// DTM submission automation program
-// Author: Michael Goldish <mgoldish@xxxxxxxxxx>
-// Based on sample code by Microsoft.
-
-// Note: this program has only been tested with DTM version 1.5.
-// It might fail to work with other versions, specifically because it uses
-// a few undocumented methods/attributes.
-
-using System;
-using System.Collections.Generic;
-using System.Text.RegularExpressions;
-using Microsoft.DistributedAutomation.DeviceSelection;
-using Microsoft.DistributedAutomation.SqlDataStore;
-
-namespace automate0
-{
-    class AutoJob
-    {
-        static int Main(string[] args)
-        {
-            if (args.Length != 5)
-            {
-                Console.WriteLine("Error: incorrect number of command line arguments");
-                Console.WriteLine("Usage: {0} serverName clientName machinePoolName submissionName timeout",
-                    System.Environment.GetCommandLineArgs()[0]);
-                return 1;
-            }
-            string serverName = args[0];
-            string clientName = args[1];
-            string machinePoolName = args[2];
-            string submissionName = args[3];
-            double timeout = Convert.ToDouble(args[4]);
-
-            try
-            {
-                // Initialize DeviceScript and connect to data store
-                Console.WriteLine("Initializing DeviceScript object");
-                DeviceScript script = new DeviceScript();
-                Console.WriteLine("Connecting to data store");
-
-                script.ConnectToNamedDataStore(serverName);
-
-                // Find client machine
-                IResourcePool rootPool = script.GetResourcePoolByName("$");
-                Console.WriteLine("Looking for client machine '{0}'", clientName);
-                IResource machine = null;
-                while (true)
-                {
-                    try
-                    {
-                        machine = rootPool.GetResourceByName(clientName);
-                    }
-                    catch (Exception e)
-                    {
-                        Console.WriteLine("Warning: " + e.Message);
-                    }
-                    // Make sure the machine is valid
-                    if (machine != null &&
-                        machine.OperatingSystem != null &&
-                        machine.OperatingSystem.Length > 0 &&
-                        machine.ProcessorArchitecture != null &&
-                        machine.ProcessorArchitecture.Length > 0 &&
-                        machine.GetDevices().Length > 0)
-                        break;
-                    System.Threading.Thread.Sleep(1000);
-                }
-                Console.WriteLine("Client machine '{0}' found ({1}, {2})",
-                    clientName, machine.OperatingSystem, machine.ProcessorArchitecture);
-
-                // Create machine pool and add client machine to it
-                // (this must be done because jobs cannot be scheduled for machines in the
-                // default pool)
-                try
-                {
-                    script.CreateResourcePool(machinePoolName, rootPool.ResourcePoolId);
-                }
-                catch (Exception e)
-                {
-                    Console.WriteLine("Warning: " + e.Message);
-                }
-                IResourcePool newPool = script.GetResourcePoolByName(machinePoolName);
-                Console.WriteLine("Moving the client machine to pool '{0}'", machinePoolName);
-                machine.ChangeResourcePool(newPool);
-
-                // Reset client machine
-                if (machine.Status != "Ready")
-                {
-                    Console.WriteLine("Changing the client machine's status to 'Reset'");
-                    while (true)
-                    {
-                        try
-                        {
-                            machine = rootPool.GetResourceByName(clientName);
-                            machine.ChangeResourceStatus("Unsafe");
-                            System.Threading.Thread.Sleep(5000);
-                            machine.ChangeResourceStatus("Reset");
-                            break;
-                        }
-                        catch (Exception e)
-                        {
-                            Console.WriteLine("Warning: " + e.Message);
-                        }
-                        System.Threading.Thread.Sleep(5000);
-                    }
-                    Console.WriteLine("Waiting for client machine to be ready");
-                    while (machine.Status != "Ready")
-                    {
-                        try
-                        {
-                            machine = rootPool.GetResourceByName(clientName);
-                        }
-                        catch (Exception e)
-                        {
-                            Console.WriteLine("Warning: " + e.Message);
-                        }
-                        System.Threading.Thread.Sleep(1000);
-                    }
-                }
-                Console.WriteLine("Client machine is ready");
-
-                // Get requested device regex and look for a matching device
-                Console.WriteLine("Device to test: ");
-                Regex deviceRegex = new Regex(Console.ReadLine(), RegexOptions.IgnoreCase);
-                Console.WriteLine("Looking for device '{0}'", deviceRegex);
-                IDevice device;
-                DateTime endTime = DateTime.Now.AddSeconds(120);
-                while (DateTime.Now < endTime)
-                {
-                    machine = rootPool.GetResourceByName(clientName);
-                    Console.WriteLine("(Client machine has {0} devices)", machine.GetDevices().Length);
-                    foreach (IDevice d in machine.GetDevices())
-                    {
-                        if (deviceRegex.IsMatch(d.FriendlyName))
-                        {
-                            device = d;
-                            goto deviceFound;
-                        }
-                    }
-                    System.Threading.Thread.Sleep(5000);
-                }
-                Console.WriteLine("Error: device '{0}' not found", deviceRegex);
-                return 1;
-
-            deviceFound:
-                Console.WriteLine("Found device '{0}'", device.FriendlyName);
-
-                // Get requested jobs regex
-                Console.WriteLine("Jobs to run: ");
-                Regex jobRegex = new Regex(Console.ReadLine(), RegexOptions.IgnoreCase);
-
-                // Create submission
-                Object[] existingSubmissions = script.GetSubmissionByName(submissionName);
-                if (existingSubmissions.Length > 0)
-                {
-                    Console.WriteLine("Submission '{0}' already exists -- removing it",
-                        submissionName);
-                    script.DeleteSubmission(((ISubmission)existingSubmissions[0]).Id);
-                }
-                Console.WriteLine("Creating submission '{0}'", submissionName);
-                ISubmission submission = script.CreateHardwareSubmission(submissionName,
-                    newPool.ResourcePoolId, device.InstanceId);
-
-                // Get DeviceData objects from the user
-                List<Object> deviceDataList = new List<Object>();
-                while (true)
-                {
-                    ISubmissionDeviceData dd = script.CreateNewSubmissionDeviceData();
-                    Console.WriteLine("DeviceData name: ");
-                    dd.Name = Console.ReadLine();
-                    if (dd.Name.Length == 0)
-                        break;
-                    Console.WriteLine("DeviceData data: ");
-                    dd.Data = Console.ReadLine();
-                    deviceDataList.Add(dd);
-                }
-
-                // Set the submission's DeviceData
-                submission.SetDeviceData(deviceDataList.ToArray());
-
-                // Get descriptors from the user
-                List<Object> descriptorList = new List<Object>();
-                while (true)
-                {
-                    Console.WriteLine("Descriptor path: ");
-                    string descriptorPath = Console.ReadLine();
-                    if (descriptorPath.Length == 0)
-                        break;
-                    descriptorList.Add(script.GetDescriptorByPath(descriptorPath));
-                }
-
-                // Set the submission's descriptors
-                submission.SetLogoDescriptors(descriptorList.ToArray());
-
-                // Create a schedule
-                ISchedule schedule = script.CreateNewSchedule();
-                Console.WriteLine("Scheduling jobs:");
-                int jobCount = 0;
-                foreach (IJob j in submission.GetJobs())
-                {
-                    if (jobRegex.IsMatch(j.Name))
-                     {
-                        Console.WriteLine("  " + j.Name);
-                        schedule.AddDeviceJob(device, j);
-                        jobCount++;
-                    }
-                }
-                if (jobCount == 0)
-                {
-                    Console.WriteLine("Error: no submission jobs match pattern '{0}'", jobRegex);
-                    return 1;
-                }
-                schedule.AddSubmission(submission);
-                schedule.SetResourcePool(newPool);
-                script.RunSchedule(schedule);
-
-                // Wait for jobs to complete
-                Console.WriteLine("Waiting for all jobs to complete (timeout={0})", timeout);
-                endTime = DateTime.Now.AddSeconds(timeout);
-                int numCompleted = 0, numFailed = 0;
-                while (numCompleted < submission.GetResults().Length && DateTime.Now < endTime)
-                {
-                    // Sleep for 30 seconds
-                    System.Threading.Thread.Sleep(30000);
-                    // Count completed submission jobs
-                    numCompleted = 0;
-                    foreach (IResult r in submission.GetResults())
-                        if (r.ResultStatus != "InProgress")
-                            numCompleted++;
-                    // Report results in a Python readable format and count failed schedule jobs
-                    // (submission jobs are a subset of schedule jobs)
-                    Console.WriteLine();
-                    Console.WriteLine("---- [");
-                    numFailed = 0;
-                    foreach (IResult r in schedule.GetResults())
-                    {
-                        Console.WriteLine("  {");
-                        Console.WriteLine("    'id': {0}, 'job': r'''{1}''',", r.Job.Id, r.Job.Name);
-                        Console.WriteLine("    'logs': r'''{0}''',", r.LogLocation);
-                        if (r.ResultStatus != "InProgress")
-                            Console.WriteLine("    'report': r'''{0}''',",
-                                submission.GetSubmissionResultReport(r));
-                        Console.WriteLine("    'status': '{0}',", r.ResultStatus);
-                        Console.WriteLine("    'pass': {0}, 'fail': {1}, 'notrun': {2}, 'notapplicable': {3}",
-                            r.Pass, r.Fail, r.NotRun, r.NotApplicable);
-                        Console.WriteLine("  },");
-                        numFailed += r.Fail;
-                    }
-                    Console.WriteLine("] ----");
-                }
-                Console.WriteLine();
-
-                // Cancel incomplete jobs
-                foreach (IResult r in schedule.GetResults())
-                    if (r.ResultStatus == "InProgress")
-                        r.Cancel();
-
-                // Set the machine's status to Unsafe and then Reset
-                try
-                {
-                    machine = rootPool.GetResourceByName(clientName);
-                    machine.ChangeResourceStatus("Unsafe");
-                    System.Threading.Thread.Sleep(5000);
-                    machine.ChangeResourceStatus("Reset");
-                }
-                catch (Exception e)
-                {
-                    Console.WriteLine("Warning: " + e.Message);
-                }
-
-                // Report failures
-                if (numCompleted < submission.GetResults().Length)
-                    Console.WriteLine("Some jobs did not complete on time.");
-                if (numFailed > 0)
-                    Console.WriteLine("Some jobs failed.");
-
-                if (numFailed > 0 || numCompleted < submission.GetResults().Length)
-                    return 1;
-
-                Console.WriteLine("All jobs completed.");
-                return 0;
-            }
-            catch (Exception e)
-            {
-                Console.WriteLine("Error: " + e.Message);
-                return 1;
-            }
-        }
-    }
-}
+// DTM submission automation program
+// Author: Michael Goldish <mgoldish@xxxxxxxxxx>
+// Based on sample code by Microsoft.
+
+// Note: this program has only been tested with DTM version 1.5.
+// It might fail to work with other versions, specifically because it uses
+// a few undocumented methods/attributes.
+
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using Microsoft.DistributedAutomation.DeviceSelection;
+using Microsoft.DistributedAutomation.SqlDataStore;
+
+namespace automate0
+{
+    class AutoJob
+    {
+        // Wait for a machine to show up in the data store
+        static void FindMachine(IResourcePool rootPool, string machineName)
+        {
+            Console.WriteLine("Looking for machine '{0}'", machineName);
+            IResource machine = null;
+            while (true)
+            {
+                try
+                {
+                    machine = rootPool.GetResourceByName(machineName);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine("Warning: " + e.Message);
+                }
+                // Make sure the machine is valid
+                if (machine != null &&
+                    machine.OperatingSystem != null &&
+                    machine.OperatingSystem.Length > 0 &&
+                    machine.ProcessorArchitecture != null &&
+                    machine.ProcessorArchitecture.Length > 0 &&
+                    machine.GetDevices().Length > 0)
+                    break;
+                System.Threading.Thread.Sleep(1000);
+            }
+            Console.WriteLine("Client machine '{0}' found ({1}, {2})",
+                machineName, machine.OperatingSystem, machine.ProcessorArchitecture);
+        }
+
+        // Delete a machine pool if it exists
+        static void DeleteResourcePool(IDeviceScript script, string poolName)
+        {
+            while (true)
+            {
+                try
+                {
+                    IResourcePool pool = script.GetResourcePoolByName(poolName);
+                    if (pool != null)
+                        script.DeleteResourcePool(pool);
+                    break;
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine("Warning: " + e.Message);
+                    System.Threading.Thread.Sleep(1000);
+                }
+            }
+        }
+
+        // Set the machine's status to 'Reset' and optionally wait for it to become ready
+        static void ResetMachine(IResourcePool rootPool, string machineName, bool wait)
+        {
+            Console.WriteLine("Resetting machine '{0}'", machineName);
+            IResource machine;
+            while (true)
+            {
+                try
+                {
+                    machine = rootPool.GetResourceByName(machineName);
+                    machine.ChangeResourceStatus("Reset");
+                    break;
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine("Warning: " + e.Message);
+                    System.Threading.Thread.Sleep(5000);
+                }
+            }
+            if (wait)
+            {
+                Console.WriteLine("Waiting for machine '{0}' to be ready", machineName);
+                while (machine.Status != "Ready")
+                {
+                    try
+                    {
+                        machine = rootPool.GetResourceByName(machineName);
+                    }
+                    catch (Exception e)
+                    {
+                        Console.WriteLine("Warning: " + e.Message);
+                    }
+                    System.Threading.Thread.Sleep(1000);
+                }
+                Console.WriteLine("Machine '{0}' is ready", machineName);
+            }
+        }
+
+        // Look for a device in a machine, and if not found, keep trying for 3 minutes
+        static IDevice GetDevice(IResourcePool rootPool, string machineName, string regexStr)
+        {
+            Regex deviceRegex = new Regex(regexStr, RegexOptions.IgnoreCase);
+            int numAttempts = 1;
+            DateTime endTime = DateTime.Now.AddSeconds(180);
+            while (DateTime.Now < endTime)
+            {
+                IResource machine = rootPool.GetResourceByName(machineName);
+                Console.WriteLine("Looking for device '{0}' in machine '{1}' (machine has {2} devices)",
+                    regexStr, machineName, machine.GetDevices().Length);
+                foreach (IDevice d in machine.GetDevices())
+                {
+                    if (deviceRegex.IsMatch(d.FriendlyName))
+                    {
+                        Console.WriteLine("Found device '{0}'", d.FriendlyName);
+                        return d;
+                    }
+                }
+                Console.WriteLine("Device not found");
+                if (numAttempts % 5 == 0)
+                    ResetMachine(rootPool, machineName, true);
+                else
+                    System.Threading.Thread.Sleep(5000);
+                numAttempts++;
+            }
+            Console.WriteLine("Error: device '{0}' not found", deviceRegex);
+            return null;
+        }
+
+        static int Main(string[] args)
+        {
+            if (args.Length < 5)
+            {
+                Console.WriteLine("Error: incorrect number of command line arguments");
+                Console.WriteLine("Usage: {0} serverName machinePoolName submissionName timeout machineName0 machineName1 ...",
+                    System.Environment.GetCommandLineArgs()[0]);
+                return 1;
+            }
+            string serverName = args[0];
+            string machinePoolName = args[1];
+            string submissionName = args[2];
+            double timeout = Convert.ToDouble(args[3]);
+
+            List<string> machines = new List<string>();
+            for (int i = 4; i < args.Length; i++)
+                machines.Add(args[i]);
+
+            try
+            {
+                // Initialize DeviceScript and connect to data store
+                Console.WriteLine("Initializing DeviceScript object");
+                DeviceScript script = new DeviceScript();
+                Console.WriteLine("Connecting to data store");
+                script.ConnectToNamedDataStore(serverName);
+
+                // Wait for client machines to become available
+                IResourcePool rootPool = script.GetResourcePoolByName("$");
+                foreach (string machineName in machines)
+                    FindMachine(rootPool, machineName);
+
+                // Delete the machine pool if it already exists
+                DeleteResourcePool(script, machinePoolName);
+
+                // Create the machine pool and add the client machines to it
+                // (this must be done because jobs cannot be scheduled for machines in the
+                // default pool)
+                try
+                {
+                    script.CreateResourcePool(machinePoolName, rootPool.ResourcePoolId);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine("Warning: " + e.Message);
+                }
+                IResourcePool newPool = script.GetResourcePoolByName(machinePoolName);
+                foreach (string machineName in machines)
+                {
+                    Console.WriteLine("Moving machine '{0}' to pool '{1}'", machineName, machinePoolName);
+                    rootPool.GetResourceByName(machineName).ChangeResourcePool(newPool);
+                }
+
+                // Reset client machine
+                foreach (string machineName in machines)
+                    ResetMachine(rootPool, machineName, true);
+
+                // Get requested device regex and look for a matching device in the first machine
+                Console.WriteLine("Device to test:");
+                IDevice device = GetDevice(rootPool, machines[0], Console.ReadLine());
+                if (device == null)
+                    return 1;
+
+                // Get requested jobs regex
+                Console.WriteLine("Jobs to run:");
+                Regex jobRegex = new Regex(Console.ReadLine(), RegexOptions.IgnoreCase);
+
+                // Create a submission
+                Object[] existingSubmissions = script.GetSubmissionByName(submissionName);
+                if (existingSubmissions.Length > 0)
+                {
+                    Console.WriteLine("Submission '{0}' already exists -- removing it",
+                        submissionName);
+                    script.DeleteSubmission(((ISubmission)existingSubmissions[0]).Id);
+                }
+                string hardwareId = device.InstanceId.Remove(device.InstanceId.LastIndexOf("\\"));
+                Console.WriteLine("Creating submission '{0}' (hardware ID: {1})", submissionName, hardwareId);
+                ISubmission submission = script.CreateHardwareSubmission(submissionName, newPool.ResourcePoolId, hardwareId);
+
+                // Set submission DeviceData
+                List<Object> deviceDataList = new List<Object>();
+                while (true)
+                {
+                    ISubmissionDeviceData dd = script.CreateNewSubmissionDeviceData();
+                    Console.WriteLine("DeviceData name:");
+                    dd.Name = Console.ReadLine();
+                    if (dd.Name.Length == 0)
+                        break;
+                    Console.WriteLine("DeviceData data:");
+                    dd.Data = Console.ReadLine();
+                    deviceDataList.Add(dd);
+                }
+                submission.SetDeviceData(deviceDataList.ToArray());
+
+                // Set submission descriptors
+                List<Object> descriptorList = new List<Object>();
+                while (true)
+                {
+                    Console.WriteLine("Descriptor path:");
+                    string descriptorPath = Console.ReadLine();
+                    if (descriptorPath.Length == 0)
+                        break;
+                    descriptorList.Add(script.GetDescriptorByPath(descriptorPath));
+                }
+                submission.SetLogoDescriptors(descriptorList.ToArray());
+
+                // Set machine dimensions
+                foreach (string machineName in machines)
+                {
+                    IResource machine = rootPool.GetResourceByName(machineName);
+                    while (true)
+                    {
+                        Console.WriteLine("Dimension name ({0}):", machineName);
+                        string dimName = Console.ReadLine();
+                        if (dimName.Length == 0)
+                            break;
+                        Console.WriteLine("Dimension value ({0}):", machineName);
+                        machine.SetDimension(dimName, Console.ReadLine());
+                    }
+                    // Set the WDKSubmissionId dimension for all machines
+                    machine.SetDimension("WDKSubmissionId", submission.Id.ToString() + "_" + submission.Name);
+                }
+
+                // Get job parameters
+                List<string> paramNames = new List<string>();
+                List<string> paramValues = new List<string>();
+                foreach (string machineName in machines)
+                {
+                    while (true)
+                    {
+                        Console.WriteLine("Parameter name ({0}):", machineName);
+                        string paramName = Console.ReadLine();
+                        if (paramName.Length == 0)
+                            break;
+                        Console.WriteLine("Device regex ({0}):", machineName);
+                        IDevice d = GetDevice(rootPool, machineName, Console.ReadLine());
+                        if (d == null)
+                            return 1;
+                        string deviceName = d.GetAttribute("name")[0].ToString();
+                        Console.WriteLine("Setting parameter value to '{0}'", deviceName);
+                        paramNames.Add(paramName);
+                        paramValues.Add(deviceName);
+                    }
+                }
+
+                // Find jobs that match the requested pattern
+                Console.WriteLine("Scheduling jobs:");
+                List<IJob> jobs = new List<IJob>();
+                foreach (IJob j in submission.GetJobs())
+                {
+                    if (jobRegex.IsMatch(j.Name))
+                    {
+                        Console.WriteLine("    " + j.Name);
+                        // Set job parameters
+                        for (int i = 0; i < paramNames.Count; i++)
+                        {
+                            IParameter p = j.GetParameterByName(paramNames[i]);
+                            if (p != null)
+                                p.ScheduleValue = paramValues[i];
+                        }
+                        jobs.Add(j);
+                    }
+                }
+                if (jobs.Count == 0)
+                {
+                    Console.WriteLine("Error: no submission jobs match pattern '{0}'", jobRegex);
+                    return 1;
+                }
+
+                // Create a schedule, add jobs to it and run it
+                ISchedule schedule = script.CreateNewSchedule();
+                foreach (IScheduleItem item in submission.ProcessJobs(jobs.ToArray()))
+                {
+                    item.Device = device;
+                    schedule.AddScheduleItem(item);
+                }
+                schedule.AddSubmission(submission);
+                schedule.SetResourcePool(newPool);
+                script.RunSchedule(schedule);
+
+                // Wait for jobs to complete
+                Console.WriteLine("Waiting for all jobs to complete (timeout={0}s)", timeout);
+                DateTime endTime = DateTime.Now.AddSeconds(timeout);
+                int numCompleted, numFailed;
+                do
+                {
+                    System.Threading.Thread.Sleep(30000);
+                    // Report results in a Python readable format and count completed and failed schedule jobs
+                    numCompleted = numFailed = 0;
+                    Console.WriteLine();
+                    Console.WriteLine("---- [");
+                    foreach (IResult r in schedule.GetResults())
+                    {
+                        if (r.ResultStatus != "InProgress") numCompleted++;
+                        if (r.ResultStatus == "Investigate") numFailed++;
+                        Console.WriteLine("  {");
+                        Console.WriteLine("    'id': {0}, 'job': r'''{1}''',", r.Job.Id, r.Job.Name);
+                        Console.WriteLine("    'logs': r'''{0}''',", r.LogLocation);
+                        if (r.ResultStatus != "InProgress")
+                            Console.WriteLine("    'report': r'''{0}''',",
+                                submission.GetSubmissionResultReport(r));
+                        Console.WriteLine("    'status': '{0}',", r.ResultStatus);
+                        Console.WriteLine("    'pass': {0}, 'fail': {1}, 'notrun': {2}, 'notapplicable': {3}",
+                            r.Pass, r.Fail, r.NotRun, r.NotApplicable);
+                        Console.WriteLine("  },");
+                    }
+                    Console.WriteLine("] ----");
+                } while (numCompleted < schedule.GetResults().Length && DateTime.Now < endTime);
+
+                Console.WriteLine();
+
+                // Cancel incomplete jobs
+                foreach (IResult r in schedule.GetResults())
+                    if (r.ResultStatus == "InProgress")
+                        r.Cancel();
+
+                // Reset the machines
+                foreach (string machineName in machines)
+                    ResetMachine(rootPool, machineName, false);
+
+                // Report failures
+                if (numCompleted < schedule.GetResults().Length)
+                    Console.WriteLine("Some jobs did not complete on time.");
+                if (numFailed > 0)
+                    Console.WriteLine("Some jobs failed.");
+                if (numFailed > 0 || numCompleted < schedule.GetResults().Length)
+                    return 1;
+
+                Console.WriteLine("All jobs completed.");
+                return 0;
+            }
+            catch (Exception e)
+            {
+                Console.WriteLine("Error: " + e.Message);
+                return 1;
+            }
+        }
+    }
+}
diff --git a/client/tests/kvm/deps/whql_submission_15.exe b/client/tests/kvm/deps/whql_submission_15.exe
index 4f30aa801c8f200bd96b0de6c10d2b59c2bad268..605e2e3aa1f0761155f4aac5191e3ad0f0ddecc6 100644
GIT binary patch
literal 12288
zcmeHNdvIJ=dH?P{u2w5c$=S8_uq69RerRRAWhb#6JFzU;i5$sREZN3H#$N4SOPlQO
zQSRN9r6d#)+JakS$V^B<lTI?7{LyqKEoG7cTA+l&l(eC3Iy@R41{!Ds45b~)s|C{E
zch0@qUD*!#qyIHmy65qI-}zqWJKs5bcE{iPehLy%2=8mJ5q%6bKL=F#`o%KPb({a9
zj-IXlLfglTu`jeu&1GC`-jTCTGH0cd`MfMxGq&Xv^HwHrjU1b_ax!iA*49S1daNf7
z5seu^YX8Yk#(ZhtqJU9jM2NNnf{LF{ji9jbp2N%9S*QY9Z?u&DT+@?y0p}-32Y!d>
zM)hB+y{;yqH29w4_Bo<|W+$wDmWkE@_UtfG%Uay)^{rLa6;%ZNJIm;M3-)3G{m-%^
zfK#@5dZVWv3tII$u9HGf>2?kc6K?`<*>eC6>2>U^1R?86%XpbrFW$1}0MWs+u4+CB
z`}twr0u-iCbP@f$fyh8Vtfevf4L{BOA$l*UF<v+*=7R}%HM%|;2ukpaMz1v1I5z=a
zC1FW4QN;!3IC_Q5?psV*jb?YBu4NRR`hhl6)-be5*B-RSl(02=IaA-i&U!Tnr_`d`
z^+9Kpg=z+RK%x$a9jp^VfvA?E4b%vZMst6FZUF?Y5>W33kVVbzs(>j$7`@VdZADAM
zuKeu|k0;Uk_0;b3d>lNXkZ?Y*YQJy2>3j|_DPE8Nd*YrkS6&EppLAYYWzyB{v$@i^
zPB}4hl@m+anKA|`POCxl32=<|e8%ZSThGT1Lj`jUZLqZ7M8_#Qt^E?m9DW@(VB$om
z!CjBR(T1AD2GnsgW^MrrF?Rkyq8^ZSyJ01Hiq$%&AR-j3_U7gi3^kPhH6&hDACGkH
z7F8m$QUH4+9x_EpBHp615k+@{^DQv&Kq_)XC?39YTtph)5|4<8sJfgP{Ip{RU=&A9
zRr8r1gjPcW{x-X!)~fHS8s4HcjwG5uxA&gcUVF_+V3?`nafb-+XzmZwcM*VyOA)x7
zY3^%v(mrG7^W55qq4g_jM8l3i{Yo^@uu>CfTv-=rT1hkmS?j`&(RgTMJlrKjC?LWs
zvK0^!hQ47D2HU|`op%I1b=^%&Z932U0AZSPGt-;Sj{%bw<EvnQ8#j8oVzHh?J2zpu
zh$OI}Oh7OL(Vp*Ksf+dex*oyXEVlyibzxS)Kr&H_D!C0Hi3A$mer034T1AVfzA9Wm
zt7S%gnY~@xW4&5tjq@|;Bx<S>9pD26YSa=IQHcZ;1(nj{CFs#xp@*k^UspWVr#Y>|
z;B|VE08+!ggJFH;!<_vg&r2W<)yq!ENMPAU+5Kj`R@C-H8sgpG*oC~|obhQDg^tm9
zoiHO3`70W~LEM1EFkm?sKo`+*WQrT=R|<$SLw18k_Mi~85;@YZoG*b|_A#;_Mf^tR
zWgaeW)b^VZSk7)0H_8FV3wMyKm@wr|Zp59PVc&$fQcUh*^0-98i>kcU-R7KFOLH^R
z#6X2y6z%$m5F#db^ME}lw21Z21%H&{8{Z%{XnrxVUYO|Y<pGMp7r|h?W^gNj{=D{g
zEyeovY}CWS^=c~P4bF$aSkx;QY1WORQL}DwK3Bqt2GOWZ+6SgxpBD|zj~OeJGvfoX
zfy90woVclgx1pW59ff3a2<AE8`ydt}roF#>+Vuiz($z*^+rixrBdf#)wZhF#1G8-|
z)3QahXf4k;!zCOHYgSt7m}wQQI%ZTVYjy4er91#l>R03ts`w_S$b&cO2x$UOwRks*
zO{&rLZfKx1ceLh%<(fMm#aNYv+=I+)Gn_HZjh*%JU0rV%JH@V*0+NIgw?v(=<Opy&
z%iZI=%wjdDv<243+q$}iC7MK=ihg;BxwVV-?i)qB*yuwK11j2(U)n^QXp(oJJNSaw
z40hPWqhc2@yTmTB^K$0)MW`({OKgeJp7qWvEVNG9$_wcSNEQg)BS6NtAZRwryU>(l
z+`p+leskAWag(@trGT7e#J7sASS;hfi7jHQJj#umA?apuv$zRK?u!5mc}(M@a)KMC
z*dp&n!@0eR=BqCu&l!3K)lPCt?3QY^gT%L4{Y7uftKSyyh;L6E=TT2!lyWcFOac(!
zT_4}GyBRr9>=t`g3dog4JRuTtim9JrYToC%O|cz{ZTAs>&4?2k(E&t<^EIEb^9ov^
zI|-~jg+lFDK7i*-t$e2=qR8ID03@TR&$=Wt?wn#80nBNi%K3dgjMY=aoaX@8_+A)b
z#P^E5^&oyLa7ySOiM<Wz-&59aM1Q)h--P~|vVP)i=s0_;eJ|~SaeKrbu^SWr)n8+~
z8(Bqd_gr{6(<jtEF1G1y@nO&@Z~H#JwLZQNL3pd!=Q+4jbe1_-`LPP1vyd(JLE=8K
zPuz+Cc+p?FJxKfBrnLj>aYwz{oBdsj(|+ZyRi<0iIliiLNBuz5St;xG*NQUIFe3O$
zxAQ5VpdQ`Uc$f390En(N`yA7f=;kxv&}a40_%XB*kn|khI=ue#cWB_;MBl)R-JkeO
zhjl_x%o>tB?MLm?Gkgkt3w2=){9k~)2>5-$O8WPs{{g_aA&$!V+=X)p^SKYT4@Z{}
z0F(jjb9j-Pls=mRiF43s$HB=X2Ms=H<Kc7v!eDQI?@j%;3}PSPQ%x4-BIIoOEu!zB
z{xEjVEt3T&lb>~&;!zm)^J>6OOwy$d`g75iJ5G#_0QWfh)39O7!K|ENVm#;?cWekm
zFofv8jh%$^A4|9J0;fKjfF+&73p4p##fewZa34B*+M2YR2GDzm#!x<{%Fm)SsoQ8W
z8t4Gnnk0>5hDq;J<#Ptp|Bb;kub^$xFH|`lxGPXcF9x1L`A;ZK`euN!|D($7L2e&X
z<^92v!6@AhDU<Xl)G_HZfJf=35bN-$Du1EMV_}9&tMV~behQ_DWXbefOk_wr+V9%`
zH^LR0U@!M3EXF=UzXw`_e(Cj|A_1RFv}xQ&sS(uc+jZ|ncrZf0^m<>WPR#PNU5xwP
z#;;MInp=aujedZ>kJ6x@s`jr`y95uCX)v}NQllX2jZXR+;&%_qJv4-}k4EW}#%<8*
zC1Zq6(+`a!l!Snz3O+@5Q&oT=&4JTsZ%6xXIuJNR8A=6a6wNF(Q6{i}@+r*Y-Sm#Y
zMU;=KQBMRe)BTY0hxAT*HSiS5{|G!yZDa(G03Hke86X>j&%#TC!GE9+(RlESuu|K)
z82lzZt=do1L&1w^vy^r8MDV+`o}LN*7s`(Ze?cFozYq37|1SqmgXVj|Un#9Z#@+PG
z;6tFlAyi|GQe#LMkQ~}zJWoR*ro1z>2b!D?v2}ByW+O?3kOi6tL)-A7h;?|H{v@>1
zc!542x=1h3OQ3v#zKV7Wv2HE&N@zE@{UCH3Muo#8Xg7s#gWT@$5#u852_K>7)hPDo
zgwaL^!yAlh1>dUpKTlKPGxV~ed6{OzZ!^A44}>p5=S$(!v{gyjs@hM}li{?n)sy&<
zaK`vgdNDk2e2=~oUO@S8;fGNEB>WI`4x10d-cK6u0i@A9Lx$01vW@%9%V^(eUNT-*
z9x#jvz@Mjc<|Bq-JYb#%_FbyHq{=@)$$UNt?9*l+RV$r)5tE5PFU4s9Wi#y!NP3ta
zrBBck;QQ~?W1KQV%LvGd=NUvFX9oW<ysHaC<pm##b0VHfIvlX|vu|9ci&^_N8kriO
zvfV<jy=c==u^^8iFC|guk_Ee;a&AgG*~|=0F1ZCeN5^LFwNnKeu@^Eadotx@<_k2Q
zNjcJ$=L@|f85e1ArdY7k%nB?rGT-YF+gXf6pGJ?{t}Hq!dqT=A9nR#_<H^)qCU4Vd
z1>X^jDA*Om=-7od)O3ek(Bo)4naNXcsvsS5q%0`@InDcMGG~*k6`qG&)xVI;6v(k>
z?M0+-N;=5j+D=%KoXtzOkV&~PcBVKxo1DqoLj}(~4~C2nH`=SCju-QVOwR5dmbv*%
z)^;XshyCP|1}JwNx3kGb)pAShyn##x)J!IuDJ+$+-V}jL-j!LKPC1!^J*FJydxxFi
zw+}8U;|?vR?0MzzS-Ws{+;-jMtWA2KVpgeSff#da-gY1~KkH2~!^Z6VY++8JCmab2
zT<Husn4|*cu;|z&pRd|JHRssLG}w@C(PY-P=galt>F{(Lo=fIuSEqKekSr8kCCzKf
z`Lm;W`~G5*9pTT(6t4DqkK41wY|=TjIPX9!gp*4E*^9b7rVPfEA$X@S6N=|id4Yz~
z=}9{!^J$kx^M##*3US!U*!gr;yKB@PPZm;h1kq#QJOvHuP<|of$b8Pu7uXO?PcE5H
zv*(96R^Y@1+bPhL9FawYhp*_c%zClz_TFLVZ6}kWF)WR91FQ!R%fpRRlBX#RRg;s5
zSR2xcIi3ncFNBr#ct(bcj>Bq<%LV%=-*Jv&vCT`D!;*AUTkA&wdwUeZFe_^oQKj?M
zB4G)myuOcP4pn49gURAdE~9+x&Dv-QqWlN;dX27y=iNdwpR$1;OS*;8eA-?-b{@QQ
zP~Ow%&ZLvRkaX;_p=Ip#dS=&n<VWobCCV}^(!p63?QDT+FhmWUl0%M@MD+OSK|zsX
zbQaKFF7*yBO(YTWpdOR6vIIwfg8^bG&(Vgt_Er!aR9c~<oHYSKg#epK;+`I<OU>hI
z$8#d<H<_BVIs2YWW{Y~(h9y!5jrxc(t36Wk)}fZ*)2TUQj$Q)fF!u&16!%wWkqs%2
z4no%PsmY6A*Z`7Zwg5A9L(Q~qmKNtBe`&JY*><ksw=s#V!fpbel3+dhxIHhiSeSAm
z>FVWvIGM>R$Wd8<Hhyzxem<K?sn8zgK*EW8=NL)+-G`GB$jGAe=sBo!*o#v%r{FgF
z9rPge)7w!S@l(J#px0ye0!w!d<NTf>8_>e*j@A;2pi6@;0j>k!+Ku*uG>BFw8GXmW
z$HiN~pRd>J=u=aor@L6UOxd{KY>x$M3HdXSXu&QU9MZHzhT^N+)H8lve^`*`R*qbL
z_x0SG2KxmJNGV|@r!tc~m|V+Dc2Ea!iL0=4P>Mxs*I7`z#NB0AZaWOqd}o(qD<-sU
zq->l8JtSOGK2db*AvnfSp~2IAJx<peYosd2vYjc#iCvvilJd|orzU_yPa+5{MUzt&
z>rmnF$}6vN@Xe|bI^6K-aQV3ta6}Sdn^-{|tCor)2M)I}<S6ht#h}v36A&m7y(PR0
z<};w>d38}r2rgkcCYcXA_Uc&o>H48Xr5qnK08Fp?qoGl7*3p_&>*oRV^g1k`l1Vtj
z0cKvwlhEy6$Y<Blz_8Me9m9Qpw2tUBEH2<>KQM$HPQk52hsM$4Fx0Xt^(+q!e{IZz
zo_(ZoIv|bB<vRM51;}7&xZ!O&f-y7b`8Wq2%QNQ8<TX58DVP0GT8kFAE<#`C%=~)r
zr`Kw(!V>nS5yUOT>0xhl$tTyE)k>KO<U0<oG+j_N=Yvri@gj<?#*(U7EF-HP&ZEUq
z0qd?q;m?Lo;g-#+&bjl7+Z^-g8iBRUFAsZl1oGnYBbf6hXX8$$?YS210%)`FGTTr=
zz4;U@x(l}oSI77$ET<su%DN^n8Yg$90naEe%j-z*sEh)hGhT&@Yvoq^<7sFT`10J&
zcCVFIuD_1vQnEDKcty;COBz<_#K3#ky<WV*C)F1E{73gxrXNmjydJK$zBFF1+=7x}
zVb$>}(q7;cU8cv`>+Lq|dtNQRM*a%m#Fqodtnz@Ko63B$-<w%Z-&D(Su4DZkGB$9w
zn1H7wJj{OP>_iO=<qXGbBLfU;i*;Vb);~xz?0cdE*fg{+<wxFiJ3JjV+(DoJ^scPi
z-Pn;!JhDo<3mWEAqSt2w9s&BDC3@DAvxft%UUy7qX$)J!Ruo~A9#ir;xG%q}M9P!H
zZqQ_UQE0Qy1CxQHeT;w1=zzO;f78jx$MVrbkJH#Bl;&yTojQpm#uMh#?gXu&0*uE{
z?HM@2pD5#wNpRQOOlk7d%Jrq)@#-+(Erx?cFZy2GWL}GpZxaVTXgY?VS7n&3<*ev#
zs9XZLsd(9V?2qF|Umf}F?f3lr)DI7Ti6S3;VESa!u5Vp5@oSb?b)Bke>#8Woy+DM5
zF@Z)ME)65CbwP`auw@vc33pLOB#be2Edazi83m$1U0q!WH#OXige+=p;a;SQLLh8u
zF{=Wx&R7qRsIEhaib2(}9_Gj6!5~r<WTIG)X@R}K_pKy`KrDu8K^1^*fq)niL$Rvr
z*q#XLPN*ovP-{yBq^&H!wl2IOb~hM;Yn&n$Z4jr?2y;~jwc-T*0^kodA|`Bwa-an%
zZd`d(YYWJX2p7=D3dWLk5zVs&!((<Brhy;Ww<`NH7zAbEz92kMXR`EI4*1ypEH73x
znWv&PVuHuTmQ<NkWf5Wxrptk@F1Gwmo`$;CmgRR>?FYMBWlSV?0_wssm=0LH2Rg=*
zjA@D4ssgyPPLZmDsZ=+Xi{&D*`(wq}k{F6az#ji#6jAP2Q)nQ3iGSr&Ryz_gyo)b1
zBI?F#6JH`v1?o>Z$@!x)zj`}@+X{KXH53i4{$478{v&w*aTxD5{EB08WODk_pCA5&
z@zdewe)g3oKl6?EJ-!k6;l0x%GKJ6W`GRZVZotB4EWS%|r-#zHOrGz<_yhg)_>#3s
zH+?eW;%38|EELljX$|)G@1Dk|JHA+Rr@gzL)mqNnJB=$NJL%d!tam;=!|Gld;Zi?;
z%5BT*tf~CpB2|LV4olpk*y_qm-SXLXZ#v80ZQvJc+sHbgT5lHr7aj=kujP1pmhn3Y
zd`CmesLCGwZ3n*GT`mKnso`_ifHU1P7Zs0=;`amg<T;`d4r1?l6%q4&Kga<a3E(Fb
zbP}hlvnUVYS4xvOK_0`WhqI`U(qXju`DExvzh*twKKgFC^gUELo<q?9S3WBpR$Cix
zcCl^nCtQBEQV8V~ut`8&Yz(}I>a(LFpbv+hf=B~8@%E=r|5tNbRy6%MU+TYI*#7##
zpc_lySe4ADzVZ`nNsa~Deq@Id?Mcu&pf1t$WBV$-Ja_9r&(`?oEIvUUg$2Bo!mqGv
z9Irm*^rBu=@PyK73~fI5WQjiw&EvB>TQ`dnB0eDz>vKTyKjy*Na;EleUp<*#SI#bF
zk3M_xnUg<jt+lFT58K8zu@8n|H{WIO$J;D4vwq_^45c6a<zMZ)eKD7{a1qM)WBc0&
zdi&cgJD-w#7rejy#MI%Q-R%}%2&R)+nYZ`1FWGMU?YGrN_u&@E&dp?(EHKEs``e37
zey@w`t6b9U$@yO@^rU2NZ_>^6E)29=xPHx?$F~(H%cX&pWvwzBO(PfM*0@3=e1QwC
z^0<}T-#)(NT`#FGCzA8??R}a<!71`(<$3uA^%-ne;~?9$Q$^g#FL`}H99&*QPdh!~
zWESvMz^v`QL9RP}t`?pWAEfG3ku@H(7woK+<#K;}(jCn&$op)k-702=QhZUrzx{kN
z>)M`Cib>x!wDDE%tI&O4->MTpu&>XbA#`}Q-+bP%2M*}p4t$!^?3>U3I}iLXAgw%l

delta 4268
zcmai2eQX@X6@Rn0_u;jD@$A~`v+vGXf1iDK_8AP#C$STo;3&pPtUyu{$YCE~YP@9c
z;wFXmQV1bb!A&}-0!K}uYN=2uF(fUhEvQs23Tj$4sA~C80%`@NfIun&(V#Z{&E8$?
zgg?5MdGp?H-h1=r&70YE_U_ob;n0C~_kXf>Jr$p~NMYqHK-2&LoqV2J0b-%`p-Ez)
zQ;1Y_@+tMS_wXxS!EMkUk#HZ;tLouKqhbe91>`4}6GcR$k`P;2yLKW1$U<un-(t;W
zec*g(miReaT($!oCZINPmNi+2<otN9K~u;}lQB3qh!|uvpE%u?5*rjX@AzGSD>D<}
zuBd~tJ7Wk=b25_if-`4hm_@7YRLogTshurXLbDZ0-E5^&KO2L!W~k1mC9^EnuNhuv
z2`l>HUY}hCjC-Bif!UfI_c>GG{qUP7t=jN&pWiN*1OBu1M!=cFAP?9`fEKHiL<V%0
zxIeLi`|Tiv#N|cCUIfYPX@OWJ@o<qT;uR1%@R<oSRbjpBA&Th`<1@~0WdPitcik0@
zVn`4I4RD`b4bH9sX+uP1g^0?&7?C$nSdivaDRgJRO%(TY%{HYczg>&Y*@2;02;9=2
zBKyvy6y=KKs`G_qbH8&6uw4h(I)E_TFoC{aLeI7t<%KZHzYxZ}Mf^$Uos#=_>Y}t(
z6s#*Qw$3B3^LhC~#L4HKUQaPxXS~G4>JesbHVkT1I{N|hN_Xio6sTr2>&?wrD4rYp
zVo`}Kir+pTi}Rc`DDF#s;vhDJ6%*&~mC<4vt??Ld;jObdS+YhIujExWPM2mc21)pM
zl@UC<jJMWqF<N;mZ<(36>`(JaaHHUhVit6h2b^>B@zoklXYNPpohwy@5QIzApF6j+
z4T!rd3m3<^x6E#oE6}-)8tSo}-Owv5gzM$HIh#YJFuO^{hDB@6Ulxp)p%7#tVWt+T
z&<bK0**4i#bFbY_l-G<JNAs2{qQ;%w#$nZXjol%QWPv9dovurDI;BoEwBlgm&v1iR
z+X-oq1S!m6I-5#Xu3w{;*B1Qg+~i&eU3qEYdiXOum<12r01;jaoe^@TOFm4ajvIE5
zMB7ZGJwC+Sc>8P)HOq{8&Uw9kne^svQM>an%!<(bA?6_-1bDfepbooNm*?&1+Ic%~
z!zLc%b;)YysI=s6j^&2o2oUTHyH{F;kkcz9<K-dyN-)kjsjePGO%}?iP=ql(#?@l=
znMT+&B1`VV<z}UJ6%Q9c_{@I^E{qP-=#0mC2k)HCp{kh?<xzWybiT!ht&wLJMHfb-
zVcrSa$vb%m;{8KxsZ^ShO7cWw7DTlSoDnyT#F9^N26&t&ie4L>yGxqRK5)Fj&f|jD
zY+Tlwdk%oOfJ|g;C5Usfpm&u7<TVV9O9C^T&!molkyRxvC*gP9OI6p*xeh@6gUA3g
zl00cbxesFp^YZllfG2AK^h=;9haj(BAlE@&vp|jwKycoG)5292DcWyA1`^0nf+u)<
zW?~hcKY!kN`+}NGxy<;?L^pT$DPKIN%WnPe`SWfzO9|{UjqZ4Yck%9G_7?NS3zj|z
zF9|=OtHJPYOxex5c^5Y2)cjTR|F6|Nu6L_&Gg=qljPixEAY9G)_`)h8W{>794_sKg
znW|i<>P6L_O*l7|6ikRwqtUq^0B^i_mb@mFi8tXm_dkSk!E-m7eEA<hLsd&2`$psk
z#r9Ag`i1yT$X&p{e+hmXvh;K268r_!k<|YHeak{eC^)LY8+{WR?rnGC1|_yuJXaaJ
zY5*_K=5G+qq2Ge<g67ejGck2*7J3xG8~$Ou`I|S7(xZ6a;#Ew|Yc~&}NN5i7AtoAv
zZBZO8_Y*~arXVI-iw18l&qO*H&im+Qd<d1(G*BP?i0%XZolD=R9`OHhX@vEVMoG4e
zX>^ZE9|!f(Z-D#gRhKSTI+Y;(Re28d45*LZQ{<@6BW)`@(smfsM<+awcr^N@M55H9
z%7N=$dKajVo>is6%izV8i(2xlATS!o4NbH^G=QZRR<n*c4Nx^Tg4R<9Xcc1pm=@Dg
zDq}4!-$DJ<!n$e59UY?ESuglQY$<J|Ikt+@bczkq1ii_IXe+(r^5@tHc&{=Dx>y;_
z(@t8gY;oOfhuJmC2wdEtWH3CXOoDzBsuOfjalp?iz0f?S?4~`K@@{&7K347p{aiUf
zk5IK|5O}L+21wE);Ag4l2t7tyJ<{h6Pd#*YLT8BfdLBbD$R*{6=(Oh;Jxw2ZdO<(K
z@YBT9UJPsM3DBVW3%D@VU&Ab=z65^2<<sgZ@L6?~UUscsq`m5CdXXMfUjsd&PD1}D
z>YLDc0XnbIX?4(D<l7i6Ab(T;NiXN!=wxC)gwlIZdW_DgpMbCSj(|p8+UC+8P^tMj
z^0CJ2V06S=MP=?fx=>`>SQi<T0S(g>Q`i%v?!~TN^!y!?+~Cq9rgXD5DU0_am|-Fk
zkLXvI4W}ojsB0`|i)W0ViI-|p;!S;}h?t?Kp-q{ry~`QPY_RP~TARr&lvnS$HhqW4
zRR2&+n=7kFbLrf!EZKM7G&Ggjxhp+6k=r90YRtf<OnQ9%#8gJCui5M$8pupfjAg{5
zHErTWbFB#1)`?GRnnZ8yh^K#iT-;N;?uwz&vD-4^yCyTV#>u2}nQJp&y?`Kub@t6!
z8UiI46x=}&`$N}>cSC6t0Sp$Wr#0!RvCO1gxHu8cuy2d{NTPg<CXw6>O;JuPk6bVQ
z7}-<bzn*OTzJh@%x|J-t1(Jh)OdP3e7yIhcp^;%g(-#zSbQ^pW6r#e3R(Gf9>(oxH
zd=v0AjiXHIrZvz?yDm$P7utmf+9?YkS;|2xN4s1phj}b`l76yiIq9`>GrVM>bqf%3
z=O~QPusCFen2a+TDwy636${Z{2%}rf)-~u8uJ;9SqnL>{s1{L=c%nX~vi=xJMX@NX
z)yTzISVYEM@)-Iob%^QkRU#E`V`1@BG~|Wl_EE92p;5dSUCu5Soeh^MtXb@9u%y#w
z@py?j*<g8q2*1_DyrRoWv+W4_%kB79tQo?(HXv9VX_v8O@uSKl6@C`oK<s$DF{0d9
z{cmme>mSZtdvEu@j=u5yv0qW(*{^N8uCDj32fV}r(I94EUW+kase%SnVl0U7n`qFB
zKXro<$dy4k5`>c8;nJiulH<Xkm}?H@Ed>Js=|t}U3K%zi&}YG(_rn(-#26ox6DUDH
z6!@SEMI%AK<<U0=13_<%zL`a^etoO-V-T_wm6Wk)1XSOSAV^R9;Zz^iGjc%R=+bl^
zvoSfE@sS51OZ}jJhfl^IjcoO&f_N(RR=q<{>OJ^%xOAmH%m?9XWi+C1bf?$1x?NCD
z2lPxJpx@#CU=iSh(MSOQN_{i}A;1dvf;iBcN|yeUKwcn+(N5yq?j&BUn??sl&wTHL
z=~s^aWBAB}KYsuB_l-Z~6K}WX^?k)Z+o4QHC2t*WG?hauj(@jpHxqrVN(5tB-bXYb
feHNd0S0DdQ%=VVydtJWn=T;o&T_3T-DgXZf6xhmD

diff --git a/client/tests/kvm/tests/whql_submission.py b/client/tests/kvm/tests/whql_submission.py
index 8dd48b8..a0ff87c 100644
--- a/client/tests/kvm/tests/whql_submission.py
+++ b/client/tests/kvm/tests/whql_submission.py
@@ -6,20 +6,24 @@ import kvm_subprocess, kvm_test_utils, kvm_utils, rss_file_transfer
 def run_whql_submission(test, params, env):
     """
     WHQL submission test:
-    1) Log into the guest (the client machine) and into a DTM server machine
+    1) Log into the client machines and into a DTM server machine
     2) Copy the automation program binary (dsso_test_binary) to the server machine
     3) Run the automation program
     4) Pass the program all relevant parameters (e.g. device_data)
     5) Wait for the program to terminate
     6) Parse and report job results
-    (logs and HTML reports are placed in test.bindir)
+    (logs and HTML reports are placed in test.debugdir)
 
     @param test: kvm test object
     @param params: Dictionary with the test parameters
     @param env: Dictionary with test environment.
     """
-    vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
-    session = kvm_test_utils.wait_for_login(vm, 0, 240)
+    # Log into all client VMs
+    vms = []
+    sessions = []
+    for vm_name in kvm_utils.get_sub_dict_names(params, "vms"):
+        vms.append(kvm_test_utils.get_living_vm(env, vm_name))
+        sessions.append(kvm_test_utils.wait_for_login(vms[-1], 0, 240))
 
     # Collect parameters
     server_address = params.get("server_address")
@@ -30,47 +34,54 @@ def run_whql_submission(test, params, env):
     dsso_test_binary = params.get("dsso_test_binary",
                                   "deps/whql_submission_15.exe")
     dsso_test_binary = kvm_utils.get_path(test.bindir, dsso_test_binary)
-    test_device = params.get("test_device")
-    job_filter = params.get("job_filter", ".*")
+    dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
+                                            "deps/whql_delete_machine_15.exe")
+    dsso_delete_machine_binary = kvm_utils.get_path(test.bindir,
+                                                    dsso_delete_machine_binary)
     test_timeout = float(params.get("test_timeout", 600))
-    wtt_services = params.get("wtt_services")
 
-    # Restart WTT service(s) on the client
-    logging.info("Restarting WTT services on client")
-    for svc in wtt_services.split():
-        kvm_test_utils.stop_windows_service(session, svc)
-    for svc in wtt_services.split():
-        kvm_test_utils.start_windows_service(session, svc)
-
-    # Run whql_pre_command
-    if params.get("whql_pre_command"):
-        session.cmd(params.get("whql_pre_command"),
-                    int(params.get("whql_pre_command_timeout", 600)))
-
-    # Copy dsso_test_binary to the server
-    rss_file_transfer.upload(server_address, server_file_transfer_port,
-                             dsso_test_binary, server_studio_path, timeout=60)
+    # Copy dsso binaries to the server
+    for filename in dsso_test_binary, dsso_delete_machine_binary:
+        rss_file_transfer.upload(server_address, server_file_transfer_port,
+                                 filename, server_studio_path, timeout=60)
 
     # Open a shell session with the server
     server_session = kvm_utils.remote_login("nc", server_address,
                                             server_shell_port, "", "",
-                                            session.prompt, session.linesep)
-    server_session.set_status_test_command(session.status_test_command)
+                                            sessions[0].prompt,
+                                            sessions[0].linesep)
+    server_session.set_status_test_command(sessions[0].status_test_command)
 
-    # Get the computer names of the server and client
+    # Get the computer names of the server and clients
     cmd = "echo %computername%"
     server_name = server_session.cmd_output(cmd).strip()
-    client_name = session.cmd_output(cmd).strip()
-    session.close()
+    client_names = [session.cmd_output(cmd).strip() for session in sessions]
 
-    # Run the automation program on the server
+    # Delete all client machines from the server's data store
     server_session.cmd("cd %s" % server_studio_path)
+    for client_name in client_names:
+        cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
+                            server_name, client_name)
+        server_session.cmd(cmd, print_func=logging.debug)
+
+    # Reboot the client machines
+    sessions = kvm_utils.parallel((kvm_test_utils.reboot, (vm, session))
+                                  for vm, session in zip(vms, sessions))
+
+    # Run whql_pre_command and close the sessions
+    if params.get("whql_pre_command"):
+        for session in sessions:
+            session.cmd(params.get("whql_pre_command"),
+                        int(params.get("whql_pre_command_timeout", 600)))
+            session.close()
+
+    # Run the automation program on the server
     cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
                                  server_name,
-                                 client_name,
-                                 "%s_pool" % client_name,
-                                 "%s_submission" % client_name,
-                                 test_timeout)
+                                 "%s_pool" % client_names[0],
+                                 "%s_submission" % client_names[0],
+                                 test_timeout,
+                                 " ".join(client_names))
     server_session.sendline(cmd)
 
     # Helper function: wait for a given prompt and raise an exception if an
@@ -89,13 +100,13 @@ def run_whql_submission(test, params, env):
 
     # Tell the automation program which device to test
     find_prompt("Device to test:")
-    server_session.sendline(test_device)
+    server_session.sendline(params.get("test_device"))
 
     # Tell the automation program which jobs to run
     find_prompt("Jobs to run:")
-    server_session.sendline(job_filter)
+    server_session.sendline(params.get("job_filter", ".*"))
 
-    # Give the automation program all the device data supplied by the user
+    # Set submission DeviceData
     find_prompt("DeviceData name:")
     for dd in kvm_utils.get_sub_dict_names(params, "device_data"):
         dd_params = kvm_utils.get_sub_dict(params, dd)
@@ -104,8 +115,7 @@ def run_whql_submission(test, params, env):
             server_session.sendline(dd_params.get("dd_data"))
     server_session.sendline()
 
-    # Give the automation program all the descriptor information supplied by
-    # the user
+    # Set submission descriptors
     find_prompt("Descriptor path:")
     for desc in kvm_utils.get_sub_dict_names(params, "descriptors"):
         desc_params = kvm_utils.get_sub_dict(params, desc)
@@ -113,6 +123,31 @@ def run_whql_submission(test, params, env):
             server_session.sendline(desc_params.get("desc_path"))
     server_session.sendline()
 
+    # Set machine dimensions for each client machine
+    for vm_name in kvm_utils.get_sub_dict_names(params, "vms"):
+        vm_params = kvm_utils.get_sub_dict(params, vm_name)
+        find_prompt(r"Dimension name\b.*:")
+        for dp in kvm_utils.get_sub_dict_names(vm_params, "dimensions"):
+            dp_params = kvm_utils.get_sub_dict(vm_params, dp)
+            if dp_params.get("dim_name") and dp_params.get("dim_value"):
+                server_session.sendline(dp_params.get("dim_name"))
+                server_session.sendline(dp_params.get("dim_value"))
+        server_session.sendline()
+
+    # Set extra parameters for tests that require them (e.g. NDISTest)
+    for vm_name in kvm_utils.get_sub_dict_names(params, "vms"):
+        vm_params = kvm_utils.get_sub_dict(params, vm_name)
+        find_prompt(r"Parameter name\b.*:")
+        for dp in kvm_utils.get_sub_dict_names(vm_params, "device_params"):
+            dp_params = kvm_utils.get_sub_dict(vm_params, dp)
+            if dp_params.get("dp_name") and dp_params.get("dp_regex"):
+                server_session.sendline(dp_params.get("dp_name"))
+                server_session.sendline(dp_params.get("dp_regex"))
+                # Make sure the prompt appears again (if the device isn't found
+                # the automation program will terminate)
+                find_prompt(r"Parameter name\b.*:")
+        server_session.sendline()
+
     # Wait for the automation program to terminate
     try:
         o = server_session.read_up_to_prompt(print_func=logging.info,
@@ -162,38 +197,63 @@ def run_whql_submission(test, params, env):
                 except (KeyError, OSError):
                     pass
 
-    # Print result summary
-    logging.info("")
-    logging.info("Result summary:")
-    name_length = max(len(r.get("job", "")) for r in results)
-    fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
-    logging.info(fmt % ("ID", "Job", "Status", "Pass", "Fail", "NotRun",
-                        "NotApplicable"))
-    logging.info(fmt % ("--", "---", "------", "----", "----", "------",
-                        "-------------"))
-    for r in results:
-        logging.info(fmt % (r.get("id"), r.get("job"), r.get("status"),
-                            r.get("pass"), r.get("fail"), r.get("notrun"),
-                            r.get("notapplicable")))
-    logging.info("(see logs and HTML reports in %s)" % test.debugdir)
-
-    # Kill the VM and fail if the automation program did not terminate on time
+    # Print result summary (both to the regular logs and to a file named
+    # 'summary' in test.debugdir)
+    def print_summary_line(f, line):
+        logging.info(line)
+        f.write(line + "\n")
+    if results:
+        # Make sure all results have the required keys
+        for r in results:
+            r["id"] = str(r.get("id"))
+            r["job"] = str(r.get("job"))
+            r["status"] = str(r.get("status"))
+            r["pass"] = int(r.get("pass", 0))
+            r["fail"] = int(r.get("fail", 0))
+            r["notrun"] = int(r.get("notrun", 0))
+            r["notapplicable"] = int(r.get("notapplicable", 0))
+        # Sort the results by failures and total test count in descending order
+        results = [(r["fail"],
+                    r["pass"] + r["fail"] + r["notrun"] + r["notapplicable"],
+                    r) for r in results]
+        results.sort(reverse=True)
+        results = [r[-1] for r in results]
+        # Print results
+        logging.info("")
+        logging.info("Result summary:")
+        name_length = max(len(r["job"]) for r in results)
+        fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
+        f = open(os.path.join(test.debugdir, "summary"), "w")
+        print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail",
+                                     "NotRun", "NotApplicable"))
+        print_summary_line(f, fmt % ("--", "---", "------", "----", "----",
+                                     "------", "-------------"))
+        for r in results:
+            print_summary_line(f, fmt % (r["id"], r["job"], r["status"],
+                                         r["pass"], r["fail"], r["notrun"],
+                                         r["notapplicable"]))
+        f.close()
+        logging.info("(see logs and HTML reports in %s)" % test.debugdir)
+
+    # Kill the client VMs and fail if the automation program did not terminate
+    # on time
     if not done:
-        vm.destroy()
+        kvm_utils.parallel(vm.destroy for vm in vms)
         raise error.TestFail("The automation program did not terminate "
                              "on time")
 
-    # Fail if there are failed or incomplete jobs (kill the VM if there are
-    # incomplete jobs)
-    failed_jobs = [r.get("job") for r in results
-                   if r.get("status", "").lower() == "investigate"]
-    running_jobs = [r.get("job") for r in results
-                    if r.get("status", "").lower() == "inprogress"]
+    # Fail if there are failed or incomplete jobs (kill the client VMs if there
+    # are incomplete jobs)
+    failed_jobs = [r["job"] for r in results
+                   if r["status"].lower() == "investigate"]
+    running_jobs = [r["job"] for r in results
+                    if r["status"].lower() == "inprogress"]
     errors = []
     if failed_jobs:
         errors += ["Jobs failed: %s." % failed_jobs]
     if running_jobs:
-        vm.destroy()
+        for vm in vms:
+            vm.destroy()
         errors += ["Jobs did not complete on time: %s." % running_jobs]
     if errors:
         raise error.TestFail(" ".join(errors))
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index 8d533ba..e81d879 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -347,11 +347,12 @@ variants:
         server_shell_port = 10022
         server_file_transfer_port = 10023
         server_studio_path = %programfiles%\Microsoft Driver Test Manager\Studio
+        dsso_test_binary = deps/whql_submission_15.exe
+        dsso_delete_machine_binary = deps/whql_delete_machine_15.exe
         wtt_services = wttsvc
         variants:
             - client_install:
                 type = whql_client_install
-                dsso_delete_machine_binary = deps/whql_delete_machine_15.exe
                 # The username and password are required for accessing the DTM client
                 # installer binary shared by the server
                 server_username = administrator
@@ -363,7 +364,7 @@ variants:
             - submission:    client_install
                 type = whql_submission
                 extra_params += " -snapshot"
-                dsso_test_binary = deps/whql_submission_15.exe
+                restart_vm = yes
                 test_timeout = 3600
                 device_data = cat0 cat1 cat2 cat3 logoarch logoos whqlos whqlqual prog desc filter virt
                 descriptors = desc1 desc2 desc3
-- 
1.7.3.3

--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at  http://vger.kernel.org/majordomo-info.html


[Index of Archives]     [KVM ARM]     [KVM ia64]     [KVM ppc]     [Virtualization Tools]     [Spice Development]     [Libvirt]     [Libvirt Users]     [Linux USB Devel]     [Linux Audio Users]     [Yosemite Questions]     [Linux Kernel]     [Linux SCSI]     [XFree86]
  Powered by Linux