- deps/whql_submission_15.cs:
* Handle multiple machines instead of one.
* Split the code into smaller functions.
* Get job parameters from the user: each parameter is a regex that should
match a device name in a machine, and is set using GetParameterByName().
Note that job parameters differ from DeviceData objects and descriptors.
* Get machine dimensions from the user and set them for each machine.
* Set the WDKSubmissionId dimension for all machines (it seems to be
required by some or all submissions).
* Instead of AddDeviceJob() use submission.ProcessJobs() and set the .Device
attribute of the ScheduleItem objects returned. This is required for
NDISTest to work.
* Don't set status to 'Unsafe' before resetting machines (unnecessary).
* If the machine pool already exists, delete it and create a new one.
- tests/whql_submission.py:
* Use all vms instead of just main_vm.
* Pass job parameters for each machine to the automation program.
* Pass machine dimensions for each machine.
* Write result summary to a file named 'summary' in addition to the regular
logs.
* Sort results in the result summary by failures and total test count in
descending order.
* Before starting the test, delete the machines from the data store, reboot
them and wait for them to reappear in the data store.
* Don't restart any services on the client machines (not necessary).
- Set 'restart_vm' to 'yes' in tests_base.cfg.sample to force a restart of the
VMs before each test.
Signed-off-by: Michael Goldish <mgoldish@xxxxxxxxxx>
---
client/tests/kvm/deps/whql_submission_15.cs | 662 +++++++++++++++-----------
client/tests/kvm/deps/whql_submission_15.exe | Bin 10240 -> 12288 bytes
client/tests/kvm/tests/whql_submission.py | 182 +++++---
client/tests/kvm/tests_base.cfg.sample | 5 +-
4 files changed, 497 insertions(+), 352 deletions(-)
diff --git a/client/tests/kvm/deps/whql_submission_15.cs b/client/tests/kvm/deps/whql_submission_15.cs
index 8fa6856..0928548 100644
--- a/client/tests/kvm/deps/whql_submission_15.cs
+++ b/client/tests/kvm/deps/whql_submission_15.cs
@@ -1,289 +1,373 @@
-// DTM submission automation program
-// Author: Michael Goldish <mgoldish@xxxxxxxxxx>
-// Based on sample code by Microsoft.
-
-// Note: this program has only been tested with DTM version 1.5.
-// It might fail to work with other versions, specifically because it uses
-// a few undocumented methods/attributes.
-
-using System;
-using System.Collections.Generic;
-using System.Text.RegularExpressions;
-using Microsoft.DistributedAutomation.DeviceSelection;
-using Microsoft.DistributedAutomation.SqlDataStore;
-
-namespace automate0
-{
- class AutoJob
- {
- static int Main(string[] args)
- {
- if (args.Length != 5)
- {
- Console.WriteLine("Error: incorrect number of command line arguments");
- Console.WriteLine("Usage: {0} serverName clientName machinePoolName submissionName timeout",
- System.Environment.GetCommandLineArgs()[0]);
- return 1;
- }
- string serverName = args[0];
- string clientName = args[1];
- string machinePoolName = args[2];
- string submissionName = args[3];
- double timeout = Convert.ToDouble(args[4]);
-
- try
- {
- // Initialize DeviceScript and connect to data store
- Console.WriteLine("Initializing DeviceScript object");
- DeviceScript script = new DeviceScript();
- Console.WriteLine("Connecting to data store");
-
- script.ConnectToNamedDataStore(serverName);
-
- // Find client machine
- IResourcePool rootPool = script.GetResourcePoolByName("$");
- Console.WriteLine("Looking for client machine '{0}'", clientName);
- IResource machine = null;
- while (true)
- {
- try
- {
- machine = rootPool.GetResourceByName(clientName);
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
- // Make sure the machine is valid
- if (machine != null &&
- machine.OperatingSystem != null &&
- machine.OperatingSystem.Length > 0 &&
- machine.ProcessorArchitecture != null &&
- machine.ProcessorArchitecture.Length > 0 &&
- machine.GetDevices().Length > 0)
- break;
- System.Threading.Thread.Sleep(1000);
- }
- Console.WriteLine("Client machine '{0}' found ({1}, {2})",
- clientName, machine.OperatingSystem, machine.ProcessorArchitecture);
-
- // Create machine pool and add client machine to it
- // (this must be done because jobs cannot be scheduled for machines in the
- // default pool)
- try
- {
- script.CreateResourcePool(machinePoolName, rootPool.ResourcePoolId);
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
- IResourcePool newPool = script.GetResourcePoolByName(machinePoolName);
- Console.WriteLine("Moving the client machine to pool '{0}'", machinePoolName);
- machine.ChangeResourcePool(newPool);
-
- // Reset client machine
- if (machine.Status != "Ready")
- {
- Console.WriteLine("Changing the client machine's status to 'Reset'");
- while (true)
- {
- try
- {
- machine = rootPool.GetResourceByName(clientName);
- machine.ChangeResourceStatus("Unsafe");
- System.Threading.Thread.Sleep(5000);
- machine.ChangeResourceStatus("Reset");
- break;
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
- System.Threading.Thread.Sleep(5000);
- }
- Console.WriteLine("Waiting for client machine to be ready");
- while (machine.Status != "Ready")
- {
- try
- {
- machine = rootPool.GetResourceByName(clientName);
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
- System.Threading.Thread.Sleep(1000);
- }
- }
- Console.WriteLine("Client machine is ready");
-
- // Get requested device regex and look for a matching device
- Console.WriteLine("Device to test: ");
- Regex deviceRegex = new Regex(Console.ReadLine(), RegexOptions.IgnoreCase);
- Console.WriteLine("Looking for device '{0}'", deviceRegex);
- IDevice device;
- DateTime endTime = DateTime.Now.AddSeconds(120);
- while (DateTime.Now < endTime)
- {
- machine = rootPool.GetResourceByName(clientName);
- Console.WriteLine("(Client machine has {0} devices)", machine.GetDevices().Length);
- foreach (IDevice d in machine.GetDevices())
- {
- if (deviceRegex.IsMatch(d.FriendlyName))
- {
- device = d;
- goto deviceFound;
- }
- }
- System.Threading.Thread.Sleep(5000);
- }
- Console.WriteLine("Error: device '{0}' not found", deviceRegex);
- return 1;
-
- deviceFound:
- Console.WriteLine("Found device '{0}'", device.FriendlyName);
-
- // Get requested jobs regex
- Console.WriteLine("Jobs to run: ");
- Regex jobRegex = new Regex(Console.ReadLine(), RegexOptions.IgnoreCase);
-
- // Create submission
- Object[] existingSubmissions = script.GetSubmissionByName(submissionName);
- if (existingSubmissions.Length > 0)
- {
- Console.WriteLine("Submission '{0}' already exists -- removing it",
- submissionName);
- script.DeleteSubmission(((ISubmission)existingSubmissions[0]).Id);
- }
- Console.WriteLine("Creating submission '{0}'", submissionName);
- ISubmission submission = script.CreateHardwareSubmission(submissionName,
- newPool.ResourcePoolId, device.InstanceId);
-
- // Get DeviceData objects from the user
- List<Object> deviceDataList = new List<Object>();
- while (true)
- {
- ISubmissionDeviceData dd = script.CreateNewSubmissionDeviceData();
- Console.WriteLine("DeviceData name: ");
- dd.Name = Console.ReadLine();
- if (dd.Name.Length == 0)
- break;
- Console.WriteLine("DeviceData data: ");
- dd.Data = Console.ReadLine();
- deviceDataList.Add(dd);
- }
-
- // Set the submission's DeviceData
- submission.SetDeviceData(deviceDataList.ToArray());
-
- // Get descriptors from the user
- List<Object> descriptorList = new List<Object>();
- while (true)
- {
- Console.WriteLine("Descriptor path: ");
- string descriptorPath = Console.ReadLine();
- if (descriptorPath.Length == 0)
- break;
- descriptorList.Add(script.GetDescriptorByPath(descriptorPath));
- }
-
- // Set the submission's descriptors
- submission.SetLogoDescriptors(descriptorList.ToArray());
-
- // Create a schedule
- ISchedule schedule = script.CreateNewSchedule();
- Console.WriteLine("Scheduling jobs:");
- int jobCount = 0;
- foreach (IJob j in submission.GetJobs())
- {
- if (jobRegex.IsMatch(j.Name))
- {
- Console.WriteLine(" " + j.Name);
- schedule.AddDeviceJob(device, j);
- jobCount++;
- }
- }
- if (jobCount == 0)
- {
- Console.WriteLine("Error: no submission jobs match pattern '{0}'", jobRegex);
- return 1;
- }
- schedule.AddSubmission(submission);
- schedule.SetResourcePool(newPool);
- script.RunSchedule(schedule);
-
- // Wait for jobs to complete
- Console.WriteLine("Waiting for all jobs to complete (timeout={0})", timeout);
- endTime = DateTime.Now.AddSeconds(timeout);
- int numCompleted = 0, numFailed = 0;
- while (numCompleted < submission.GetResults().Length && DateTime.Now < endTime)
- {
- // Sleep for 30 seconds
- System.Threading.Thread.Sleep(30000);
- // Count completed submission jobs
- numCompleted = 0;
- foreach (IResult r in submission.GetResults())
- if (r.ResultStatus != "InProgress")
- numCompleted++;
- // Report results in a Python readable format and count failed schedule jobs
- // (submission jobs are a subset of schedule jobs)
- Console.WriteLine();
- Console.WriteLine("---- [");
- numFailed = 0;
- foreach (IResult r in schedule.GetResults())
- {
- Console.WriteLine(" {");
- Console.WriteLine(" 'id': {0}, 'job': r'''{1}''',", r.Job.Id, r.Job.Name);
- Console.WriteLine(" 'logs': r'''{0}''',", r.LogLocation);
- if (r.ResultStatus != "InProgress")
- Console.WriteLine(" 'report': r'''{0}''',",
- submission.GetSubmissionResultReport(r));
- Console.WriteLine(" 'status': '{0}',", r.ResultStatus);
- Console.WriteLine(" 'pass': {0}, 'fail': {1}, 'notrun': {2}, 'notapplicable': {3}",
- r.Pass, r.Fail, r.NotRun, r.NotApplicable);
- Console.WriteLine(" },");
- numFailed += r.Fail;
- }
- Console.WriteLine("] ----");
- }
- Console.WriteLine();
-
- // Cancel incomplete jobs
- foreach (IResult r in schedule.GetResults())
- if (r.ResultStatus == "InProgress")
- r.Cancel();
-
- // Set the machine's status to Unsafe and then Reset
- try
- {
- machine = rootPool.GetResourceByName(clientName);
- machine.ChangeResourceStatus("Unsafe");
- System.Threading.Thread.Sleep(5000);
- machine.ChangeResourceStatus("Reset");
- }
- catch (Exception e)
- {
- Console.WriteLine("Warning: " + e.Message);
- }
-
- // Report failures
- if (numCompleted < submission.GetResults().Length)
- Console.WriteLine("Some jobs did not complete on time.");
- if (numFailed > 0)
- Console.WriteLine("Some jobs failed.");
-
- if (numFailed > 0 || numCompleted < submission.GetResults().Length)
- return 1;
-
- Console.WriteLine("All jobs completed.");
- return 0;
- }
- catch (Exception e)
- {
- Console.WriteLine("Error: " + e.Message);
- return 1;
- }
- }
- }
-}
+// DTM submission automation program
+// Author: Michael Goldish <mgoldish@xxxxxxxxxx>
+// Based on sample code by Microsoft.
+
+// Note: this program has only been tested with DTM version 1.5.
+// It might fail to work with other versions, specifically because it uses
+// a few undocumented methods/attributes.
+
+using System;
+using System.Collections.Generic;
+using System.Text.RegularExpressions;
+using Microsoft.DistributedAutomation.DeviceSelection;
+using Microsoft.DistributedAutomation.SqlDataStore;
+
+namespace automate0
+{
+ class AutoJob
+ {
+ // Wait for a machine to show up in the data store
+ static void FindMachine(IResourcePool rootPool, string machineName)
+ {
+ Console.WriteLine("Looking for machine '{0}'", machineName);
+ IResource machine = null;
+ while (true)
+ {
+ try
+ {
+ machine = rootPool.GetResourceByName(machineName);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ }
+ // Make sure the machine is valid
+ if (machine != null &&
+ machine.OperatingSystem != null &&
+ machine.OperatingSystem.Length > 0 &&
+ machine.ProcessorArchitecture != null &&
+ machine.ProcessorArchitecture.Length > 0 &&
+ machine.GetDevices().Length > 0)
+ break;
+ System.Threading.Thread.Sleep(1000);
+ }
+ Console.WriteLine("Client machine '{0}' found ({1}, {2})",
+ machineName, machine.OperatingSystem, machine.ProcessorArchitecture);
+ }
+
+ // Delete a machine pool if it exists
+ static void DeleteResourcePool(IDeviceScript script, string poolName)
+ {
+ while (true)
+ {
+ try
+ {
+ IResourcePool pool = script.GetResourcePoolByName(poolName);
+ if (pool != null)
+ script.DeleteResourcePool(pool);
+ break;
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ System.Threading.Thread.Sleep(1000);
+ }
+ }
+ }
+
+ // Set the machine's status to 'Reset' and optionally wait for it to become ready
+ static void ResetMachine(IResourcePool rootPool, string machineName, bool wait)
+ {
+ Console.WriteLine("Resetting machine '{0}'", machineName);
+ IResource machine;
+ while (true)
+ {
+ try
+ {
+ machine = rootPool.GetResourceByName(machineName);
+ machine.ChangeResourceStatus("Reset");
+ break;
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ System.Threading.Thread.Sleep(5000);
+ }
+ }
+ if (wait)
+ {
+ Console.WriteLine("Waiting for machine '{0}' to be ready", machineName);
+ while (machine.Status != "Ready")
+ {
+ try
+ {
+ machine = rootPool.GetResourceByName(machineName);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ }
+ System.Threading.Thread.Sleep(1000);
+ }
+ Console.WriteLine("Machine '{0}' is ready", machineName);
+ }
+ }
+
+ // Look for a device in a machine, and if not found, keep trying for 3 minutes
+ static IDevice GetDevice(IResourcePool rootPool, string machineName, string regexStr)
+ {
+ Regex deviceRegex = new Regex(regexStr, RegexOptions.IgnoreCase);
+ int numAttempts = 1;
+ DateTime endTime = DateTime.Now.AddSeconds(180);
+ while (DateTime.Now < endTime)
+ {
+ IResource machine = rootPool.GetResourceByName(machineName);
+ Console.WriteLine("Looking for device '{0}' in machine '{1}' (machine has {2} devices)",
+ regexStr, machineName, machine.GetDevices().Length);
+ foreach (IDevice d in machine.GetDevices())
+ {
+ if (deviceRegex.IsMatch(d.FriendlyName))
+ {
+ Console.WriteLine("Found device '{0}'", d.FriendlyName);
+ return d;
+ }
+ }
+ Console.WriteLine("Device not found");
+ if (numAttempts % 5 == 0)
+ ResetMachine(rootPool, machineName, true);
+ else
+ System.Threading.Thread.Sleep(5000);
+ numAttempts++;
+ }
+ Console.WriteLine("Error: device '{0}' not found", deviceRegex);
+ return null;
+ }
+
+ static int Main(string[] args)
+ {
+ if (args.Length < 5)
+ {
+ Console.WriteLine("Error: incorrect number of command line arguments");
+ Console.WriteLine("Usage: {0} serverName machinePoolName submissionName timeout machineName0 machineName1 ...",
+ System.Environment.GetCommandLineArgs()[0]);
+ return 1;
+ }
+ string serverName = args[0];
+ string machinePoolName = args[1];
+ string submissionName = args[2];
+ double timeout = Convert.ToDouble(args[3]);
+
+ List<string> machines = new List<string>();
+ for (int i = 4; i < args.Length; i++)
+ machines.Add(args[i]);
+
+ try
+ {
+ // Initialize DeviceScript and connect to data store
+ Console.WriteLine("Initializing DeviceScript object");
+ DeviceScript script = new DeviceScript();
+ Console.WriteLine("Connecting to data store");
+ script.ConnectToNamedDataStore(serverName);
+
+ // Wait for client machines to become available
+ IResourcePool rootPool = script.GetResourcePoolByName("$");
+ foreach (string machineName in machines)
+ FindMachine(rootPool, machineName);
+
+ // Delete the machine pool if it already exists
+ DeleteResourcePool(script, machinePoolName);
+
+ // Create the machine pool and add the client machines to it
+ // (this must be done because jobs cannot be scheduled for machines in the
+ // default pool)
+ try
+ {
+ script.CreateResourcePool(machinePoolName, rootPool.ResourcePoolId);
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Warning: " + e.Message);
+ }
+ IResourcePool newPool = script.GetResourcePoolByName(machinePoolName);
+ foreach (string machineName in machines)
+ {
+ Console.WriteLine("Moving machine '{0}' to pool '{1}'", machineName, machinePoolName);
+ rootPool.GetResourceByName(machineName).ChangeResourcePool(newPool);
+ }
+
+ // Reset client machine
+ foreach (string machineName in machines)
+ ResetMachine(rootPool, machineName, true);
+
+ // Get requested device regex and look for a matching device in the first machine
+ Console.WriteLine("Device to test:");
+ IDevice device = GetDevice(rootPool, machines[0], Console.ReadLine());
+ if (device == null)
+ return 1;
+
+ // Get requested jobs regex
+ Console.WriteLine("Jobs to run:");
+ Regex jobRegex = new Regex(Console.ReadLine(), RegexOptions.IgnoreCase);
+
+ // Create a submission
+ Object[] existingSubmissions = script.GetSubmissionByName(submissionName);
+ if (existingSubmissions.Length > 0)
+ {
+ Console.WriteLine("Submission '{0}' already exists -- removing it",
+ submissionName);
+ script.DeleteSubmission(((ISubmission)existingSubmissions[0]).Id);
+ }
+ string hardwareId = device.InstanceId.Remove(device.InstanceId.LastIndexOf("\\"));
+ Console.WriteLine("Creating submission '{0}' (hardware ID: {1})", submissionName, hardwareId);
+ ISubmission submission = script.CreateHardwareSubmission(submissionName, newPool.ResourcePoolId, hardwareId);
+
+ // Set submission DeviceData
+ List<Object> deviceDataList = new List<Object>();
+ while (true)
+ {
+ ISubmissionDeviceData dd = script.CreateNewSubmissionDeviceData();
+ Console.WriteLine("DeviceData name:");
+ dd.Name = Console.ReadLine();
+ if (dd.Name.Length == 0)
+ break;
+ Console.WriteLine("DeviceData data:");
+ dd.Data = Console.ReadLine();
+ deviceDataList.Add(dd);
+ }
+ submission.SetDeviceData(deviceDataList.ToArray());
+
+ // Set submission descriptors
+ List<Object> descriptorList = new List<Object>();
+ while (true)
+ {
+ Console.WriteLine("Descriptor path:");
+ string descriptorPath = Console.ReadLine();
+ if (descriptorPath.Length == 0)
+ break;
+ descriptorList.Add(script.GetDescriptorByPath(descriptorPath));
+ }
+ submission.SetLogoDescriptors(descriptorList.ToArray());
+
+ // Set machine dimensions
+ foreach (string machineName in machines)
+ {
+ IResource machine = rootPool.GetResourceByName(machineName);
+ while (true)
+ {
+ Console.WriteLine("Dimension name ({0}):", machineName);
+ string dimName = Console.ReadLine();
+ if (dimName.Length == 0)
+ break;
+ Console.WriteLine("Dimension value ({0}):", machineName);
+ machine.SetDimension(dimName, Console.ReadLine());
+ }
+ // Set the WDKSubmissionId dimension for all machines
+ machine.SetDimension("WDKSubmissionId", submission.Id.ToString() + "_" + submission.Name);
+ }
+
+ // Get job parameters
+ List<string> paramNames = new List<string>();
+ List<string> paramValues = new List<string>();
+ foreach (string machineName in machines)
+ {
+ while (true)
+ {
+ Console.WriteLine("Parameter name ({0}):", machineName);
+ string paramName = Console.ReadLine();
+ if (paramName.Length == 0)
+ break;
+ Console.WriteLine("Device regex ({0}):", machineName);
+ IDevice d = GetDevice(rootPool, machineName, Console.ReadLine());
+ if (d == null)
+ return 1;
+ string deviceName = d.GetAttribute("name")[0].ToString();
+ Console.WriteLine("Setting parameter value to '{0}'", deviceName);
+ paramNames.Add(paramName);
+ paramValues.Add(deviceName);
+ }
+ }
+
+ // Find jobs that match the requested pattern
+ Console.WriteLine("Scheduling jobs:");
+ List<IJob> jobs = new List<IJob>();
+ foreach (IJob j in submission.GetJobs())
+ {
+ if (jobRegex.IsMatch(j.Name))
+ {
+ Console.WriteLine(" " + j.Name);
+ // Set job parameters
+ for (int i = 0; i < paramNames.Count; i++)
+ {
+ IParameter p = j.GetParameterByName(paramNames[i]);
+ if (p != null)
+ p.ScheduleValue = paramValues[i];
+ }
+ jobs.Add(j);
+ }
+ }
+ if (jobs.Count == 0)
+ {
+ Console.WriteLine("Error: no submission jobs match pattern '{0}'", jobRegex);
+ return 1;
+ }
+
+ // Create a schedule, add jobs to it and run it
+ ISchedule schedule = script.CreateNewSchedule();
+ foreach (IScheduleItem item in submission.ProcessJobs(jobs.ToArray()))
+ {
+ item.Device = device;
+ schedule.AddScheduleItem(item);
+ }
+ schedule.AddSubmission(submission);
+ schedule.SetResourcePool(newPool);
+ script.RunSchedule(schedule);
+
+ // Wait for jobs to complete
+ Console.WriteLine("Waiting for all jobs to complete (timeout={0}s)", timeout);
+ DateTime endTime = DateTime.Now.AddSeconds(timeout);
+ int numCompleted, numFailed;
+ do
+ {
+ System.Threading.Thread.Sleep(30000);
+ // Report results in a Python readable format and count completed and failed schedule jobs
+ numCompleted = numFailed = 0;
+ Console.WriteLine();
+ Console.WriteLine("---- [");
+ foreach (IResult r in schedule.GetResults())
+ {
+ if (r.ResultStatus != "InProgress") numCompleted++;
+ if (r.ResultStatus == "Investigate") numFailed++;
+ Console.WriteLine(" {");
+ Console.WriteLine(" 'id': {0}, 'job': r'''{1}''',", r.Job.Id, r.Job.Name);
+ Console.WriteLine(" 'logs': r'''{0}''',", r.LogLocation);
+ if (r.ResultStatus != "InProgress")
+ Console.WriteLine(" 'report': r'''{0}''',",
+ submission.GetSubmissionResultReport(r));
+ Console.WriteLine(" 'status': '{0}',", r.ResultStatus);
+ Console.WriteLine(" 'pass': {0}, 'fail': {1}, 'notrun': {2}, 'notapplicable': {3}",
+ r.Pass, r.Fail, r.NotRun, r.NotApplicable);
+ Console.WriteLine(" },");
+ }
+ Console.WriteLine("] ----");
+ } while (numCompleted < schedule.GetResults().Length && DateTime.Now < endTime);
+
+ Console.WriteLine();
+
+ // Cancel incomplete jobs
+ foreach (IResult r in schedule.GetResults())
+ if (r.ResultStatus == "InProgress")
+ r.Cancel();
+
+ // Reset the machines
+ foreach (string machineName in machines)
+ ResetMachine(rootPool, machineName, false);
+
+ // Report failures
+ if (numCompleted < schedule.GetResults().Length)
+ Console.WriteLine("Some jobs did not complete on time.");
+ if (numFailed > 0)
+ Console.WriteLine("Some jobs failed.");
+ if (numFailed > 0 || numCompleted < schedule.GetResults().Length)
+ return 1;
+
+ Console.WriteLine("All jobs completed.");
+ return 0;
+ }
+ catch (Exception e)
+ {
+ Console.WriteLine("Error: " + e.Message);
+ return 1;
+ }
+ }
+ }
+}
diff --git a/client/tests/kvm/deps/whql_submission_15.exe b/client/tests/kvm/deps/whql_submission_15.exe
index 4f30aa801c8f200bd96b0de6c10d2b59c2bad268..605e2e3aa1f0761155f4aac5191e3ad0f0ddecc6 100644
GIT binary patch
literal 12288
zcmeHNdvIJ=dH?P{u2w5c$=S8_uq69RerRRAWhb#6JFzU;i5$sREZN3H#$N4SOPlQO
zQSRN9r6d#)+JakS$V^B<lTI?7{LyqKEoG7cTA+l&l(eC3Iy@R41{!Ds45b~)s|C{E
zch0@qUD*!#qyIHmy65qI-}zqWJKs5bcE{iPehLy%2=8mJ5q%6bKL=F#`o%KPb({a9
zj-IXlLfglTu`jeu&1GC`-jTCTGH0cd`MfMxGq&Xv^HwHrjU1b_ax!iA*49S1daNf7
z5seu^YX8Yk#(ZhtqJU9jM2NNnf{LF{ji9jbp2N%9S*QY9Z?u&DT+@?y0p}-32Y!d>
zM)hB+y{;yqH29w4_Bo<|W+$wDmWkE@_UtfG%Uay)^{rLa6;%ZNJIm;M3-)3G{m-%^
zfK#@5dZVWv3tII$u9HGf>2?kc6K?`<*>eC6>2>U^1R?86%XpbrFW$1}0MWs+u4+CB
z`}twr0u-iCbP@f$fyh8Vtfevf4L{BOA$l*UF<v+*=7R}%HM%|;2ukpaMz1v1I5z=a
zC1FW4QN;!3IC_Q5?psV*jb?YBu4NRR`hhl6)-be5*B-RSl(02=IaA-i&U!Tnr_`d`
z^+9Kpg=z+RK%x$a9jp^VfvA?E4b%vZMst6FZUF?Y5>W33kVVbzs(>j$7`@VdZADAM
zuKeu|k0;Uk_0;b3d>lNXkZ?Y*YQJy2>3j|_DPE8Nd*YrkS6&EppLAYYWzyB{v$@i^
zPB}4hl@m+anKA|`POCxl32=<|e8%ZSThGT1Lj`jUZLqZ7M8_#Qt^E?m9DW@(VB$om
z!CjBR(T1AD2GnsgW^MrrF?Rkyq8^ZSyJ01Hiq$%&AR-j3_U7gi3^kPhH6&hDACGkH
z7F8m$QUH4+9x_EpBHp615k+@{^DQv&Kq_)XC?39YTtph)5|4<8sJfgP{Ip{RU=&A9
zRr8r1gjPcW{x-X!)~fHS8s4HcjwG5uxA&gcUVF_+V3?`nafb-+XzmZwcM*VyOA)x7
zY3^%v(mrG7^W55qq4g_jM8l3i{Yo^@uu>CfTv-=rT1hkmS?j`&(RgTMJlrKjC?LWs
zvK0^!hQ47D2HU|`op%I1b=^%&Z932U0AZSPGt-;Sj{%bw<EvnQ8#j8oVzHh?J2zpu
zh$OI}Oh7OL(Vp*Ksf+dex*oyXEVlyibzxS)Kr&H_D!C0Hi3A$mer034T1AVfzA9Wm
zt7S%gnY~@xW4&5tjq@|;Bx<S>9pD26YSa=IQHcZ;1(nj{CFs#xp@*k^UspWVr#Y>|
z;B|VE08+!ggJFH;!<_vg&r2W<)yq!ENMPAU+5Kj`R@C-H8sgpG*oC~|obhQDg^tm9
zoiHO3`70W~LEM1EFkm?sKo`+*WQrT=R|<$SLw18k_Mi~85;@YZoG*b|_A#;_Mf^tR
zWgaeW)b^VZSk7)0H_8FV3wMyKm@wr|Zp59PVc&$fQcUh*^0-98i>kcU-R7KFOLH^R
z#6X2y6z%$m5F#db^ME}lw21Z21%H&{8{Z%{XnrxVUYO|Y<pGMp7r|h?W^gNj{=D{g
zEyeovY}CWS^=c~P4bF$aSkx;QY1WORQL}DwK3Bqt2GOWZ+6SgxpBD|zj~OeJGvfoX
zfy90woVclgx1pW59ff3a2<AE8`ydt}roF#>+Vuiz($z*^+rixrBdf#)wZhF#1G8-|
z)3QahXf4k;!zCOHYgSt7m}wQQI%ZTVYjy4er91#l>R03ts`w_S$b&cO2x$UOwRks*
zO{&rLZfKx1ceLh%<(fMm#aNYv+=I+)Gn_HZjh*%JU0rV%JH@V*0+NIgw?v(=<Opy&
z%iZI=%wjdDv<243+q$}iC7MK=ihg;BxwVV-?i)qB*yuwK11j2(U)n^QXp(oJJNSaw
z40hPWqhc2@yTmTB^K$0)MW`({OKgeJp7qWvEVNG9$_wcSNEQg)BS6NtAZRwryU>(l
z+`p+leskAWag(@trGT7e#J7sASS;hfi7jHQJj#umA?apuv$zRK?u!5mc}(M@a)KMC
z*dp&n!@0eR=BqCu&l!3K)lPCt?3QY^gT%L4{Y7uftKSyyh;L6E=TT2!lyWcFOac(!
zT_4}GyBRr9>=t`g3dog4JRuTtim9JrYToC%O|cz{ZTAs>&4?2k(E&t<^EIEb^9ov^
zI|-~jg+lFDK7i*-t$e2=qR8ID03@TR&$=Wt?wn#80nBNi%K3dgjMY=aoaX@8_+A)b
z#P^E5^&oyLa7ySOiM<Wz-&59aM1Q)h--P~|vVP)i=s0_;eJ|~SaeKrbu^SWr)n8+~
z8(Bqd_gr{6(<jtEF1G1y@nO&@Z~H#JwLZQNL3pd!=Q+4jbe1_-`LPP1vyd(JLE=8K
zPuz+Cc+p?FJxKfBrnLj>aYwz{oBdsj(|+ZyRi<0iIliiLNBuz5St;xG*NQUIFe3O$
zxAQ5VpdQ`Uc$f390En(N`yA7f=;kxv&}a40_%XB*kn|khI=ue#cWB_;MBl)R-JkeO
zhjl_x%o>tB?MLm?Gkgkt3w2=){9k~)2>5-$O8WPs{{g_aA&$!V+=X)p^SKYT4@Z{}
z0F(jjb9j-Pls=mRiF43s$HB=X2Ms=H<Kc7v!eDQI?@j%;3}PSPQ%x4-BIIoOEu!zB
z{xEjVEt3T&lb>~&;!zm)^J>6OOwy$d`g75iJ5G#_0QWfh)39O7!K|ENVm#;?cWekm
zFofv8jh%$^A4|9J0;fKjfF+&73p4p##fewZa34B*+M2YR2GDzm#!x<{%Fm)SsoQ8W
z8t4Gnnk0>5hDq;J<#Ptp|Bb;kub^$xFH|`lxGPXcF9x1L`A;ZK`euN!|D($7L2e&X
z<^92v!6@AhDU<Xl)G_HZfJf=35bN-$Du1EMV_}9&tMV~behQ_DWXbefOk_wr+V9%`
zH^LR0U@!M3EXF=UzXw`_e(Cj|A_1RFv}xQ&sS(uc+jZ|ncrZf0^m<>WPR#PNU5xwP
z#;;MInp=aujedZ>kJ6x@s`jr`y95uCX)v}NQllX2jZXR+;&%_qJv4-}k4EW}#%<8*
zC1Zq6(+`a!l!Snz3O+@5Q&oT=&4JTsZ%6xXIuJNR8A=6a6wNF(Q6{i}@+r*Y-Sm#Y
zMU;=KQBMRe)BTY0hxAT*HSiS5{|G!yZDa(G03Hke86X>j&%#TC!GE9+(RlESuu|K)
z82lzZt=do1L&1w^vy^r8MDV+`o}LN*7s`(Ze?cFozYq37|1SqmgXVj|Un#9Z#@+PG
z;6tFlAyi|GQe#LMkQ~}zJWoR*ro1z>2b!D?v2}ByW+O?3kOi6tL)-A7h;?|H{v@>1
zc!542x=1h3OQ3v#zKV7Wv2HE&N@zE@{UCH3Muo#8Xg7s#gWT@$5#u852_K>7)hPDo
zgwaL^!yAlh1>dUpKTlKPGxV~ed6{OzZ!^A44}>p5=S$(!v{gyjs@hM}li{?n)sy&<
zaK`vgdNDk2e2=~oUO@S8;fGNEB>WI`4x10d-cK6u0i@A9Lx$01vW@%9%V^(eUNT-*
z9x#jvz@Mjc<|Bq-JYb#%_FbyHq{=@)$$UNt?9*l+RV$r)5tE5PFU4s9Wi#y!NP3ta
zrBBck;QQ~?W1KQV%LvGd=NUvFX9oW<ysHaC<pm##b0VHfIvlX|vu|9ci&^_N8kriO
zvfV<jy=c==u^^8iFC|guk_Ee;a&AgG*~|=0F1ZCeN5^LFwNnKeu@^Eadotx@<_k2Q
zNjcJ$=L@|f85e1ArdY7k%nB?rGT-YF+gXf6pGJ?{t}Hq!dqT=A9nR#_<H^)qCU4Vd
z1>X^jDA*Om=-7od)O3ek(Bo)4naNXcsvsS5q%0`@InDcMGG~*k6`qG&)xVI;6v(k>
z?M0+-N;=5j+D=%KoXtzOkV&~PcBVKxo1DqoLj}(~4~C2nH`=SCju-QVOwR5dmbv*%
z)^;XshyCP|1}JwNx3kGb)pAShyn##x)J!IuDJ+$+-V}jL-j!LKPC1!^J*FJydxxFi
zw+}8U;|?vR?0MzzS-Ws{+;-jMtWA2KVpgeSff#da-gY1~KkH2~!^Z6VY++8JCmab2
zT<Husn4|*cu;|z&pRd|JHRssLG}w@C(PY-P=galt>F{(Lo=fIuSEqKekSr8kCCzKf
z`Lm;W`~G5*9pTT(6t4DqkK41wY|=TjIPX9!gp*4E*^9b7rVPfEA$X@S6N=|id4Yz~
z=}9{!^J$kx^M##*3US!U*!gr;yKB@PPZm;h1kq#QJOvHuP<|of$b8Pu7uXO?PcE5H
zv*(96R^Y@1+bPhL9FawYhp*_c%zClz_TFLVZ6}kWF)WR91FQ!R%fpRRlBX#RRg;s5
zSR2xcIi3ncFNBr#ct(bcj>Bq<%LV%=-*Jv&vCT`D!;*AUTkA&wdwUeZFe_^oQKj?M
zB4G)myuOcP4pn49gURAdE~9+x&Dv-QqWlN;dX27y=iNdwpR$1;OS*;8eA-?-b{@QQ
zP~Ow%&ZLvRkaX;_p=Ip#dS=&n<VWobCCV}^(!p63?QDT+FhmWUl0%M@MD+OSK|zsX
zbQaKFF7*yBO(YTWpdOR6vIIwfg8^bG&(Vgt_Er!aR9c~<oHYSKg#epK;+`I<OU>hI
z$8#d<H<_BVIs2YWW{Y~(h9y!5jrxc(t36Wk)}fZ*)2TUQj$Q)fF!u&16!%wWkqs%2
z4no%PsmY6A*Z`7Zwg5A9L(Q~qmKNtBe`&JY*><ksw=s#V!fpbel3+dhxIHhiSeSAm
z>FVWvIGM>R$Wd8<Hhyzxem<K?sn8zgK*EW8=NL)+-G`GB$jGAe=sBo!*o#v%r{FgF
z9rPge)7w!S@l(J#px0ye0!w!d<NTf>8_>e*j@A;2pi6@;0j>k!+Ku*uG>BFw8GXmW
z$HiN~pRd>J=u=aor@L6UOxd{KY>x$M3HdXSXu&QU9MZHzhT^N+)H8lve^`*`R*qbL
z_x0SG2KxmJNGV|@r!tc~m|V+Dc2Ea!iL0=4P>Mxs*I7`z#NB0AZaWOqd}o(qD<-sU
zq->l8JtSOGK2db*AvnfSp~2IAJx<peYosd2vYjc#iCvvilJd|orzU_yPa+5{MUzt&
z>rmnF$}6vN@Xe|bI^6K-aQV3ta6}Sdn^-{|tCor)2M)I}<S6ht#h}v36A&m7y(PR0
z<};w>d38}r2rgkcCYcXA_Uc&o>H48Xr5qnK08Fp?qoGl7*3p_&>*oRV^g1k`l1Vtj
z0cKvwlhEy6$Y<Blz_8Me9m9Qpw2tUBEH2<>KQM$HPQk52hsM$4Fx0Xt^(+q!e{IZz
zo_(ZoIv|bB<vRM51;}7&xZ!O&f-y7b`8Wq2%QNQ8<TX58DVP0GT8kFAE<#`C%=~)r
zr`Kw(!V>nS5yUOT>0xhl$tTyE)k>KO<U0<oG+j_N=Yvri@gj<?#*(U7EF-HP&ZEUq
z0qd?q;m?Lo;g-#+&bjl7+Z^-g8iBRUFAsZl1oGnYBbf6hXX8$$?YS210%)`FGTTr=
zz4;U@x(l}oSI77$ET<su%DN^n8Yg$90naEe%j-z*sEh)hGhT&@Yvoq^<7sFT`10J&
zcCVFIuD_1vQnEDKcty;COBz<_#K3#ky<WV*C)F1E{73gxrXNmjydJK$zBFF1+=7x}
zVb$>}(q7;cU8cv`>+Lq|dtNQRM*a%m#Fqodtnz@Ko63B$-<w%Z-&D(Su4DZkGB$9w
zn1H7wJj{OP>_iO=<qXGbBLfU;i*;Vb);~xz?0cdE*fg{+<wxFiJ3JjV+(DoJ^scPi
z-Pn;!JhDo<3mWEAqSt2w9s&BDC3@DAvxft%UUy7qX$)J!Ruo~A9#ir;xG%q}M9P!H
zZqQ_UQE0Qy1CxQHeT;w1=zzO;f78jx$MVrbkJH#Bl;&yTojQpm#uMh#?gXu&0*uE{
z?HM@2pD5#wNpRQOOlk7d%Jrq)@#-+(Erx?cFZy2GWL}GpZxaVTXgY?VS7n&3<*ev#
zs9XZLsd(9V?2qF|Umf}F?f3lr)DI7Ti6S3;VESa!u5Vp5@oSb?b)Bke>#8Woy+DM5
zF@Z)ME)65CbwP`auw@vc33pLOB#be2Edazi83m$1U0q!WH#OXige+=p;a;SQLLh8u
zF{=Wx&R7qRsIEhaib2(}9_Gj6!5~r<WTIG)X@R}K_pKy`KrDu8K^1^*fq)niL$Rvr
z*q#XLPN*ovP-{yBq^&H!wl2IOb~hM;Yn&n$Z4jr?2y;~jwc-T*0^kodA|`Bwa-an%
zZd`d(YYWJX2p7=D3dWLk5zVs&!((<Brhy;Ww<`NH7zAbEz92kMXR`EI4*1ypEH73x
znWv&PVuHuTmQ<NkWf5Wxrptk@F1Gwmo`$;CmgRR>?FYMBWlSV?0_wssm=0LH2Rg=*
zjA@D4ssgyPPLZmDsZ=+Xi{&D*`(wq}k{F6az#ji#6jAP2Q)nQ3iGSr&Ryz_gyo)b1
zBI?F#6JH`v1?o>Z$@!x)zj`}@+X{KXH53i4{$478{v&w*aTxD5{EB08WODk_pCA5&
z@zdewe)g3oKl6?EJ-!k6;l0x%GKJ6W`GRZVZotB4EWS%|r-#zHOrGz<_yhg)_>#3s
zH+?eW;%38|EELljX$|)G@1Dk|JHA+Rr@gzL)mqNnJB=$NJL%d!tam;=!|Gld;Zi?;
z%5BT*tf~CpB2|LV4olpk*y_qm-SXLXZ#v80ZQvJc+sHbgT5lHr7aj=kujP1pmhn3Y
zd`CmesLCGwZ3n*GT`mKnso`_ifHU1P7Zs0=;`amg<T;`d4r1?l6%q4&Kga<a3E(Fb
zbP}hlvnUVYS4xvOK_0`WhqI`U(qXju`DExvzh*twKKgFC^gUELo<q?9S3WBpR$Cix
zcCl^nCtQBEQV8V~ut`8&Yz(}I>a(LFpbv+hf=B~8@%E=r|5tNbRy6%MU+TYI*#7##
zpc_lySe4ADzVZ`nNsa~Deq@Id?Mcu&pf1t$WBV$-Ja_9r&(`?oEIvUUg$2Bo!mqGv
z9Irm*^rBu=@PyK73~fI5WQjiw&EvB>TQ`dnB0eDz>vKTyKjy*Na;EleUp<*#SI#bF
zk3M_xnUg<jt+lFT58K8zu@8n|H{WIO$J;D4vwq_^45c6a<zMZ)eKD7{a1qM)WBc0&
zdi&cgJD-w#7rejy#MI%Q-R%}%2&R)+nYZ`1FWGMU?YGrN_u&@E&dp?(EHKEs``e37
zey@w`t6b9U$@yO@^rU2NZ_>^6E)29=xPHx?$F~(H%cX&pWvwzBO(PfM*0@3=e1QwC
z^0<}T-#)(NT`#FGCzA8??R}a<!71`(<$3uA^%-ne;~?9$Q$^g#FL`}H99&*QPdh!~
zWESvMz^v`QL9RP}t`?pWAEfG3ku@H(7woK+<#K;}(jCn&$op)k-702=QhZUrzx{kN
z>)M`Cib>x!wDDE%tI&O4->MTpu&>XbA#`}Q-+bP%2M*}p4t$!^?3>U3I}iLXAgw%l
delta 4268
zcmai2eQX@X6@Rn0_u;jD@$A~`v+vGXf1iDK_8AP#C$STo;3&pPtUyu{$YCE~YP@9c
z;wFXmQV1bb!A&}-0!K}uYN=2uF(fUhEvQs23Tj$4sA~C80%`@NfIun&(V#Z{&E8$?
zgg?5MdGp?H-h1=r&70YE_U_ob;n0C~_kXf>Jr$p~NMYqHK-2&LoqV2J0b-%`p-Ez)
zQ;1Y_@+tMS_wXxS!EMkUk#HZ;tLouKqhbe91>`4}6GcR$k`P;2yLKW1$U<un-(t;W
zec*g(miReaT($!oCZINPmNi+2<otN9K~u;}lQB3qh!|uvpE%u?5*rjX@AzGSD>D<}
zuBd~tJ7Wk=b25_if-`4hm_@7YRLogTshurXLbDZ0-E5^&KO2L!W~k1mC9^EnuNhuv
z2`l>HUY}hCjC-Bif!UfI_c>GG{qUP7t=jN&pWiN*1OBu1M!=cFAP?9`fEKHiL<V%0
zxIeLi`|Tiv#N|cCUIfYPX@OWJ@o<qT;uR1%@R<oSRbjpBA&Th`<1@~0WdPitcik0@
zVn`4I4RD`b4bH9sX+uP1g^0?&7?C$nSdivaDRgJRO%(TY%{HYczg>&Y*@2;02;9=2
zBKyvy6y=KKs`G_qbH8&6uw4h(I)E_TFoC{aLeI7t<%KZHzYxZ}Mf^$Uos#=_>Y}t(
z6s#*Qw$3B3^LhC~#L4HKUQaPxXS~G4>JesbHVkT1I{N|hN_Xio6sTr2>&?wrD4rYp
zVo`}Kir+pTi}Rc`DDF#s;vhDJ6%*&~mC<4vt??Ld;jObdS+YhIujExWPM2mc21)pM
zl@UC<jJMWqF<N;mZ<(36>`(JaaHHUhVit6h2b^>B@zoklXYNPpohwy@5QIzApF6j+
z4T!rd3m3<^x6E#oE6}-)8tSo}-Owv5gzM$HIh#YJFuO^{hDB@6Ulxp)p%7#tVWt+T
z&<bK0**4i#bFbY_l-G<JNAs2{qQ;%w#$nZXjol%QWPv9dovurDI;BoEwBlgm&v1iR
z+X-oq1S!m6I-5#Xu3w{;*B1Qg+~i&eU3qEYdiXOum<12r01;jaoe^@TOFm4ajvIE5
zMB7ZGJwC+Sc>8P)HOq{8&Uw9kne^svQM>an%!<(bA?6_-1bDfepbooNm*?&1+Ic%~
z!zLc%b;)YysI=s6j^&2o2oUTHyH{F;kkcz9<K-dyN-)kjsjePGO%}?iP=ql(#?@l=
znMT+&B1`VV<z}UJ6%Q9c_{@I^E{qP-=#0mC2k)HCp{kh?<xzWybiT!ht&wLJMHfb-
zVcrSa$vb%m;{8KxsZ^ShO7cWw7DTlSoDnyT#F9^N26&t&ie4L>yGxqRK5)Fj&f|jD
zY+Tlwdk%oOfJ|g;C5Usfpm&u7<TVV9O9C^T&!molkyRxvC*gP9OI6p*xeh@6gUA3g
zl00cbxesFp^YZllfG2AK^h=;9haj(BAlE@&vp|jwKycoG)5292DcWyA1`^0nf+u)<
zW?~hcKY!kN`+}NGxy<;?L^pT$DPKIN%WnPe`SWfzO9|{UjqZ4Yck%9G_7?NS3zj|z
zF9|=OtHJPYOxex5c^5Y2)cjTR|F6|Nu6L_&Gg=qljPixEAY9G)_`)h8W{>794_sKg
znW|i<>P6L_O*l7|6ikRwqtUq^0B^i_mb@mFi8tXm_dkSk!E-m7eEA<hLsd&2`$psk
z#r9Ag`i1yT$X&p{e+hmXvh;K268r_!k<|YHeak{eC^)LY8+{WR?rnGC1|_yuJXaaJ
zY5*_K=5G+qq2Ge<g67ejGck2*7J3xG8~$Ou`I|S7(xZ6a;#Ew|Yc~&}NN5i7AtoAv
zZBZO8_Y*~arXVI-iw18l&qO*H&im+Qd<d1(G*BP?i0%XZolD=R9`OHhX@vEVMoG4e
zX>^ZE9|!f(Z-D#gRhKSTI+Y;(Re28d45*LZQ{<@6BW)`@(smfsM<+awcr^N@M55H9
z%7N=$dKajVo>is6%izV8i(2xlATS!o4NbH^G=QZRR<n*c4Nx^Tg4R<9Xcc1pm=@Dg
zDq}4!-$DJ<!n$e59UY?ESuglQY$<J|Ikt+@bczkq1ii_IXe+(r^5@tHc&{=Dx>y;_
z(@t8gY;oOfhuJmC2wdEtWH3CXOoDzBsuOfjalp?iz0f?S?4~`K@@{&7K347p{aiUf
zk5IK|5O}L+21wE);Ag4l2t7tyJ<{h6Pd#*YLT8BfdLBbD$R*{6=(Oh;Jxw2ZdO<(K
z@YBT9UJPsM3DBVW3%D@VU&Ab=z65^2<<sgZ@L6?~UUscsq`m5CdXXMfUjsd&PD1}D
z>YLDc0XnbIX?4(D<l7i6Ab(T;NiXN!=wxC)gwlIZdW_DgpMbCSj(|p8+UC+8P^tMj
z^0CJ2V06S=MP=?fx=>`>SQi<T0S(g>Q`i%v?!~TN^!y!?+~Cq9rgXD5DU0_am|-Fk
zkLXvI4W}ojsB0`|i)W0ViI-|p;!S;}h?t?Kp-q{ry~`QPY_RP~TARr&lvnS$HhqW4
zRR2&+n=7kFbLrf!EZKM7G&Ggjxhp+6k=r90YRtf<OnQ9%#8gJCui5M$8pupfjAg{5
zHErTWbFB#1)`?GRnnZ8yh^K#iT-;N;?uwz&vD-4^yCyTV#>u2}nQJp&y?`Kub@t6!
z8UiI46x=}&`$N}>cSC6t0Sp$Wr#0!RvCO1gxHu8cuy2d{NTPg<CXw6>O;JuPk6bVQ
z7}-<bzn*OTzJh@%x|J-t1(Jh)OdP3e7yIhcp^;%g(-#zSbQ^pW6r#e3R(Gf9>(oxH
zd=v0AjiXHIrZvz?yDm$P7utmf+9?YkS;|2xN4s1phj}b`l76yiIq9`>GrVM>bqf%3
z=O~QPusCFen2a+TDwy636${Z{2%}rf)-~u8uJ;9SqnL>{s1{L=c%nX~vi=xJMX@NX
z)yTzISVYEM@)-Iob%^QkRU#E`V`1@BG~|Wl_EE92p;5dSUCu5Soeh^MtXb@9u%y#w
z@py?j*<g8q2*1_DyrRoWv+W4_%kB79tQo?(HXv9VX_v8O@uSKl6@C`oK<s$DF{0d9
z{cmme>mSZtdvEu@j=u5yv0qW(*{^N8uCDj32fV}r(I94EUW+kase%SnVl0U7n`qFB
zKXro<$dy4k5`>c8;nJiulH<Xkm}?H@Ed>Js=|t}U3K%zi&}YG(_rn(-#26ox6DUDH
z6!@SEMI%AK<<U0=13_<%zL`a^etoO-V-T_wm6Wk)1XSOSAV^R9;Zz^iGjc%R=+bl^
zvoSfE@sS51OZ}jJhfl^IjcoO&f_N(RR=q<{>OJ^%xOAmH%m?9XWi+C1bf?$1x?NCD
z2lPxJpx@#CU=iSh(MSOQN_{i}A;1dvf;iBcN|yeUKwcn+(N5yq?j&BUn??sl&wTHL
z=~s^aWBAB}KYsuB_l-Z~6K}WX^?k)Z+o4QHC2t*WG?hauj(@jpHxqrVN(5tB-bXYb
feHNd0S0DdQ%=VVydtJWn=T;o&T_3T-DgXZf6xhmD
diff --git a/client/tests/kvm/tests/whql_submission.py b/client/tests/kvm/tests/whql_submission.py
index 8dd48b8..a0ff87c 100644
--- a/client/tests/kvm/tests/whql_submission.py
+++ b/client/tests/kvm/tests/whql_submission.py
@@ -6,20 +6,24 @@ import kvm_subprocess, kvm_test_utils, kvm_utils, rss_file_transfer
def run_whql_submission(test, params, env):
"""
WHQL submission test:
- 1) Log into the guest (the client machine) and into a DTM server machine
+ 1) Log into the client machines and into a DTM server machine
2) Copy the automation program binary (dsso_test_binary) to the server machine
3) Run the automation program
4) Pass the program all relevant parameters (e.g. device_data)
5) Wait for the program to terminate
6) Parse and report job results
- (logs and HTML reports are placed in test.bindir)
+ (logs and HTML reports are placed in test.debugdir)
@param test: kvm test object
@param params: Dictionary with the test parameters
@param env: Dictionary with test environment.
"""
- vm = kvm_test_utils.get_living_vm(env, params.get("main_vm"))
- session = kvm_test_utils.wait_for_login(vm, 0, 240)
+ # Log into all client VMs
+ vms = []
+ sessions = []
+ for vm_name in kvm_utils.get_sub_dict_names(params, "vms"):
+ vms.append(kvm_test_utils.get_living_vm(env, vm_name))
+ sessions.append(kvm_test_utils.wait_for_login(vms[-1], 0, 240))
# Collect parameters
server_address = params.get("server_address")
@@ -30,47 +34,54 @@ def run_whql_submission(test, params, env):
dsso_test_binary = params.get("dsso_test_binary",
"deps/whql_submission_15.exe")
dsso_test_binary = kvm_utils.get_path(test.bindir, dsso_test_binary)
- test_device = params.get("test_device")
- job_filter = params.get("job_filter", ".*")
+ dsso_delete_machine_binary = params.get("dsso_delete_machine_binary",
+ "deps/whql_delete_machine_15.exe")
+ dsso_delete_machine_binary = kvm_utils.get_path(test.bindir,
+ dsso_delete_machine_binary)
test_timeout = float(params.get("test_timeout", 600))
- wtt_services = params.get("wtt_services")
- # Restart WTT service(s) on the client
- logging.info("Restarting WTT services on client")
- for svc in wtt_services.split():
- kvm_test_utils.stop_windows_service(session, svc)
- for svc in wtt_services.split():
- kvm_test_utils.start_windows_service(session, svc)
-
- # Run whql_pre_command
- if params.get("whql_pre_command"):
- session.cmd(params.get("whql_pre_command"),
- int(params.get("whql_pre_command_timeout", 600)))
-
- # Copy dsso_test_binary to the server
- rss_file_transfer.upload(server_address, server_file_transfer_port,
- dsso_test_binary, server_studio_path, timeout=60)
+ # Copy dsso binaries to the server
+ for filename in dsso_test_binary, dsso_delete_machine_binary:
+ rss_file_transfer.upload(server_address, server_file_transfer_port,
+ filename, server_studio_path, timeout=60)
# Open a shell session with the server
server_session = kvm_utils.remote_login("nc", server_address,
server_shell_port, "", "",
- session.prompt, session.linesep)
- server_session.set_status_test_command(session.status_test_command)
+ sessions[0].prompt,
+ sessions[0].linesep)
+ server_session.set_status_test_command(sessions[0].status_test_command)
- # Get the computer names of the server and client
+ # Get the computer names of the server and clients
cmd = "echo %computername%"
server_name = server_session.cmd_output(cmd).strip()
- client_name = session.cmd_output(cmd).strip()
- session.close()
+ client_names = [session.cmd_output(cmd).strip() for session in sessions]
- # Run the automation program on the server
+ # Delete all client machines from the server's data store
server_session.cmd("cd %s" % server_studio_path)
+ for client_name in client_names:
+ cmd = "%s %s %s" % (os.path.basename(dsso_delete_machine_binary),
+ server_name, client_name)
+ server_session.cmd(cmd, print_func=logging.debug)
+
+ # Reboot the client machines
+ sessions = kvm_utils.parallel((kvm_test_utils.reboot, (vm, session))
+ for vm, session in zip(vms, sessions))
+
+ # Run whql_pre_command and close the sessions
+ if params.get("whql_pre_command"):
+ for session in sessions:
+ session.cmd(params.get("whql_pre_command"),
+ int(params.get("whql_pre_command_timeout", 600)))
+ session.close()
+
+ # Run the automation program on the server
cmd = "%s %s %s %s %s %s" % (os.path.basename(dsso_test_binary),
server_name,
- client_name,
- "%s_pool" % client_name,
- "%s_submission" % client_name,
- test_timeout)
+ "%s_pool" % client_names[0],
+ "%s_submission" % client_names[0],
+ test_timeout,
+ " ".join(client_names))
server_session.sendline(cmd)
# Helper function: wait for a given prompt and raise an exception if an
@@ -89,13 +100,13 @@ def run_whql_submission(test, params, env):
# Tell the automation program which device to test
find_prompt("Device to test:")
- server_session.sendline(test_device)
+ server_session.sendline(params.get("test_device"))
# Tell the automation program which jobs to run
find_prompt("Jobs to run:")
- server_session.sendline(job_filter)
+ server_session.sendline(params.get("job_filter", ".*"))
- # Give the automation program all the device data supplied by the user
+ # Set submission DeviceData
find_prompt("DeviceData name:")
for dd in kvm_utils.get_sub_dict_names(params, "device_data"):
dd_params = kvm_utils.get_sub_dict(params, dd)
@@ -104,8 +115,7 @@ def run_whql_submission(test, params, env):
server_session.sendline(dd_params.get("dd_data"))
server_session.sendline()
- # Give the automation program all the descriptor information supplied by
- # the user
+ # Set submission descriptors
find_prompt("Descriptor path:")
for desc in kvm_utils.get_sub_dict_names(params, "descriptors"):
desc_params = kvm_utils.get_sub_dict(params, desc)
@@ -113,6 +123,31 @@ def run_whql_submission(test, params, env):
server_session.sendline(desc_params.get("desc_path"))
server_session.sendline()
+ # Set machine dimensions for each client machine
+ for vm_name in kvm_utils.get_sub_dict_names(params, "vms"):
+ vm_params = kvm_utils.get_sub_dict(params, vm_name)
+ find_prompt(r"Dimension name\b.*:")
+ for dp in kvm_utils.get_sub_dict_names(vm_params, "dimensions"):
+ dp_params = kvm_utils.get_sub_dict(vm_params, dp)
+ if dp_params.get("dim_name") and dp_params.get("dim_value"):
+ server_session.sendline(dp_params.get("dim_name"))
+ server_session.sendline(dp_params.get("dim_value"))
+ server_session.sendline()
+
+ # Set extra parameters for tests that require them (e.g. NDISTest)
+ for vm_name in kvm_utils.get_sub_dict_names(params, "vms"):
+ vm_params = kvm_utils.get_sub_dict(params, vm_name)
+ find_prompt(r"Parameter name\b.*:")
+ for dp in kvm_utils.get_sub_dict_names(vm_params, "device_params"):
+ dp_params = kvm_utils.get_sub_dict(vm_params, dp)
+ if dp_params.get("dp_name") and dp_params.get("dp_regex"):
+ server_session.sendline(dp_params.get("dp_name"))
+ server_session.sendline(dp_params.get("dp_regex"))
+ # Make sure the prompt appears again (if the device isn't found
+ # the automation program will terminate)
+ find_prompt(r"Parameter name\b.*:")
+ server_session.sendline()
+
# Wait for the automation program to terminate
try:
o = server_session.read_up_to_prompt(print_func=logging.info,
@@ -162,38 +197,63 @@ def run_whql_submission(test, params, env):
except (KeyError, OSError):
pass
- # Print result summary
- logging.info("")
- logging.info("Result summary:")
- name_length = max(len(r.get("job", "")) for r in results)
- fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
- logging.info(fmt % ("ID", "Job", "Status", "Pass", "Fail", "NotRun",
- "NotApplicable"))
- logging.info(fmt % ("--", "---", "------", "----", "----", "------",
- "-------------"))
- for r in results:
- logging.info(fmt % (r.get("id"), r.get("job"), r.get("status"),
- r.get("pass"), r.get("fail"), r.get("notrun"),
- r.get("notapplicable")))
- logging.info("(see logs and HTML reports in %s)" % test.debugdir)
-
- # Kill the VM and fail if the automation program did not terminate on time
+ # Print result summary (both to the regular logs and to a file named
+ # 'summary' in test.debugdir)
+ def print_summary_line(f, line):
+ logging.info(line)
+ f.write(line + "\n")
+ if results:
+ # Make sure all results have the required keys
+ for r in results:
+ r["id"] = str(r.get("id"))
+ r["job"] = str(r.get("job"))
+ r["status"] = str(r.get("status"))
+ r["pass"] = int(r.get("pass", 0))
+ r["fail"] = int(r.get("fail", 0))
+ r["notrun"] = int(r.get("notrun", 0))
+ r["notapplicable"] = int(r.get("notapplicable", 0))
+ # Sort the results by failures and total test count in descending order
+ results = [(r["fail"],
+ r["pass"] + r["fail"] + r["notrun"] + r["notapplicable"],
+ r) for r in results]
+ results.sort(reverse=True)
+ results = [r[-1] for r in results]
+ # Print results
+ logging.info("")
+ logging.info("Result summary:")
+ name_length = max(len(r["job"]) for r in results)
+ fmt = "%%-6s %%-%ds %%-15s %%-8s %%-8s %%-8s %%-15s" % name_length
+ f = open(os.path.join(test.debugdir, "summary"), "w")
+ print_summary_line(f, fmt % ("ID", "Job", "Status", "Pass", "Fail",
+ "NotRun", "NotApplicable"))
+ print_summary_line(f, fmt % ("--", "---", "------", "----", "----",
+ "------", "-------------"))
+ for r in results:
+ print_summary_line(f, fmt % (r["id"], r["job"], r["status"],
+ r["pass"], r["fail"], r["notrun"],
+ r["notapplicable"]))
+ f.close()
+ logging.info("(see logs and HTML reports in %s)" % test.debugdir)
+
+ # Kill the client VMs and fail if the automation program did not terminate
+ # on time
if not done:
- vm.destroy()
+ kvm_utils.parallel(vm.destroy for vm in vms)
raise error.TestFail("The automation program did not terminate "
"on time")
- # Fail if there are failed or incomplete jobs (kill the VM if there are
- # incomplete jobs)
- failed_jobs = [r.get("job") for r in results
- if r.get("status", "").lower() == "investigate"]
- running_jobs = [r.get("job") for r in results
- if r.get("status", "").lower() == "inprogress"]
+ # Fail if there are failed or incomplete jobs (kill the client VMs if there
+ # are incomplete jobs)
+ failed_jobs = [r["job"] for r in results
+ if r["status"].lower() == "investigate"]
+ running_jobs = [r["job"] for r in results
+ if r["status"].lower() == "inprogress"]
errors = []
if failed_jobs:
errors += ["Jobs failed: %s." % failed_jobs]
if running_jobs:
- vm.destroy()
+ for vm in vms:
+ vm.destroy()
errors += ["Jobs did not complete on time: %s." % running_jobs]
if errors:
raise error.TestFail(" ".join(errors))
diff --git a/client/tests/kvm/tests_base.cfg.sample b/client/tests/kvm/tests_base.cfg.sample
index 8d533ba..e81d879 100644
--- a/client/tests/kvm/tests_base.cfg.sample
+++ b/client/tests/kvm/tests_base.cfg.sample
@@ -347,11 +347,12 @@ variants:
server_shell_port = 10022
server_file_transfer_port = 10023
server_studio_path = %programfiles%\Microsoft Driver Test Manager\Studio
+ dsso_test_binary = deps/whql_submission_15.exe
+ dsso_delete_machine_binary = deps/whql_delete_machine_15.exe
wtt_services = wttsvc
variants:
- client_install:
type = whql_client_install
- dsso_delete_machine_binary = deps/whql_delete_machine_15.exe
# The username and password are required for accessing the DTM client
# installer binary shared by the server
server_username = administrator
@@ -363,7 +364,7 @@ variants:
- submission: client_install
type = whql_submission
extra_params += " -snapshot"
- dsso_test_binary = deps/whql_submission_15.exe
+ restart_vm = yes
test_timeout = 3600
device_data = cat0 cat1 cat2 cat3 logoarch logoos whqlos whqlqual prog desc filter virt
descriptors = desc1 desc2 desc3
--
1.7.3.3
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
