On Thu, Oct 15, 2009 at 3:45 AM, Yolkfull Chow <yzhou@xxxxxxxxxx> wrote:
> On Wed, Oct 14, 2009 at 09:08:00AM -0300, Lucas Meneghel Rodrigues wrote:
>> Add a new PCI pass trough test. It supports both SR-IOV virtual
>> functions and physical NIC card pass through.
>>
>> Single Root I/O Virtualization (SR-IOV) allows a single PCI device to
>> be shared amongst multiple virtual machines while retaining the
>> performance benefit of assigning a PCI device to a virtual machine.
>> A common example is where a single SR-IOV capable NIC - with perhaps
>> only a single physical network port - might be shared with multiple
>> virtual machines by assigning a virtual function to each VM.
>>
>> SR-IOV support is implemented in the kernel. The core implementation is
>> contained in the PCI subsystem, but there must also be driver support
>> for both the Physical Function (PF) and Virtual Function (VF) devices.
>> With an SR-IOV capable device one can allocate VFs from a PF. The VFs
>> surface as PCI devices which are backed on the physical PCI device by
>> resources (queues, and register sets).
>>
>> Device support:
>>
>> In 2.6.30, the Intel® 82576 Gigabit Ethernet Controller is the only
>> SR-IOV capable device supported. The igb driver has PF support and the
>> igbvf has VF support.
>>
>> In 2.6.31 the Neterion® X3100™ is supported as well. This device uses
>> the same vxge driver for the PF as well as the VFs.
>
> Wow, new NIC card supports SR-IOV...
> At this rate, do we need to move the driver name and its parameter into
> config file so that in future if a new NIC card using different driver
> is supported, we could handle it without changing code ?
Absolutely, it didn't occur to me at first, but yes, we ought to move
the driver name and parameters to the config file.
>>
>> In order to configure the test:
>>
>> * For SR-IOV virtual functions passthrough, we could specify the
>> module parameter 'max_vfs' in config file.
>> * For physical NIC card pass through, we should specify the device
>> name(s).
>>
>> Signed-off-by: Yolkfull Chow <yzhou@xxxxxxxxxx>
>> ---
>> client/tests/kvm/kvm_tests.cfg.sample | 11 ++-
>> client/tests/kvm/kvm_utils.py | 278 +++++++++++++++++++++++++++++++++
>> client/tests/kvm/kvm_vm.py | 72 +++++++++
>> 3 files changed, 360 insertions(+), 1 deletions(-)
>>
>> diff --git a/client/tests/kvm/kvm_tests.cfg.sample b/client/tests/kvm/kvm_tests.cfg.sample
>> index cc3228a..1dad188 100644
>> --- a/client/tests/kvm/kvm_tests.cfg.sample
>> +++ b/client/tests/kvm/kvm_tests.cfg.sample
>> @@ -786,13 +786,22 @@ variants:
>> only default
>> image_format = raw
>>
>> -
>> variants:
>> - @smallpages:
>> - hugepages:
>> pre_command = "/usr/bin/python scripts/hugepage.py /mnt/kvm_hugepage"
>> extra_params += " -mem-path /mnt/kvm_hugepage"
>>
>> +variants:
>> + - @no_passthrough:
>> + pass_through = no
>> + - nic_passthrough:
>> + pass_through = pf
>> + passthrough_devs = eth1
>> + - vfs_passthrough:
>> + pass_through = vf
>> + max_vfs = 7
>> + vfs_count = 7
>>
>> variants:
>> - @basic:
>> diff --git a/client/tests/kvm/kvm_utils.py b/client/tests/kvm/kvm_utils.py
>> index 53b664a..0e3398c 100644
>> --- a/client/tests/kvm/kvm_utils.py
>> +++ b/client/tests/kvm/kvm_utils.py
>> @@ -788,3 +788,281 @@ def md5sum_file(filename, size=None):
>> size -= len(data)
>> f.close()
>> return o.hexdigest()
>> +
>> +
>> +def get_full_id(pci_id):
>> + """
>> + Get full PCI ID of pci_id.
>> + """
>> + cmd = "lspci -D | awk '/%s/ {print $1}'" % pci_id
>> + status, full_id = commands.getstatusoutput(cmd)
>> + if status != 0:
>> + return None
>> + return full_id
>> +
>> +
>> +def get_vendor_id(pci_id):
>> + """
>> + Check out the device vendor ID according to PCI ID.
>> + """
>> + cmd = "lspci -n | awk '/%s/ {print $3}'" % pci_id
>> + return re.sub(":", " ", commands.getoutput(cmd))
>> +
>> +
>> +def release_dev(pci_id, pci_dict):
>> + """
>> + Release a single PCI device.
>> +
>> + @param pci_id: PCI ID of a given PCI device
>> + @param pci_dict: Dictionary with information about PCI devices
>> + """
>> + base_dir = "/sys/bus/pci"
>> + full_id = get_full_id(pci_id)
>> + vendor_id = get_vendor_id(pci_id)
>> + drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id)
>> + if 'pci-stub' in os.readlink(drv_path):
>> + cmd = "echo '%s' > %s/new_id" % (vendor_id, drv_path)
>> + if os.system(cmd):
>> + return False
>> +
>> + stub_path = os.path.join(base_dir, "drivers/pci-stub")
>> + cmd = "echo '%s' > %s/unbind" % (full_id, stub_path)
>> + if os.system(cmd):
>> + return False
>> +
>> + prev_driver = pci_dict[pci_id]
>> + cmd = "echo '%s' > %s/bind" % (full_id, prev_driver)
>> + if os.system(cmd):
>> + return False
>> + return True
>> +
>> +
>> +def release_pci_devs(pci_dict):
>> + """
>> + Release all PCI devices assigned to host.
>> +
>> + @param pci_dict: Dictionary with information about PCI devices
>> + """
>> + for pci_id in pci_dict:
>> + if not release_dev(pci_id, pci_dict):
>> + logging.error("Failed to release device [%s] to host" % pci_id)
>> + else:
>> + logging.info("Release device [%s] successfully" % pci_id)
>> +
>> +
>> +class PassThrough(object):
>> + """
>> + Request passthroughable devices on host. It will check whether to request
>> + PF (physical NIC cards) or VF (Virtual Functions).
>> + """
>> + def __init__(self, type="nic_vf", max_vfs=None, names=None):
>> + """
>> + Initialize parameter 'type' which could be:
>> + nic_vf: Virtual Functions
>> + nic_pf: Physical NIC card
>> + mixed: Both includes VFs and PFs
>> +
>> + If pass through Physical NIC cards, we need to specify which devices
>> + to be assigned, e.g. 'eth1 eth2'.
>> +
>> + If pass through Virtual Functions, we need to specify how many vfs
>> + are going to be assigned, e.g. passthrough_count = 8 and max_vfs in
>> + config file.
>> +
>> + @param type: Pass through device's type
>> + @param max_vfs: parameter of module 'igb'
>> + @param names: Physical NIC cards' names, e.g.'eth1 eth2 ...'
>> + """
>> + self.type = type
>> + if max_vfs:
>> + self.max_vfs = int(max_vfs)
>> + if names:
>> + self.name_list = names.split()
>> +
>> +
>> + def sr_iov_setup(self):
>> + """
>> + Setup SR-IOV environment, check if module 'igb' is loaded with
>> + parameter 'max_vfs'.
>> + """
>> + re_probe = False
>> + # Check whether the module 'igb' is loaded
>> + s, o = commands.getstatusoutput('lsmod | grep igb')
>> + if s:
> I would take extra consideration that here could also happen timeout
> and return status is None problem. Could change to:
> if s != 0:
Fair enough, will change it!
>> + re_probe = True
>> + elif not self.chk_vfs_count():
>> + os.system("modprobe -r igb")
>> + re_probe = True
>> +
>> + # Re-probe module 'igb'
>> + if re_probe:
>> + cmd = "modprobe igb max_vfs=%d" % self.max_vfs
>> + s, o = commands.getstatusoutput(cmd)
>> + if s:
>
> ==> if s != 0:
Idem
>> + return False
>> + if not self.chk_vfs_count():
>> + return False
>> + return True
>> +
>> +
>> + def get_vfs_count(self):
>> + """
>> + Get Vfs count number according to 'max_vfs'
>> + """
>> + cmd = "lspci | grep 'Virtual Function' | wc -l"
>> + return int(commands.getoutput(cmd))
>> +
>> +
>> + def chk_vfs_count(self):
>> + """
>> + Check VFs count number according to parameter 'max_vfs'.
>> + """
>> + if self.get_vfs_count() != (self.max_vfs * 2):
>> + return False
>> + return True
>> +
>> +
>> + def _get_pci_id(self, name, search_str):
>> + """
>> + Get the device's PCI ID according to name.
>> + """
>> + cmd = "ethtool -i %s | awk '/bus-info/ {print $2}'" % name
>> + s, pci_id = commands.getstatusoutput(cmd)
>> + if not (s or "Cannot get driver information" in pci_id):
>> + return pci_id[5:]
>> +
>> + cmd = "lspci | awk '/%s/ {print $1}'" % search_str
>> + pci_ids = [id for id in commands.getoutput(cmd).splitlines()]
>> + nic_id = int(re.search('[0-9]+', name).group(0))
>> + if (len(pci_ids) - 1) < nic_id:
>> + return None
>> + return pci_ids[nic_id]
>> +
>> +
>> + def _write_to_file(self, content, file_path):
>> + """
>> + Write some content to a file.
>> +
>> + @param content: Content to be written.
>> + @param file_path: Path for the file.
>> + """
>> + success = False
>> + try:
>> + file = open(file_path, 'w')
>> + except IOError:
>> + return success
>> +
>> + try:
>> + file.write(content)
>> + success = True
>> + except IOError:
>> + pass
>> + finally:
>> + file.close()
>> +
>> + return success
>> +
>> +
>> + def is_binded_to_stub(self, full_id):
>> + """
>> + Verify whether the device with full_id is already binded to pci-stub.
>> + """
>> + base_dir = "/sys/bus/pci"
>> + stub_path = os.path.join(base_dir, "drivers/pci-stub")
>> + if os.path.exists(os.path.join(stub_path, full_id)):
>> + return True
>> + return False
>> +
>> +
>> + def request_devs(self, count, pt_file, setup=True):
>> + """
>> + Implement setup process: unbind the PCI device and then bind it
>> + to pci-stub driver.
>> +
>> + @return: a list of successfully requested devices' PCI IDs.
>> + """
>> + base_dir = "/sys/bus/pci"
>> + stub_path = os.path.join(base_dir, "drivers/pci-stub")
>> +
>> + self.pci_ids = self.catch_devs(int(count))
>> + logging.debug("Catch_devs has found pci_ids: %s" % self.pci_ids)
>> +
>> + if setup:
>> + requested_pci_ids = []
>> + logging.debug("Going to setup devices: %s" % self.pci_ids)
>> + dev_prev_drivers = {}
>> +
>> + # Setup all devices specified for passthrough to guest
>> + for pci_id in self.pci_ids:
>> + full_id = get_full_id(pci_id)
>> + if not full_id:
>> + continue
>> + drv_path = os.path.join(base_dir, "devices/%s/driver" % full_id)
>> + dev_prev_driver= os.path.realpath(os.path.join(drv_path,
>> + os.readlink(drv_path)))
>> + dev_prev_drivers[pci_id] = dev_prev_driver
>> +
>> + # Judge whether the device driver has been binded to stub
>> + if not self.is_binded_to_stub(full_id):
>> + logging.debug("Setting up device: %s" % full_id)
>> + stub_new_id = os.path.join(stub_path, 'new_id')
>> + unbind_dev = os.path.join(drv_path, 'unbind')
>> + stub_bind = os.path.join(stub_path, 'bind')
>> +
>> + vendor_id = get_vendor_id(pci_id)
>> + if not self._write_to_file(vendor_id, stub_new_id):
>> + continue
>> + if not self._write_to_file(full_id, unbind_dev):
>> + continue
>> + if not self._write_to_file(full_id, stub_bind):
>> + continue
>> + if not self.is_binded_to_stub(full_id):
>> + logging.error("Request device failed: %s" % pci_id)
>> + continue
>> + else:
>> + logging.debug("Device [%s] already binded to stub" % pci_id)
>> + requested_pci_ids.append(pci_id)
>> + cPickle.dump(dev_prev_drivers, pt_file)
>> + self.pci_ids = requested_pci_ids
>> + return self.pci_ids
>> +
>> +
>> + def catch_vf_devs(self):
>> + """
>> + Catch all VFs PCI IDs and return as a list.
>> + """
>> + if not self.sr_iov_setup():
>> + return []
>> +
>> + cmd = "lspci | awk '/Virtual Function/ {print $1}'"
>> + return commands.getoutput(cmd).split()
>> +
>> +
>> + def catch_pf_devs(self):
>> + """
>> + Catch all PFs PCI IDs.
>> + """
>> + pf_ids = []
>> + for name in self.name_list:
>> + pf_id = self._get_pci_id(name, "Ethernet")
>> + if not pf_id:
>> + continue
>> + pf_ids.append(pf_id)
>> + return pf_ids
>> +
>> +
>> + def catch_devs(self, count):
>> + """
>> + Check out all devices' PCI IDs according to their name.
>> +
>> + @param count: count number of PCI devices needed for pass through
>> + @return: a list of all devices' PCI IDs
>> + """
>> + if self.type == "nic_vf":
>> + vf_ids = self.catch_vf_devs()
>> + elif self.type == "nic_pf":
>> + vf_ids = self.catch_pf_devs()
>> + elif self.type == "mixed":
>> + vf_ids = self.catch_vf_devs()
>> + vf_ids.extend(self.catch_pf_devs())
>> + return vf_ids[0:count]
>> diff --git a/client/tests/kvm/kvm_vm.py b/client/tests/kvm/kvm_vm.py
>> index 82f1eb4..a0b01fe 100755
>> --- a/client/tests/kvm/kvm_vm.py
>> +++ b/client/tests/kvm/kvm_vm.py
>> @@ -298,6 +298,30 @@ class VM:
>> elif params.get("uuid"):
>> qemu_cmd += " -uuid %s" % params.get("uuid")
>>
>> + # Check whether pci_passthrough is turned on
>> + if not params.get("pass_through") == "no":
>> + pt_type = params.get("pass_through")
>> +
>> + # Virtual Functions pass through
>> + if params.get("pass_through") == "nic_vf":
>> + pt = kvm_utils.PassThrough(type=pt_type,
>> + max_vfs=params.get("max_vfs"))
>> + # Physical NIC card pass through
>> + elif params.get("pass_through") == "nic_pf":
>> + pt = kvm_utils.PassThrough(type=pt_type,
>> + names=params.get("passthrough_devs"))
>> + elif params.get("pass_through") == "mixed":
>> + pt = kvm_utils.PassThrough(type=pt_type,
>> + max_vfs=params.get("max_vfs"),
>> + names=params.get("passthrough_devs"))
>> + pt_count = params.get("passthrough_count")
>> + pt_db_file = open("/tmp/passthrough_db", "w")
>> + self.pt_pci_ids = pt.request_devs(pt_count, pt_file=pt_db_file,
>> + setup=False)
>> + pt_db_file.close()
>> + for pci_id in self.pt_pci_ids:
>> + qemu_cmd += " -pcidevice host=%s" % pci_id
>> +
>> return qemu_cmd
>>
>>
>> @@ -380,6 +404,37 @@ class VM:
>> self.uuid = f.read().strip()
>> f.close()
>>
>> + # Request specified PCI devices for assignment
>> + if not params.get("pass_through") == "no":
>> + pt_type = params.get("pass_through")
>> +
>> + # Virtual Functions pass through
>> + if params.get("pass_through") == "nic_vf":
>> + logging.debug("Going to assign VFs to guest")
>> + pt = kvm_utils.PassThrough(type=pt_type,
>> + max_vfs = params.get("max_vfs"))
>> + # Physical NIC card pass through
>> + elif params.get("pass_through") == "nic_pf":
>> + logging.debug("Going to assign physical NICs to guest")
>> + pt = kvm_utils.PassThrough(type=pt_type,
>> + names=params.get("passthrough_devs"))
>> + elif params.get("pass_through") == "mixed":
>> + logging.debug("Assigned NICs may contain both VFs and PFs")
>> + pt = kvm_utils.PassThrough(type = pt_type,
>> + max_vfs = params.get("max_vfs"),
>> + names = params.get("passthrough_devs"))
>> + pt_count = params.get("passthrough_count")
>> + pt_db_file = open('/tmp/passthrough_db', 'w')
>> + self.pt_pci_ids = pt.request_devs(pt_count, pt_file=pt_db_file,
>> + setup=True)
>> + pt_db_file.close()
>> + if not self.pt_pci_ids:
>> + logging.error("Request specified PCI device(s) failed;"
>> + " pt_pci_ids are: %s", self.pt_pci_ids)
>> + return False
>> + logging.debug("Requested host PCI device(s): %s",
>> + self.pt_pci_ids)
>> +
>> # Make qemu command
>> qemu_command = self.make_qemu_command()
>>
>> @@ -504,6 +559,19 @@ class VM:
>> return (0, data)
>>
>>
>> + def _release_pt_devs(self):
>> + """
>> + Release the assigned passthrough PCI devices or VFs to host.
>> + """
>> + try:
>> + db_file = open('/tmp/passthrough_db', 'r')
>> + pt_db = cPickle.load(db_file)
>> + kvm_utils.release_pci_devs(pt_db)
>> + db_file.close()
>> + except:
>> + return
>> +
>> +
>> def destroy(self, gracefully=True):
>> """
>> Destroy the VM.
>> @@ -520,6 +588,7 @@ class VM:
>> # Is it already dead?
>> if self.is_dead():
>> logging.debug("VM is already down")
>> + self._release_pt_devs()
>> return
>>
>> logging.debug("Destroying VM with PID %d..." %
>> @@ -540,6 +609,7 @@ class VM:
>> return
>> finally:
>> session.close()
>> + self._release_pt_devs()
>>
>> # Try to destroy with a monitor command
>> logging.debug("Trying to kill VM with monitor command...")
>> @@ -549,6 +619,7 @@ class VM:
>> # Wait for the VM to be really dead
>> if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5):
>> logging.debug("VM is down")
>> + self._release_pt_devs()
>> return
>>
>> # If the VM isn't dead yet...
>> @@ -558,6 +629,7 @@ class VM:
>> # Wait for the VM to be really dead
>> if kvm_utils.wait_for(self.is_dead, 5, 0.5, 0.5):
>> logging.debug("VM is down")
>> + self._release_pt_devs()
>> return
>>
>> logging.error("Process %s is a zombie!" % self.process.get_pid())
>> --
>> 1.6.2.5
> _______________________________________________
> Autotest mailing list
> Autotest@xxxxxxxxxxxxxxx
> http://test.kernel.org/cgi-bin/mailman/listinfo/autotest
>
--
Lucas
--
To unsubscribe from this list: send the line "unsubscribe kvm" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
