From: Edward Srouji <edwards@xxxxxxxxxx>
Add a test that opens an mlx5 vfio context, creates two DevX QPs on
it, and modifies them to RTS state. Then performs traffic with SEND_IMM.
An additional cmd line argument was added for the tests, to allow users
to pass a PCI device instead of an RDMA device for this test.
Reviewed-by: Ido Kalir <idok@xxxxxxxxxx>
Signed-off-by: Edward Srouji <edwards@xxxxxxxxxx>
---
tests/CMakeLists.txt | 1 +
tests/args_parser.py | 5 +++
tests/test_mlx5_vfio.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 110 insertions(+)
create mode 100644 tests/test_mlx5_vfio.py
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 7b079d8..cabeba0 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -36,6 +36,7 @@ rdma_python_test(tests
test_mlx5_uar.py
test_mlx5_udp_sport.py
test_mlx5_var.py
+ test_mlx5_vfio.py
test_mr.py
test_odp.py
test_pd.py
diff --git a/tests/args_parser.py b/tests/args_parser.py
index 5bc53b0..4312121 100644
--- a/tests/args_parser.py
+++ b/tests/args_parser.py
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
# Copyright (c) 2020 Kamal Heib <kamalheib1@xxxxxxxxx>, All rights reserved. See COPYING file
+# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file
import argparse
import sys
@@ -16,6 +17,10 @@ class ArgsParser(object):
parser = argparse.ArgumentParser()
parser.add_argument('--dev',
help='RDMA device to run the tests on')
+ parser.add_argument('--pci-dev',
+ help='PCI device to run the tests on, which is '
+ 'needed by some tests where the RDMA device is '
+ 'not available (e.g. VFIO)')
parser.add_argument('--port',
help='Use port <port> of RDMA device', type=int,
default=1)
diff --git a/tests/test_mlx5_vfio.py b/tests/test_mlx5_vfio.py
new file mode 100644
index 0000000..06da8dd
--- /dev/null
+++ b/tests/test_mlx5_vfio.py
@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB)
+# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file
+"""
+Test module for pyverbs' mlx5_vfio module.
+"""
+
+from threading import Thread
+import unittest
+import select
+import errno
+
+from pyverbs.providers.mlx5.mlx5_vfio import Mlx5VfioAttr, Mlx5VfioContext
+from tests.mlx5_base import Mlx5DevxRcResources, Mlx5DevxTrafficBase
+from pyverbs.pyverbs_error import PyverbsRDMAError
+
+
+class Mlx5VfioResources(Mlx5DevxRcResources):
+ def __init__(self, ib_port, pci_name, gid_index=None, ctx=None):
+ self.pci_name = pci_name
+ self.ctx = ctx
+ super().__init__(None, ib_port, gid_index)
+
+ def create_context(self):
+ """
+ Opens an mlx5 VFIO context.
+ Since only one context is allowed to be opened on a VFIO, the user must
+ pass that context for the remaining resources, which in that case, the
+ same context would be used.
+ :return: None
+ """
+ if self.ctx:
+ return
+ try:
+ vfio_attr = Mlx5VfioAttr(pci_name=self.pci_name)
+ vfio_attr.pci_name = self.pci_name
+ self.ctx = Mlx5VfioContext(attr=vfio_attr)
+ except PyverbsRDMAError as ex:
+ if ex.error_code == errno.EOPNOTSUPP:
+ raise unittest.SkipTest(f'Mlx5 VFIO is not supported ({ex})')
+ raise ex
+
+ def query_gid(self):
+ """
+ Currently Mlx5VfioResources does not support Eth port type.
+ Query GID would just be skipped.
+ """
+ pass
+
+
+class Mlx5VfioTrafficTest(Mlx5DevxTrafficBase):
+ """
+ Test various functionality of an mlx5-vfio device.
+ """
+ def setUp(self):
+ """
+ Verifies that the user has passed a PCI device name to work with.
+ """
+ self.pci_dev = self.config['pci_dev']
+ if not self.pci_dev:
+ raise unittest.SkipTest('PCI device must be passed by the user')
+
+ def create_players(self):
+ self.server = Mlx5VfioResources(ib_port=self.ib_port, pci_name=self.pci_dev)
+ self.client = Mlx5VfioResources(ib_port=self.ib_port, pci_name=self.pci_dev,
+ ctx=self.server.ctx)
+
+ def vfio_processs_events(self):
+ """
+ Processes mlx5 vfio device events.
+ This method should run from application thread to maintain the events.
+ """
+ # Server and client use the same context
+ events_fd = self.server.ctx.get_events_fd()
+ with select.epoll() as epoll_events:
+ epoll_events.register(events_fd, select.EPOLLIN)
+ while self.proc_events:
+ for fd, event in epoll_events.poll(timeout=0.1):
+ if fd == events_fd:
+ if not (event & select.EPOLLIN):
+ self.event_ex.append(PyverbsRDMAError(f'Unexpected vfio event: {event}'))
+ self.server.ctx.process_events()
+
+ def test_mlx5vfio_rc_qp_send_imm_traffic(self):
+ """
+ Opens one mlx5 vfio context, creates two DevX RC QPs on it, and modifies
+ them to RTS state.
+ Then does SEND_IMM traffic.
+ """
+ self.create_players()
+ if self.server.is_eth():
+ raise unittest.SkipTest(f'{self.__class__.__name__} is currently supported over IB only')
+ self.event_ex = []
+ self.proc_events = True
+ proc_events = Thread(target=self.vfio_processs_events)
+ proc_events.start()
+ # Move the DevX QPs ro RTS state
+ self.pre_run()
+ # Send traffic
+ self.send_imm_traffic()
+ # Stop listening to events
+ self.proc_events = False
+ proc_events.join()
+ if self.event_ex:
+ raise PyverbsRDMAError(f'Received unexpected vfio events: {self.event_ex}')
--
1.8.3.1
