From: Edward Srouji <edwards@xxxxxxxxxx> Add a test that opens an mlx5 vfio context, creates two DevX QPs on it, and modifies them to RTS state. Then performs traffic with SEND_IMM. An additional cmd line argument was added for the tests, to allow users to pass a PCI device instead of an RDMA device for this test. Reviewed-by: Ido Kalir <idok@xxxxxxxxxx> Signed-off-by: Edward Srouji <edwards@xxxxxxxxxx> --- tests/CMakeLists.txt | 1 + tests/args_parser.py | 5 +++ tests/test_mlx5_vfio.py | 104 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+) create mode 100644 tests/test_mlx5_vfio.py diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7b079d8..cabeba0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,6 +36,7 @@ rdma_python_test(tests test_mlx5_uar.py test_mlx5_udp_sport.py test_mlx5_var.py + test_mlx5_vfio.py test_mr.py test_odp.py test_pd.py diff --git a/tests/args_parser.py b/tests/args_parser.py index 5bc53b0..4312121 100644 --- a/tests/args_parser.py +++ b/tests/args_parser.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) # Copyright (c) 2020 Kamal Heib <kamalheib1@xxxxxxxxx>, All rights reserved. See COPYING file +# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file import argparse import sys @@ -16,6 +17,10 @@ class ArgsParser(object): parser = argparse.ArgumentParser() parser.add_argument('--dev', help='RDMA device to run the tests on') + parser.add_argument('--pci-dev', + help='PCI device to run the tests on, which is ' + 'needed by some tests where the RDMA device is ' + 'not available (e.g. VFIO)') parser.add_argument('--port', help='Use port <port> of RDMA device', type=int, default=1) diff --git a/tests/test_mlx5_vfio.py b/tests/test_mlx5_vfio.py new file mode 100644 index 0000000..06da8dd --- /dev/null +++ b/tests/test_mlx5_vfio.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) +# Copyright (c) 2021 Nvidia, Inc. All rights reserved. See COPYING file +""" +Test module for pyverbs' mlx5_vfio module. +""" + +from threading import Thread +import unittest +import select +import errno + +from pyverbs.providers.mlx5.mlx5_vfio import Mlx5VfioAttr, Mlx5VfioContext +from tests.mlx5_base import Mlx5DevxRcResources, Mlx5DevxTrafficBase +from pyverbs.pyverbs_error import PyverbsRDMAError + + +class Mlx5VfioResources(Mlx5DevxRcResources): + def __init__(self, ib_port, pci_name, gid_index=None, ctx=None): + self.pci_name = pci_name + self.ctx = ctx + super().__init__(None, ib_port, gid_index) + + def create_context(self): + """ + Opens an mlx5 VFIO context. + Since only one context is allowed to be opened on a VFIO, the user must + pass that context for the remaining resources, which in that case, the + same context would be used. + :return: None + """ + if self.ctx: + return + try: + vfio_attr = Mlx5VfioAttr(pci_name=self.pci_name) + vfio_attr.pci_name = self.pci_name + self.ctx = Mlx5VfioContext(attr=vfio_attr) + except PyverbsRDMAError as ex: + if ex.error_code == errno.EOPNOTSUPP: + raise unittest.SkipTest(f'Mlx5 VFIO is not supported ({ex})') + raise ex + + def query_gid(self): + """ + Currently Mlx5VfioResources does not support Eth port type. + Query GID would just be skipped. + """ + pass + + +class Mlx5VfioTrafficTest(Mlx5DevxTrafficBase): + """ + Test various functionality of an mlx5-vfio device. + """ + def setUp(self): + """ + Verifies that the user has passed a PCI device name to work with. + """ + self.pci_dev = self.config['pci_dev'] + if not self.pci_dev: + raise unittest.SkipTest('PCI device must be passed by the user') + + def create_players(self): + self.server = Mlx5VfioResources(ib_port=self.ib_port, pci_name=self.pci_dev) + self.client = Mlx5VfioResources(ib_port=self.ib_port, pci_name=self.pci_dev, + ctx=self.server.ctx) + + def vfio_processs_events(self): + """ + Processes mlx5 vfio device events. + This method should run from application thread to maintain the events. + """ + # Server and client use the same context + events_fd = self.server.ctx.get_events_fd() + with select.epoll() as epoll_events: + epoll_events.register(events_fd, select.EPOLLIN) + while self.proc_events: + for fd, event in epoll_events.poll(timeout=0.1): + if fd == events_fd: + if not (event & select.EPOLLIN): + self.event_ex.append(PyverbsRDMAError(f'Unexpected vfio event: {event}')) + self.server.ctx.process_events() + + def test_mlx5vfio_rc_qp_send_imm_traffic(self): + """ + Opens one mlx5 vfio context, creates two DevX RC QPs on it, and modifies + them to RTS state. + Then does SEND_IMM traffic. + """ + self.create_players() + if self.server.is_eth(): + raise unittest.SkipTest(f'{self.__class__.__name__} is currently supported over IB only') + self.event_ex = [] + self.proc_events = True + proc_events = Thread(target=self.vfio_processs_events) + proc_events.start() + # Move the DevX QPs ro RTS state + self.pre_run() + # Send traffic + self.send_imm_traffic() + # Stop listening to events + self.proc_events = False + proc_events.join() + if self.event_ex: + raise PyverbsRDMAError(f'Received unexpected vfio events: {self.event_ex}') -- 1.8.3.1