2021년 12월 25일 (토) 오후 1:24, Namjae Jeon <linkinjeon@xxxxxxxxxx>님이 작성:
>
> 2021-12-25 8:38 GMT+09:00, Hyunchul Lee <hyc.lee@xxxxxxxxx>:
> > 2021년 12월 25일 (토) 오전 8:05, Namjae Jeon <linkinjeon@xxxxxxxxxx>님이 작성:
> >>
> >> 2021-12-25 7:53 GMT+09:00, Hyunchul Lee <hyc.lee@xxxxxxxxx>:
> >> > 2021년 12월 24일 (금) 오후 4:27, Namjae Jeon <linkinjeon@xxxxxxxxxx>님이 작성:
> >> >>
> >> >> 2021-12-23 7:43 GMT+09:00, Hyunchul Lee <hyc.lee@xxxxxxxxx>:
> >> >> > Set the rdma capability using ib_clien, Because
> >> >> > For some devices, ib_device_get_by_netdev() returns NULL.
> >> >> >
> >> >> > Signed-off-by: Hyunchul Lee <hyc.lee@xxxxxxxxx>
> >> >> > ---
> >> >> > fs/ksmbd/transport_rdma.c | 89
> >> >> > ++++++++++++++++++++++++++++++++++++---
> >> >> > 1 file changed, 83 insertions(+), 6 deletions(-)
> >> >> >
> >> >> > diff --git a/fs/ksmbd/transport_rdma.c b/fs/ksmbd/transport_rdma.c
> >> >> > index 7e57cbb0bb35..0820c6a9d75e 100644
> >> >> > --- a/fs/ksmbd/transport_rdma.c
> >> >> > +++ b/fs/ksmbd/transport_rdma.c
> >> >> > @@ -79,6 +79,14 @@ static int smb_direct_max_read_write_size = 1024
> >> >> > *
> >> >> > 1024;
> >> >> >
> >> >> > static int smb_direct_max_outstanding_rw_ops = 8;
> >> >> >
> >> >> > +static LIST_HEAD(smb_direct_device_list);
> >> >> > +static DEFINE_RWLOCK(smb_direct_device_lock);
> >> >> > +
> >> >> > +struct smb_direct_device {
> >> >> > + struct ib_device *ib_dev;
> >> >> > + struct list_head list;
> >> >> > +};
> >> >> > +
> >> >> > static struct smb_direct_listener {
> >> >> > struct rdma_cm_id *cm_id;
> >> >> > } smb_direct_listener;
> >> >> > @@ -2007,12 +2015,62 @@ static int smb_direct_listen(int port)
> >> >> > return ret;
> >> >> > }
> >> >> >
> >> >> > +static int smb_direct_ib_client_add(struct ib_device *ib_dev)
> >> >> > +{
> >> >> > + struct smb_direct_device *smb_dev;
> >> >> > +
> >> >> > + if (!ib_dev->ops.get_netdev ||
> >> >> > + ib_dev->node_type != RDMA_NODE_IB_CA ||
> >> >> > + !rdma_frwr_is_supported(&ib_dev->attrs))
> >> >> > + return 0;
> >> >> > +
> >> >> > + smb_dev = kzalloc(sizeof(*smb_dev), GFP_KERNEL);
> >> >> > + if (!smb_dev)
> >> >> > + return -ENOMEM;
> >> >> > + smb_dev->ib_dev = ib_dev;
> >> >> > +
> >> >> > + write_lock(&smb_direct_device_lock);
> >> >> > + list_add(&smb_dev->list, &smb_direct_device_list);
> >> >> > + write_unlock(&smb_direct_device_lock);
> >> >> > +
> >> >> > + ksmbd_debug(RDMA, "ib device added: name %s\n", ib_dev->name);
> >> >> > + return 0;
> >> >> > +}
> >> >> > +
> >> >> > +static void smb_direct_ib_client_remove(struct ib_device *ib_dev,
> >> >> > + void *client_data)
> >> >> > +{
> >> >> When is this function called? Is rdma shutdown needed here?
> >> >>
> >> >
> >> > This is called when a device is unplugged. After this call,
> >> > RDMA_CM_EVENT_DEVICE_REMOVAL event is generated.
> >> > And ksmbd handles this event.
> >> Okay, Have you actually tested it while unplugging NICs?
> >> And This patch doesn't work on Chelsio NICs(iWARP).
> >>
> >
> > Not yet. I will test it but need some time because of
> > setting up my test environment.
> Don't send untested patch to the list.
> >
> > Is the rdma capability set on Chelsio NICs without
> > this patch?
> Yes.
Okay, After testing device removal and fixing this,
I will send the v2 patch.
> >
> >> >
> >> >> > + struct smb_direct_device *smb_dev, *tmp;
> >> >> > +
> >> >> > + write_lock(&smb_direct_device_lock);
> >> >> > + list_for_each_entry_safe(smb_dev, tmp,
> >> >> > &smb_direct_device_list,
> >> >> > list) {
> >> >> > + if (smb_dev->ib_dev == ib_dev) {
> >> >> > + list_del(&smb_dev->list);
> >> >> > + kfree(smb_dev);
> >> >> > + break;
> >> >> > + }
> >> >> > + }
> >> >> > + write_unlock(&smb_direct_device_lock);
> >> >> > +}
> >> >> > +
> >> >> > +static struct ib_client smb_direct_ib_client = {
> >> >> > + .name = "ksmbd_smb_direct_ib",
> >> >> Move smb_direct_ib_client_add and smb_direct_ib_client_remove to here.
> >> >>
> >> >> > +};
> >> >> > +
> >> >> > int ksmbd_rdma_init(void)
> >> >> > {
> >> >> > int ret;
> >> >> >
> >> >> > smb_direct_listener.cm_id = NULL;
> >> >> >
> >> >> > + smb_direct_ib_client.add = smb_direct_ib_client_add;
> >> >> > + smb_direct_ib_client.remove = smb_direct_ib_client_remove;
> >> >> > + ret = ib_register_client(&smb_direct_ib_client);
> >> >> > + if (ret) {
> >> >> > + pr_err("failed to ib_register_client\n");
> >> >> > + return ret;
> >> >> > + }
> >> >> > +
> >> >> > /* When a client is running out of send credits, the credits
> >> >> > are
> >> >> > * granted by the server's sending a packet using this queue.
> >> >> > * This avoids the situation that a clients cannot send
> >> >> > packets
> >> >> > @@ -2046,20 +2104,39 @@ int ksmbd_rdma_destroy(void)
> >> >> > destroy_workqueue(smb_direct_wq);
> >> >> > smb_direct_wq = NULL;
> >> >> > }
> >> >> > +
> >> >> > + if (smb_direct_ib_client.add)
> >> >> > + ib_unregister_client(&smb_direct_ib_client);
> >> >> > + smb_direct_ib_client.add = NULL;
> >> >> Why ?
> >> >>
> >> >
> >> > To avoid multiple calls of ib_unregister_client, because
> >> > ksmbd_rdma_destory() can be called without ksmbd_rdma_init().
> >> > I will clean two functions and remove these statements.
> >> >
> >> >> > return 0;
> >> >> > }
> >> >> >
> >> >> > bool ksmbd_rdma_capable_netdev(struct net_device *netdev)
> >> >> > {
> >> >> > - struct ib_device *ibdev;
> >> >> > + struct smb_direct_device *smb_dev;
> >> >> > + int i;
> >> >> > bool rdma_capable = false;
> >> >> >
> >> >> > - ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_UNKNOWN);
> >> >> > - if (ibdev) {
> >> >> > - if (rdma_frwr_is_supported(&ibdev->attrs))
> >> >> > - rdma_capable = true;
> >> >> > - ib_device_put(ibdev);
> >> >> > + read_lock(&smb_direct_device_lock);
> >> >> > + list_for_each_entry(smb_dev, &smb_direct_device_list, list) {
> >> >> > + for (i = 0; i < smb_dev->ib_dev->phys_port_cnt; i++) {
> >> >> > + struct net_device *ndev;
> >> >> > +
> >> >> > + ndev =
> >> >> > smb_dev->ib_dev->ops.get_netdev(smb_dev->ib_dev,
> >> >> > + i + 1);
> >> >> > + if (!ndev)
> >> >> > + continue;
> >> >> > +
> >> >> > + if (ndev == netdev) {
> >> >> > + dev_put(ndev);
> >> >> > + rdma_capable = true;
> >> >> > + goto out;
> >> >> > + }
> >> >> > + dev_put(ndev);
> >> >> > + }
> >> >> > }
> >> >> > +out:
> >> >> > + read_unlock(&smb_direct_device_lock);
> >> >> > return rdma_capable;
> >> >> > }
> >> >> >
> >> >> > --
> >> >> > 2.25.1
> >> >> >
> >> >> >
> >> >
> >> >
> >> >
> >> > --
> >> > Thanks,
> >> > Hyunchul
> >> >
> >
> >
> >
> > --
> > Thanks,
> > Hyunchul
> >
--
Thanks,
Hyunchul
