On 05-Dec-18 22:53, Parav Pandit wrote:
>
>
>> -----Original Message-----
>> From: linux-rdma-owner@xxxxxxxxxxxxxxx <linux-rdma-
>> owner@xxxxxxxxxxxxxxx> On Behalf Of Gal Pressman
>> Sent: Tuesday, December 4, 2018 6:04 AM
>> To: Doug Ledford <dledford@xxxxxxxxxx>; Jason Gunthorpe <jgg@xxxxxxxx>
>> Cc: Alexander Matushevsky <matua@xxxxxxxxxx>; Yossi Leybovich
>> <sleybo@xxxxxxxxxx>; linux-rdma@xxxxxxxxxxxxxxx; Tom Tucker
>> <tom@xxxxxxxxxxxxxxxxxxxxx>; Gal Pressman <galpress@xxxxxxxxxx>
>> Subject: [PATCH rdma-next 11/13] RDMA/efa: Add EFA verbs implementation
>>
>> Add a file that implements the EFA verbs.
>>
>> Signed-off-by: Gal Pressman <galpress@xxxxxxxxxx>
>> ---
>> drivers/infiniband/hw/efa/efa_verbs.c | 1827
>> +++++++++++++++++++++++++++++++++
>> 1 file changed, 1827 insertions(+)
>> create mode 100644 drivers/infiniband/hw/efa/efa_verbs.c
>>
>> diff --git a/drivers/infiniband/hw/efa/efa_verbs.c
>> b/drivers/infiniband/hw/efa/efa_verbs.c
>> new file mode 100644
>> index 000000000000..ec887648060e
>> --- /dev/null
>> +++ b/drivers/infiniband/hw/efa/efa_verbs.c
>> @@ -0,0 +1,1827 @@
>> +// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
>> +/*
>> + * Copyright 2018 Amazon.com, Inc. or its affiliates.
>> + */
>> +
>
> [..]
>
>> +static int efa_get_ah_by_id(struct efa_dev *dev, u8 *id,
>> + u16 *ah_res, bool ref_update)
>> +{
>> + struct efa_ah_id *ah_id;
>> +
>> + list_for_each_entry(ah_id, &dev->efa_ah_list, list) {
>> + if (efa_ah_id_equal(ah_id->id, id)) {
>> + *ah_res = ah_id->address_handle;
>> + if (ref_update)
>> + ah_id->ref_count++;
>> + return 0;
>> + }
>> + }
>> +
>> + return -EINVAL;
>> +}
>> +
>> +static int efa_add_ah_id(struct efa_dev *dev, u8 *id,
>> + u16 address_handle)
>> +{
>> + struct efa_ah_id *ah_id;
>> +
>> + ah_id = kzalloc(sizeof(*ah_id), GFP_KERNEL);
>> + if (!ah_id)
>> + return -ENOMEM;
>> +
>> + memcpy(ah_id->id, id, sizeof(ah_id->id));
>> + ah_id->address_handle = address_handle;
>> + ah_id->ref_count = 1;
>> + list_add_tail(&ah_id->list, &dev->efa_ah_list);
>> +
>> + return 0;
>> +}
>> +
>> +static void efa_remove_ah_id(struct efa_dev *dev, u8 *id, u32
>> +*ref_count) {
>> + struct efa_ah_id *ah_id, *tmp;
>> +
>> + list_for_each_entry_safe(ah_id, tmp, &dev->efa_ah_list, list) {
>> + if (efa_ah_id_equal(ah_id->id, id)) {
>> + *ref_count = --ah_id->ref_count;
>> + if (ah_id->ref_count == 0) {
>> + list_del(&ah_id->list);
>> + kfree(ah_id);
>> + return;
>> + }
>> + }
>> + }
>> +}
>> +
>> +static void ah_destroy_on_device(struct efa_dev *dev, u16 device_ah) {
>> + struct efa_com_destroy_ah_params params;
>> + int err;
>> +
>> + params.ah = device_ah;
>> + err = efa_com_destroy_ah(dev->edev, ¶ms);
>> + if (err)
>> + pr_err("efa_com_destroy_ah failed (%d)\n", err); }
>> +
>> +static int efa_create_ah_id(struct efa_dev *dev, u8 *id,
>> + u16 *efa_address_handle)
>> +{
>> + struct efa_com_create_ah_params params = {};
>> + struct efa_com_create_ah_result result = {};
>> + int err;
>> +
>> + mutex_lock(&dev->ah_list_lock);
>> + err = efa_get_ah_by_id(dev, id, efa_address_handle, true);
>> + if (err) {
>> + memcpy(params.dest_addr, id, sizeof(params.dest_addr));
>> + err = efa_com_create_ah(dev->edev, ¶ms, &result);
>> + if (err) {
>> + pr_err("efa_com_create_ah failed %d\n", err);
>> + goto err_unlock;
>> + }
>> +
>> + pr_debug("create address handle %u for address %pI6\n",
>> + result.ah, params.dest_addr);
>> +
>> + err = efa_add_ah_id(dev, id, result.ah);
>> + if (err) {
>> + pr_err("efa_add_ah_id failed %d\n", err);
>> + goto err_destroy_ah;
>> + }
>> +
>> + *efa_address_handle = result.ah;
>> + }
>> + mutex_unlock(&dev->ah_list_lock);
>> +
>> + return 0;
>> +
>> +err_destroy_ah:
>> + ah_destroy_on_device(dev, result.ah);
>> +err_unlock:
>> + mutex_unlock(&dev->ah_list_lock);
>> + return err;
>> +}
>> +
>> +static void efa_destroy_ah_id(struct efa_dev *dev, u8 *id) {
>> + u16 device_ah;
>> + u32 ref_count;
>> + int err;
>> +
>> + mutex_lock(&dev->ah_list_lock);
>
> Create and destroy ah for kernel consumers cannot sleep.
> Though I understand that currently you don't have kernel consumers.
> This should be changed to spin lock along with other allocation to GFP_ATOMIC.
Create/destroy AH must be communicated with our device (in the form of an admin
command), this requires us to sleep regardless of the mutex/memory allocation
(we shouldn't busy wait while waiting for the device to complete the command).
None of the currently supported flows are done in an atomic context, we will
have to figure that out when we add support for those.
>
>> + err = efa_get_ah_by_id(dev, id, &device_ah, false);
>> + if (err) {
>> + WARN_ON(1);
>> + goto out_unlock;
>> + }
>> +
>> + efa_remove_ah_id(dev, id, &ref_count);
>> + if (!ref_count)
>> + ah_destroy_on_device(dev, device_ah);
>> +
>> +out_unlock:
>> + mutex_unlock(&dev->ah_list_lock);
>> +}
>> +
>> +struct ib_ah *efa_create_ah(struct ib_pd *ibpd,
>> + struct rdma_ah_attr *ah_attr,
>> + struct ib_udata *udata)
>> +{
>> + struct efa_dev *dev = to_edev(ibpd->device);
>> + struct efa_ibv_create_ah_resp resp = {};
>> + u16 efa_address_handle;
>> + struct efa_ah *ah;
>> + int err;
>> +
>> + pr_debug("--->\n");
>> +
>> + if (udata && udata->inlen &&
>> + !ib_is_udata_cleared(udata, 0, udata->inlen)) {
>> + pr_err_ratelimited("Incompatiable ABI params\n");
>> + return ERR_PTR(-EINVAL);
>> + }
>> +
>> + ah = kzalloc(sizeof(*ah), GFP_KERNEL);
>> + if (!ah) {
>> + dev->stats.sw_stats.create_ah_alloc_err++;
>> + return ERR_PTR(-ENOMEM);
>> + }
>> +
>> + err = efa_create_ah_id(dev, ah_attr->grh.dgid.raw,
>> &efa_address_handle);
>> + if (err)
>> + goto err_free;
>> +
>> + resp.efa_address_handle = efa_address_handle;
>> +
>> + if (udata && udata->outlen) {
>> + err = ib_copy_to_udata(udata, &resp,
>> + min(sizeof(resp), udata->outlen));
>> + if (err) {
>> + pr_err_ratelimited("failed to copy udata for
>> create_ah response\n");
>> + goto err_destroy_ah;
>> + }
>> + }
>> +
>> + memcpy(ah->id, ah_attr->grh.dgid.raw, sizeof(ah->id));
>> + return &ah->ibah;
>> +
>> +err_destroy_ah:
>> + efa_destroy_ah_id(dev, ah_attr->grh.dgid.raw);
>> +err_free:
>> + kfree(ah);
>> + return ERR_PTR(err);
>> +}
>> +
