Hi Doug, Christoph, Sagi, Or and all,
I completely forgot to write a cover letter for this series, my bad.
This is an attempt to make a device pool out of the common code
which is used on nvme (host, target) and iser (host, target) and
caches the ib_device and ib_pd. This device pool api would be used
for IBTRS (rdma transport library) as well.
Guys, could you please take a look?
--
Roman
On Tue, Apr 24, 2018 at 4:58 PM, Roman Pen
<roman.penyaev@xxxxxxxxxxxxxxxx> wrote:
> Logic for caching ib_device,ib_pd pair by node_guid is used for nvme,
> nvmet, iser and isert. Make a common functionality out of that.
>
> Signed-off-by: Roman Pen <roman.penyaev@xxxxxxxxxxxxxxxx>
> Cc: Christoph Hellwig <hch@xxxxxx>
> Cc: Steve Wise <swise@xxxxxxxxxxxxxxxxxxxxx>
> Cc: Bart Van Assche <bart.vanassche@xxxxxxxxxxx>
> Cc: Sagi Grimberg <sagi@xxxxxxxxxxx>
> Cc: Doug Ledford <dledford@xxxxxxxxxx>
> ---
> drivers/infiniband/core/Makefile | 2 +-
> drivers/infiniband/core/dev_pool.c | 136 +++++++++++++++++++++++++++++++++++++
> include/rdma/dev_pool.h | 62 +++++++++++++++++
> 3 files changed, 199 insertions(+), 1 deletion(-)
> create mode 100644 drivers/infiniband/core/dev_pool.c
> create mode 100644 include/rdma/dev_pool.h
>
> diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
> index dda9e856e3fa..8068c641e48d 100644
> --- a/drivers/infiniband/core/Makefile
> +++ b/drivers/infiniband/core/Makefile
> @@ -9,7 +9,7 @@ obj-$(CONFIG_INFINIBAND_USER_ACCESS) += ib_uverbs.o ib_ucm.o \
> $(user_access-y)
>
> ib_core-y := packer.o ud_header.o verbs.o cq.o rw.o sysfs.o \
> - device.o fmr_pool.o cache.o netlink.o \
> + device.o fmr_pool.o cache.o netlink.o dev_pool.o \
> roce_gid_mgmt.o mr_pool.o addr.o sa_query.o \
> multicast.o mad.o smi.o agent.o mad_rmpp.o \
> security.o nldev.o restrack.o
> diff --git a/drivers/infiniband/core/dev_pool.c b/drivers/infiniband/core/dev_pool.c
> new file mode 100644
> index 000000000000..641c8ad45590
> --- /dev/null
> +++ b/drivers/infiniband/core/dev_pool.c
> @@ -0,0 +1,136 @@
> +/*
> + * Copyright (c) 2018 ProfitBricks GmbH. All rights reserved.
> + *
> + * This program is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU General Public License
> + * as published by the Free Software Foundation; either version 2
> + * of the License, or (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU General Public License for more details.
> + */
> +
> +#include <rdma/dev_pool.h>
> +
> +void ib_pool_dev_init(enum ib_pd_flags pd_flags,
> + struct ib_pool_device *(*alloc)(void),
> + void (*free)(struct ib_pool_device *),
> + int (*init)(struct ib_pool_device *),
> + struct ib_device_pool *pool)
> +{
> + INIT_LIST_HEAD(&pool->list);
> + mutex_init(&pool->mutex);
> +
> + pool->pd_flags = pd_flags;
> +
> + BUG_ON(!alloc ^ !free);
> + pool->alloc = alloc;
> + pool->free = free;
> + pool->init = init;
> +}
> +EXPORT_SYMBOL(ib_pool_dev_init);
> +
> +void ib_pool_dev_deinit(struct ib_device_pool *pool)
> +{
> + WARN_ON(!list_empty(&pool->list));
> +}
> +EXPORT_SYMBOL(ib_pool_dev_deinit);
> +
> +static void dev_free(struct kref *ref)
> +{
> + struct ib_pool_device *dev;
> + struct ib_device_pool *pool;
> + struct ib_pd *ib_pd;
> +
> + dev = container_of(ref, typeof(*dev), ref);
> + pool = dev->pool;
> +
> + mutex_lock(&pool->mutex);
> + list_del(&dev->entry);
> + mutex_unlock(&pool->mutex);
> +
> + ib_pd = dev->ib_pd;
> + if (pool->free)
> + pool->free(dev);
> + else
> + kfree(dev);
> +
> + ib_dealloc_pd(ib_pd);
> +}
> +
> +int ib_pool_dev_put(struct ib_pool_device *dev)
> +{
> + return kref_put(&dev->ref, dev_free);
> +}
> +EXPORT_SYMBOL(ib_pool_dev_put);
> +
> +int ib_pool_dev_get(struct ib_pool_device *dev)
> +{
> + return kref_get_unless_zero(&dev->ref);
> +}
> +EXPORT_SYMBOL(ib_pool_dev_get);
> +
> +bool ib_pool_dev_exists(struct ib_device *ib_dev,
> + struct ib_device_pool *pool)
> +{
> + struct ib_pool_device *dev;
> + bool found = false;
> +
> + mutex_lock(&pool->mutex);
> + list_for_each_entry(dev, &pool->list, entry) {
> + if (dev->ib_dev == ib_dev) {
> + found = true;
> + break;
> + }
> + }
> + mutex_unlock(&pool->mutex);
> +
> + return false;
> +}
> +EXPORT_SYMBOL(ib_pool_dev_exists);
> +
> +struct ib_pool_device *
> +ib_pool_dev_find_get_or_create(struct ib_device *ib_dev,
> + struct ib_device_pool *pool)
> +{
> + struct ib_pool_device *dev;
> +
> + mutex_lock(&pool->mutex);
> + list_for_each_entry(dev, &pool->list, entry) {
> + if (dev->ib_dev->node_guid == ib_dev->node_guid &&
> + ib_pool_dev_get(dev))
> + goto out_unlock;
> + }
> + if (pool->alloc)
> + dev = pool->alloc();
> + else
> + dev = kzalloc(sizeof(*dev), GFP_KERNEL);
> + if (unlikely(IS_ERR_OR_NULL(dev)))
> + goto out_err;
> +
> + kref_init(&dev->ref);
> + dev->pool = pool;
> + dev->ib_dev = ib_dev;
> + dev->ib_pd = ib_alloc_pd(ib_dev, pool->pd_flags);
> + if (unlikely(IS_ERR(dev->ib_pd)))
> + goto out_free_dev;
> +
> + if (pool->init && pool->init(dev))
> + goto out_free_pd;
> +
> + list_add(&dev->entry, &pool->list);
> +out_unlock:
> + mutex_unlock(&pool->mutex);
> + return dev;
> +
> +out_free_pd:
> + ib_dealloc_pd(dev->ib_pd);
> +out_free_dev:
> + pool->free ? pool->free(dev) : kfree(dev);
> +out_err:
> + mutex_unlock(&pool->mutex);
> + return NULL;
> +}
> +EXPORT_SYMBOL(ib_pool_dev_find_get_or_create);
> diff --git a/include/rdma/dev_pool.h b/include/rdma/dev_pool.h
> new file mode 100644
> index 000000000000..74fd27b8ed25
> --- /dev/null
> +++ b/include/rdma/dev_pool.h
> @@ -0,0 +1,62 @@
> +/*
> + * Copyright (c) 2018 ProfitBricks GmbH. All rights reserved.
> + *
> + * This program is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU General Public License
> + * as published by the Free Software Foundation; either version 2
> + * of the License, or (at your option) any later version.
> + *
> + * This program is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
> + * GNU General Public License for more details.
> + */
> +
> +#ifndef __DEV_POOL__
> +#define __DEV_POOL__
> +
> +#include <linux/mutex.h>
> +#include <linux/list.h>
> +#include <linux/kref.h>
> +#include <rdma/rdma_cm.h>
> +
> +struct ib_pool_device;
> +
> +struct ib_device_pool {
> + struct mutex mutex;
> + struct list_head list;
> +
> + enum ib_pd_flags pd_flags;
> +
> + struct ib_pool_device *(*alloc)(void);
> + void (*free)(struct ib_pool_device *);
> + int (*init)(struct ib_pool_device *);
> +};
> +
> +struct ib_pool_device {
> + struct ib_device *ib_dev;
> + struct ib_pd *ib_pd;
> + struct kref ref;
> + struct list_head entry;
> + struct ib_device_pool *pool;
> +};
> +
> +void ib_pool_dev_init(enum ib_pd_flags pd_flags,
> + struct ib_pool_device *(*alloc)(void),
> + void (*free)(struct ib_pool_device *),
> + int (*init)(struct ib_pool_device *),
> + struct ib_device_pool *pool);
> +
> +void ib_pool_dev_deinit(struct ib_device_pool *pool);
> +
> +struct ib_pool_device *
> +ib_pool_dev_find_get_or_create(struct ib_device *ib_dev,
> + struct ib_device_pool *pool);
> +
> +bool ib_pool_dev_exists(struct ib_device *ib_dev,
> + struct ib_device_pool *pool);
> +
> +int ib_pool_dev_get(struct ib_pool_device *dev);
> +int ib_pool_dev_put(struct ib_pool_device *dev);
> +
> +#endif /* __DEV_POOL__ */
> --
> 2.13.1
>
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
