From: Leon Romanovsky <leonro@xxxxxxxxxxxx>
The sysfs layout is created by CM incorrectly presented RDMA devices
with InfiniBand link layer. Layout of such devices represents device
tree of connections. By moving CM statistics to be under relevant port
of IB device, we will fix the following issues:
* Symlink name - It used device name instead of specific identifier.
* Target location - It was supposed to point to PCI-ID/infiniband_cm/
instead of PCI-ID/infiniband/
* Target name - It created extra device file under already existing
device folder, e.g. mlx5_0/mlx5_0
* Splash during boot with RDMA persistent naming patches.
[ 21.169052] sysfs: cannot create duplicate filename '/class/infiniband_cm/mlx5_0'
[ 21.169597] CPU: 29 PID: 433 Comm: modprobe Not tainted 5.0.0-rc5+ #178
[ 21.169957] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS ?-20180724_192412-buildhw-07.phx2.fedoraproject.org-1.fc29 04/01/2014
[ 21.170569] Call Trace:
[ 21.170703] dump_stack+0xcc/0x180
[ 21.171224] sysfs_warn_dup.cold.3+0x17/0x2d
[ 21.171510] sysfs_do_create_link_sd.isra.2+0xd0/0xf0
[ 21.171787] device_add+0x7cb/0x1450
[ 21.173167] device_create_groups_vargs+0x1ae/0x220
[ 21.173404] device_create+0x93/0xc0
[ 21.174348] cm_add_one+0x38f/0xf60 [ib_cm]
[ 21.175841] add_client_context+0x167/0x210 [ib_core]
[ 21.176111] enable_device_and_get+0x230/0x3f0 [ib_core]
[ 21.177994] ib_register_device+0x823/0xbf0 [ib_core]
[ 21.178499] __mlx5_ib_add+0x45/0x150 [mlx5_ib]
[ 21.178754] mlx5_ib_add+0x1b3/0x5e0 [mlx5_ib]
[ 21.179821] mlx5_add_device+0x130/0x3a0 [mlx5_core]
[ 21.180649] mlx5_register_interface+0x1a9/0x270 [mlx5_core]
[ 21.181419] do_one_initcall+0x14f/0x5de
[ 21.185096] do_init_module+0x247/0x7c0
[ 21.185441] load_module+0x4c2f/0x60d0
[ 21.190521] entry_SYSCALL_64_after_hwframe+0x49/0xbe
[ 21.190706] RIP: 0033:0x7f4ddc24eefd
[ 21.190848] Code: 00 c3 66 2e 0f 1f 84 00 00 00 00 00 90 f3 0f 1e fa
48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f
05 <48> 3d 01 f0 ff ff 73 01 c3 48 8b 0d 5b 7f 0c 00 f7 d8 64 89 01 48
[ 21.191481] RSP: 002b:00007ffd82110ac8 EFLAGS: 00000246 ORIG_RAX: 0000000000000139
[ 21.191771] RAX: ffffffffffffffda RBX: 0000559aaa577980 RCX: 00007f4ddc24eefd
[ 21.192062] RDX: 0000000000000000 RSI: 0000559aa91a4346 RDI: 0000000000000001
[ 21.192349] RBP: 0000559aa91a4346 R08: 0000000000000000 R09: 0000000000000000
[ 21.192627] R10: 0000000000000001 R11: 0000000000000246 R12: 0000000000000000
[ 21.192905] R13: 0000559aaa577a00 R14: 0000000000040000 R15: 0000000000000000
After this change:
[leonro@server ~]$ ls -al /sys/class/infiniband/ibp0s12f0/ports/1/
drwxr-xr-x 2 root root 0 Mar 11 11:17 cm_rx_duplicates
drwxr-xr-x 2 root root 0 Mar 11 11:17 cm_rx_msgs
drwxr-xr-x 2 root root 0 Mar 11 11:17 cm_tx_msgs
drwxr-xr-x 2 root root 0 Mar 11 11:17 cm_tx_retries
Fixes: 110cf374a809 ("infiniband: make cm_device use a struct device and not a kobject.")
Signed-off-by: Leon Romanovsky <leonro@xxxxxxxxxxxx>
---
Changelog:
v0->v1:
* Rewrote https://patchwork.kernel.org/patch/10735545/
---
drivers/infiniband/core/cm.c | 46 +++++------------------------
drivers/infiniband/core/core_priv.h | 4 +++
drivers/infiniband/core/sysfs.c | 43 +++++++++++++++++++++++++++
3 files changed, 55 insertions(+), 38 deletions(-)
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index b9416a6fca36..44066dc7a242 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -52,6 +52,7 @@
#include <rdma/ib_cache.h>
#include <rdma/ib_cm.h>
#include "cm_msgs.h"
+#include "core_priv.h"
MODULE_AUTHOR("Sean Hefty");
MODULE_DESCRIPTION("InfiniBand CM");
@@ -219,7 +220,6 @@ struct cm_port {
struct cm_device {
struct list_head list;
struct ib_device *ib_device;
- struct device *device;
u8 ack_delay;
int going_down;
struct cm_port *port[0];
@@ -4276,18 +4276,6 @@ static struct kobj_type cm_counter_obj_type = {
.default_attrs = cm_counter_default_attrs
};
-static void cm_release_port_obj(struct kobject *obj)
-{
- struct cm_port *cm_port;
-
- cm_port = container_of(obj, struct cm_port, port_obj);
- kfree(cm_port);
-}
-
-static struct kobj_type cm_port_obj_type = {
- .release = cm_release_port_obj
-};
-
static char *cm_devnode(struct device *dev, umode_t *mode)
{
if (mode)
@@ -4306,19 +4294,12 @@ static int cm_create_port_fs(struct cm_port *port)
{
int i, ret;
- ret = kobject_init_and_add(&port->port_obj, &cm_port_obj_type,
- &port->cm_dev->device->kobj,
- "%d", port->port_num);
- if (ret) {
- kfree(port);
- return ret;
- }
-
for (i = 0; i < CM_COUNTER_GROUPS; i++) {
- ret = kobject_init_and_add(&port->counter_group[i].obj,
- &cm_counter_obj_type,
- &port->port_obj,
- "%s", counter_group_names[i]);
+ ret = ib_port_register_module_stat(port->cm_dev->ib_device,
+ port->port_num,
+ &port->counter_group[i].obj,
+ &cm_counter_obj_type,
+ counter_group_names[i]);
if (ret)
goto error;
}
@@ -4327,8 +4308,7 @@ static int cm_create_port_fs(struct cm_port *port)
error:
while (i--)
- kobject_put(&port->counter_group[i].obj);
- kobject_put(&port->port_obj);
+ ib_port_unregister_module_stat(&port->counter_group[i].obj);
return ret;
}
@@ -4338,9 +4318,8 @@ static void cm_remove_port_fs(struct cm_port *port)
int i;
for (i = 0; i < CM_COUNTER_GROUPS; i++)
- kobject_put(&port->counter_group[i].obj);
+ ib_port_unregister_module_stat(&port->counter_group[i].obj);
- kobject_put(&port->port_obj);
}
static void cm_add_one(struct ib_device *ib_device)
@@ -4367,13 +4346,6 @@ static void cm_add_one(struct ib_device *ib_device)
cm_dev->ib_device = ib_device;
cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
cm_dev->going_down = 0;
- cm_dev->device = device_create(&cm_class, &ib_device->dev,
- MKDEV(0, 0), NULL,
- "%s", dev_name(&ib_device->dev));
- if (IS_ERR(cm_dev->device)) {
- kfree(cm_dev);
- return;
- }
set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
for (i = 1; i <= ib_device->phys_port_cnt; i++) {
@@ -4440,7 +4412,6 @@ static void cm_add_one(struct ib_device *ib_device)
cm_remove_port_fs(port);
}
free:
- device_unregister(cm_dev->device);
kfree(cm_dev);
}
@@ -4494,7 +4465,6 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
cm_remove_port_fs(port);
}
- device_unregister(cm_dev->device);
kfree(cm_dev);
}
diff --git a/drivers/infiniband/core/core_priv.h b/drivers/infiniband/core/core_priv.h
index d4d5f0015f9b..8a019d2e5283 100644
--- a/drivers/infiniband/core/core_priv.h
+++ b/drivers/infiniband/core/core_priv.h
@@ -346,4 +346,8 @@ int ib_setup_port_attrs(struct ib_core_device *coredev,
int rdma_compatdev_set(u8 enable);
int ib_device_set_netns_put(struct sk_buff *skb,
struct ib_device *dev, u32 ns_fd);
+int ib_port_register_module_stat(struct ib_device *device, u8 port_num,
+ struct kobject *kobj, struct kobj_type *ktype,
+ const char *name);
+void ib_port_unregister_module_stat(struct kobject *kobj);
#endif /* _CORE_PRIV_H */
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index 2ce3f58157a5..2fe89754e592 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -1354,3 +1354,46 @@ void ib_device_unregister_sysfs(struct ib_device *device)
ib_free_port_attrs(&device->coredev);
}
+
+/**
+ * ib_port_register_module_stat - add module counters under relevant port
+ * of IB device.
+ *
+ * @device: IB device to add counters
+ * @port_num: valid port number
+ * @kobj: pointer to the kobject to initialize
+ * @ktype: pointer to the ktype for this kobject.
+ * @name: the name of the kobject
+ */
+int ib_port_register_module_stat(struct ib_device *device, u8 port_num,
+ struct kobject *kobj, struct kobj_type *ktype,
+ const char *name)
+{
+ struct kobject *p, *t;
+ int ret;
+
+ list_for_each_entry_safe(p, t, &device->coredev.port_list, entry) {
+ struct ib_port *port = container_of(p, struct ib_port, kobj);
+
+ if (port->port_num != port_num)
+ continue;
+
+ ret = kobject_init_and_add(kobj, ktype, &port->kobj, "%s",
+ name);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL(ib_port_register_module_stat);
+
+/**
+ * ib_port_unregister_module_stat - release module counters
+ * @kobj: pointer to the kobject to release
+ */
+void ib_port_unregister_module_stat(struct kobject *kobj)
+{
+ kobject_put(kobj);
+}
+EXPORT_SYMBOL(ib_port_unregister_module_stat);
--
2.19.1
