From: Bartosz Golaszewski <bartosz.golaszewski@xxxxxxxxxx>
Commit 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU")
caused a massive drop in performance of requesting GPIO lines due to the
call to synchronize_srcu() on each label change. Rework the code to not
wait until all read-only users are done with reading the label but
instead atomically replace the label pointer and schedule its release
after all read-only critical sections are done.
To that end wrap the descriptor label in a struct that also contains the
rcu_head struct required for deferring tasks using call_srcu() and stop
using kstrdup_const() as we're required to allocate memory anyway. Just
allocate enough for the label string and rcu_head in one go.
Reported-by: Neil Armstrong <neil.armstrong@xxxxxxxxxx>
Closes: https://lore.kernel.org/linux-gpio/CAMRc=Mfig2oooDQYTqo23W3PXSdzhVO4p=G4+P8y1ppBOrkrJQ@xxxxxxxxxxxxxx/
Fixes: 1f2bcb8c8ccd ("gpio: protect the descriptor label with SRCU")
Suggested-by: Paul E. McKenney <paulmck@xxxxxxxxxx>
Signed-off-by: Bartosz Golaszewski <bartosz.golaszewski@xxxxxxxxxx>
---
drivers/gpio/gpiolib.c | 31 ++++++++++++++++++++++++-------
drivers/gpio/gpiolib.h | 7 ++++++-
2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/drivers/gpio/gpiolib.c b/drivers/gpio/gpiolib.c
index 94903fc1c145..2fa3756c9073 100644
--- a/drivers/gpio/gpiolib.c
+++ b/drivers/gpio/gpiolib.c
@@ -101,6 +101,7 @@ static bool gpiolib_initialized;
const char *gpiod_get_label(struct gpio_desc *desc)
{
+ struct gpio_desc_label *label;
unsigned long flags;
flags = READ_ONCE(desc->flags);
@@ -108,23 +109,36 @@ const char *gpiod_get_label(struct gpio_desc *desc)
!test_bit(FLAG_REQUESTED, &flags))
return "interrupt";
- return test_bit(FLAG_REQUESTED, &flags) ?
- srcu_dereference(desc->label, &desc->srcu) : NULL;
+ if (!test_bit(FLAG_REQUESTED, &flags))
+ return NULL;
+
+ label = srcu_dereference_check(desc->label, &desc->srcu,
+ srcu_read_lock_held(&desc->srcu));
+
+ return label->str;
+}
+
+static void desc_free_label(struct rcu_head *rh)
+{
+ kfree(container_of(rh, struct gpio_desc_label, rh));
}
static int desc_set_label(struct gpio_desc *desc, const char *label)
{
- const char *new = NULL, *old;
+ struct gpio_desc_label *new = NULL, *old;
if (label) {
- new = kstrdup_const(label, GFP_KERNEL);
+ new = kzalloc(struct_size(new, str, strlen(label) + 1),
+ GFP_KERNEL);
if (!new)
return -ENOMEM;
+
+ strcpy(new->str, label);
}
old = rcu_replace_pointer(desc->label, new, 1);
- synchronize_srcu(&desc->srcu);
- kfree_const(old);
+ if (old)
+ call_srcu(&desc->srcu, &old->rh, desc_free_label);
return 0;
}
@@ -697,8 +711,11 @@ static void gpiodev_release(struct device *dev)
struct gpio_device *gdev = to_gpio_device(dev);
unsigned int i;
- for (i = 0; i < gdev->ngpio; i++)
+ for (i = 0; i < gdev->ngpio; i++) {
+ /* Free pending label. */
+ synchronize_srcu(&gdev->descs[i].srcu);
cleanup_srcu_struct(&gdev->descs[i].srcu);
+ }
ida_free(&gpio_ida, gdev->id);
kfree_const(gdev->label);
diff --git a/drivers/gpio/gpiolib.h b/drivers/gpio/gpiolib.h
index f67d5991ab1c..69a353c789f0 100644
--- a/drivers/gpio/gpiolib.h
+++ b/drivers/gpio/gpiolib.h
@@ -137,6 +137,11 @@ int gpiod_set_transitory(struct gpio_desc *desc, bool transitory);
void gpiod_line_state_notify(struct gpio_desc *desc, unsigned long action);
+struct gpio_desc_label {
+ struct rcu_head rh;
+ char str[];
+};
+
/**
* struct gpio_desc - Opaque descriptor for a GPIO
*
@@ -177,7 +182,7 @@ struct gpio_desc {
#define FLAG_EVENT_CLOCK_HTE 19 /* GPIO CDEV reports hardware timestamps in events */
/* Connection label */
- const char __rcu *label;
+ struct gpio_desc_label __rcu *label;
/* Name of the GPIO */
const char *name;
#ifdef CONFIG_OF_DYNAMIC
