Free the actual memory used for the bio using RCU for polled bios.
This will allow to store all the polling information in the bio and
thus simplify passing the cookie and allow for polling stacked devices.
Signed-off-by: Christoph Hellwig <hch@xxxxxx>
---
block/bio.c | 28 +++++++++++++++++++---------
include/linux/blk_types.h | 6 +++++-
2 files changed, 24 insertions(+), 10 deletions(-)
diff --git a/block/bio.c b/block/bio.c
index 303298996afe..7296abe293de 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -215,25 +215,35 @@ void bio_uninit(struct bio *bio)
}
EXPORT_SYMBOL(bio_uninit);
+static inline void bio_mempool_free(struct bio *bio)
+{
+ struct bio_set *bs = bio->bi_pool;
+ void *p = bio;
+
+ p -= bs->front_pad;
+ mempool_free(p, &bs->bio_pool);
+}
+
+static void bio_free_rcu(struct rcu_head *head)
+{
+ bio_mempool_free(container_of(head, struct bio, bi_rcu_free));
+}
+
static void bio_free(struct bio *bio)
{
struct bio_set *bs = bio->bi_pool;
- void *p;
bio_uninit(bio);
if (bs) {
bvec_free(&bs->bvec_pool, bio->bi_io_vec, bio->bi_max_vecs);
-
- /*
- * If we have front padding, adjust the bio pointer before freeing
- */
- p = bio;
- p -= bs->front_pad;
-
- mempool_free(p, &bs->bio_pool);
+ if (bio->bi_opf & REQ_POLLED)
+ call_rcu(&bio->bi_rcu_free, bio_free_rcu);
+ else
+ bio_mempool_free(bio);
} else {
/* Bio was allocated by bio_kmalloc() */
+ WARN_ON_ONCE(bio->bi_opf & REQ_POLLED);
kfree(bio);
}
}
diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h
index ac60432752e3..183a76bf24b7 100644
--- a/include/linux/blk_types.h
+++ b/include/linux/blk_types.h
@@ -10,6 +10,7 @@
#include <linux/bvec.h>
#include <linux/device.h>
#include <linux/ktime.h>
+#include <linux/rcupdate.h>
struct bio_set;
struct bio;
@@ -233,7 +234,10 @@ struct bio {
blk_status_t bi_status;
atomic_t __bi_remaining;
- struct bvec_iter bi_iter;
+ union {
+ struct bvec_iter bi_iter;
+ struct rcu_head bi_rcu_free;
+ };
bio_end_io_t *bi_end_io;
--
2.30.1
