If we're polling for IO on a device that doesn't use interrupts, then
IO completion loop (and wake of task) is done by submitting task itself.
If that is the case, then we don't need to enter the wake_up_process()
function, we can simply mark ourselves as TASK_RUNNING.
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
---
fs/block_dev.c | 6 ++----
fs/iomap.c | 3 +--
include/linux/blkdev.h | 19 +++++++++++++++++++
3 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/fs/block_dev.c b/fs/block_dev.c
index 2f920c03996e..0ed9be8906a8 100644
--- a/fs/block_dev.c
+++ b/fs/block_dev.c
@@ -181,8 +181,7 @@ static void blkdev_bio_end_io_simple(struct bio *bio)
struct task_struct *waiter = bio->bi_private;
WRITE_ONCE(bio->bi_private, NULL);
- smp_wmb();
- wake_up_process(waiter);
+ blk_wake_io_task(waiter);
}
static ssize_t
@@ -309,8 +308,7 @@ static void blkdev_bio_end_io(struct bio *bio)
struct task_struct *waiter = dio->waiter;
WRITE_ONCE(dio->waiter, NULL);
- smp_wmb();
- wake_up_process(waiter);
+ blk_wake_io_task(waiter);
}
}
diff --git a/fs/iomap.c b/fs/iomap.c
index 7898927e758e..a182699e28db 100644
--- a/fs/iomap.c
+++ b/fs/iomap.c
@@ -1526,8 +1526,7 @@ static void iomap_dio_bio_end_io(struct bio *bio)
struct task_struct *waiter = dio->submit.waiter;
WRITE_ONCE(dio->submit.waiter, NULL);
- smp_wmb();
- wake_up_process(waiter);
+ blk_wake_io_task(waiter);
} else if (dio->flags & IOMAP_DIO_WRITE) {
struct inode *inode = file_inode(dio->iocb->ki_filp);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index ad8474ec8c58..d1ef8cbbea04 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1798,4 +1798,23 @@ static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
#endif /* CONFIG_BLOCK */
+static inline void blk_wake_io_task(struct task_struct *waiter)
+{
+ /*
+ * If we're polling, the task itself is doing the completions. For
+ * that case, we don't need to signal a wakeup, it's enough to just
+ * mark us as RUNNING.
+ */
+ if (waiter == current)
+ __set_current_state(TASK_RUNNING);
+ else {
+ /*
+ * Ensure the callers waiter store is ordered and seen
+ * by the ->bi_end_io() function.
+ */
+ smp_wmb();
+ wake_up_process(waiter);
+ }
+}
+
#endif
--
2.17.1
