Allow block drivers to configure the following:
* Maximum number of hardware sectors values smaller than
PAGE_SIZE >> SECTOR_SHIFT. For PAGE_SIZE = 4096 this means that values
below 8 become supported.
* A maximum segment size below the page size. This is most useful
for page sizes above 4096 bytes.
The blk_sub_page_segments static branch will be used in later patches to
prevent that performance of block drivers that support segments >=
PAGE_SIZE and max_hw_sectors >= PAGE_SIZE >> SECTOR_SHIFT would be affected.
This patch may change the behavior of existing block drivers from not
working into working. If a block driver calls
blk_queue_max_hw_sectors() or blk_queue_max_segment_size(), this is
usually done to configure the maximum supported limits. An attempt to
configure a limit below what is supported by the block layer causes the
block layer to select a larger value. If that value is not supported by
the block driver, this may cause other data to be transferred than
requested, a kernel crash or other undesirable behavior.
Cc: Christoph Hellwig <hch@xxxxxx>
Cc: Ming Lei <ming.lei@xxxxxxxxxx>
Cc: Keith Busch <kbusch@xxxxxxxxxx>
Cc: Luis Chamberlain <mcgrof@xxxxxxxxxx>
Signed-off-by: Bart Van Assche <bvanassche@xxxxxxx>
---
block/blk-core.c | 2 ++
block/blk-settings.c | 57 ++++++++++++++++++++++++++++++++++++++++++
block/blk.h | 9 +++++++
include/linux/blkdev.h | 2 ++
4 files changed, 70 insertions(+)
diff --git a/block/blk-core.c b/block/blk-core.c
index 00c74330fa92..814bfb9c9489 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -264,6 +264,8 @@ static void blk_free_queue_rcu(struct rcu_head *rcu_head)
static void blk_free_queue(struct request_queue *q)
{
blk_free_queue_stats(q->stats);
+ blk_disable_sub_page_limits(&q->limits);
+
if (queue_is_mq(q))
blk_mq_release(q);
diff --git a/block/blk-settings.c b/block/blk-settings.c
index 95d6e836c4a7..a4ef1dfeef76 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -19,6 +19,11 @@
#include "blk-rq-qos.h"
#include "blk-wbt.h"
+/* Protects blk_nr_sub_page_limit_queues and blk_sub_page_limits changes. */
+static DEFINE_MUTEX(blk_sub_page_limit_lock);
+static uint32_t blk_nr_sub_page_limit_queues;
+DEFINE_STATIC_KEY_FALSE(blk_sub_page_limits);
+
void blk_queue_rq_timeout(struct request_queue *q, unsigned int timeout)
{
q->rq_timeout = timeout;
@@ -59,6 +64,7 @@ void blk_set_default_limits(struct queue_limits *lim)
lim->zoned = BLK_ZONED_NONE;
lim->zone_write_granularity = 0;
lim->dma_alignment = 511;
+ lim->sub_page_limits = false;
}
/**
@@ -101,6 +107,47 @@ void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce bounce)
}
EXPORT_SYMBOL(blk_queue_bounce_limit);
+/**
+ * blk_enable_sub_page_limits - enable support for max_segment_size values smaller than PAGE_SIZE and for max_hw_sectors values below PAGE_SIZE >> SECTOR_SHIFT
+ * @lim: request queue limits for which to enable support of these features.
+ *
+ * Support for these features is not enabled all the time because of the
+ * runtime overhead of these features.
+ */
+static void blk_enable_sub_page_limits(struct queue_limits *lim)
+{
+ if (lim->sub_page_limits)
+ return;
+
+ lim->sub_page_limits = true;
+
+ mutex_lock(&blk_sub_page_limit_lock);
+ if (++blk_nr_sub_page_limit_queues == 1)
+ static_branch_enable(&blk_sub_page_limits);
+ mutex_unlock(&blk_sub_page_limit_lock);
+}
+
+/**
+ * blk_disable_sub_page_limits - disable support for max_segment_size values smaller than PAGE_SIZE and for max_hw_sectors values below PAGE_SIZE >> SECTOR_SHIFT
+ * @lim: request queue limits for which to enable support of these features.
+ *
+ * Support for these features is not enabled all the time because of the
+ * runtime overhead of these features.
+ */
+void blk_disable_sub_page_limits(struct queue_limits *lim)
+{
+ if (!lim->sub_page_limits)
+ return;
+
+ lim->sub_page_limits = false;
+
+ mutex_lock(&blk_sub_page_limit_lock);
+ WARN_ON_ONCE(blk_nr_sub_page_limit_queues <= 0);
+ if (--blk_nr_sub_page_limit_queues == 0)
+ static_branch_disable(&blk_sub_page_limits);
+ mutex_unlock(&blk_sub_page_limit_lock);
+}
+
/**
* blk_queue_max_hw_sectors - set max sectors for a request for this queue
* @q: the request queue for the device
@@ -126,6 +173,11 @@ void blk_queue_max_hw_sectors(struct request_queue *q, unsigned int max_hw_secto
unsigned int min_max_hw_sectors = PAGE_SIZE >> SECTOR_SHIFT;
unsigned int max_sectors;
+ if (max_hw_sectors < min_max_hw_sectors) {
+ blk_enable_sub_page_limits(limits);
+ min_max_hw_sectors = 1;
+ }
+
if (max_hw_sectors < min_max_hw_sectors) {
max_hw_sectors = min_max_hw_sectors;
pr_info("%s: set to minimum %u\n", __func__, max_hw_sectors);
@@ -284,6 +336,11 @@ void blk_queue_max_segment_size(struct request_queue *q, unsigned int max_size)
{
unsigned int min_max_segment_size = PAGE_SIZE;
+ if (max_size < min_max_segment_size) {
+ blk_enable_sub_page_limits(&q->limits);
+ min_max_segment_size = SECTOR_SIZE;
+ }
+
if (max_size < min_max_segment_size) {
max_size = min_max_segment_size;
pr_info("%s: set to minimum %u\n", __func__, max_size);
diff --git a/block/blk.h b/block/blk.h
index 9f171b8f1e34..49526127ea08 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -13,6 +13,7 @@ struct elevator_type;
#define BLK_MAX_TIMEOUT (5 * HZ)
extern struct dentry *blk_debugfs_root;
+DECLARE_STATIC_KEY_FALSE(blk_sub_page_limits);
struct blk_flush_queue {
unsigned int flush_pending_idx:1;
@@ -32,6 +33,14 @@ struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
gfp_t flags);
void blk_free_flush_queue(struct blk_flush_queue *q);
+static inline bool blk_queue_sub_page_limits(const struct queue_limits *lim)
+{
+ return static_branch_unlikely(&blk_sub_page_limits) &&
+ lim->sub_page_limits;
+}
+
+void blk_disable_sub_page_limits(struct queue_limits *q);
+
void blk_freeze_queue(struct request_queue *q);
void __blk_mq_unfreeze_queue(struct request_queue *q, bool force_atomic);
void blk_queue_start_drain(struct request_queue *q);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index fe99948688df..e54fbb124efb 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -310,6 +310,8 @@ struct queue_limits {
* due to possible offsets.
*/
unsigned int dma_alignment;
+
+ bool sub_page_limits;
};
typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
