wbt_init() calls wbt_alloc() which adds allocated wbt instance into
q->rq_qos. This way is very dangerous because q->rq_qos is accessed in
IO fast path.
So far wbt_init() is called in the two code paths:
1) blk_register_queue(), when the block device has been exposed to
usespace, so IO may come when adding wbt into q->rq_qos
2) sysfs attribute store, in which normal IO is definitely allowed
Move wbt allocation into blk_alloc_queue() for avoiding to add wbt
instance dynamically to q->rq_qos. And before calling wbt_init(), the
wbt is disabled, so functionally it works as expected.
Reported-by: Yi Zhang <yi.zhang@xxxxxxxxxx>
Signed-off-by: Ming Lei <ming.lei@xxxxxxxxxx>
---
block/blk-core.c | 6 ++++++
block/blk-mq-debugfs.c | 3 +++
block/blk-sysfs.c | 8 --------
block/blk-wbt.c | 21 ++++++---------------
block/blk-wbt.h | 10 +++++++---
5 files changed, 22 insertions(+), 26 deletions(-)
diff --git a/block/blk-core.c b/block/blk-core.c
index 689aac2625d2..e7d9eac5d4c1 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -51,6 +51,7 @@
#include "blk-mq-sched.h"
#include "blk-pm.h"
#include "blk-rq-qos.h"
+#include "blk-wbt.h"
struct dentry *blk_debugfs_root;
@@ -579,12 +580,17 @@ struct request_queue *blk_alloc_queue(int node_id)
if (blkcg_init_queue(q))
goto fail_ref;
+ if (wbt_alloc(q))
+ goto fail_blkcg;
+
blk_queue_dma_alignment(q, 511);
blk_set_default_limits(&q->limits);
q->nr_requests = BLKDEV_MAX_RQ;
return q;
+fail_blkcg:
+ blkcg_exit_queue(q);
fail_ref:
percpu_ref_exit(&q->q_usage_counter);
fail_bdi:
diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c
index 2a75bc7401df..c1790f313479 100644
--- a/block/blk-mq-debugfs.c
+++ b/block/blk-mq-debugfs.c
@@ -948,6 +948,9 @@ void blk_mq_debugfs_register_rqos(struct rq_qos *rqos)
struct request_queue *q = rqos->q;
const char *dir_name = rq_qos_id_to_name(rqos->id);
+ if (!q->debugfs_dir)
+ return;
+
if (rqos->debugfs_dir || !rqos->ops->debugfs_attrs)
return;
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index f89e2fc3963b..7fd4ffadcdfa 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -481,7 +481,6 @@ static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)
static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
size_t count)
{
- struct rq_qos *rqos;
ssize_t ret;
s64 val;
@@ -491,13 +490,6 @@ static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
if (val < -1)
return -EINVAL;
- rqos = wbt_rq_qos(q);
- if (!rqos) {
- ret = wbt_init(q);
- if (ret)
- return ret;
- }
-
if (val == -1)
val = wbt_default_latency_nsec(q);
else if (val >= 0)
diff --git a/block/blk-wbt.c b/block/blk-wbt.c
index efff1232446f..5da48294e3e9 100644
--- a/block/blk-wbt.c
+++ b/block/blk-wbt.c
@@ -421,16 +421,14 @@ static void wbt_update_limits(struct rq_wb *rwb)
u64 wbt_get_min_lat(struct request_queue *q)
{
struct rq_qos *rqos = wbt_rq_qos(q);
- if (!rqos)
- return 0;
+
return RQWB(rqos)->min_lat_nsec;
}
void wbt_set_min_lat(struct request_queue *q, u64 val)
{
struct rq_qos *rqos = wbt_rq_qos(q);
- if (!rqos)
- return;
+
RQWB(rqos)->min_lat_nsec = val;
RQWB(rqos)->enable_state = WBT_STATE_ON_MANUAL;
wbt_update_limits(RQWB(rqos));
@@ -637,7 +635,7 @@ void wbt_enable_default(struct request_queue *q)
{
struct rq_qos *rqos = wbt_rq_qos(q);
/* Throttling already enabled? */
- if (rqos)
+ if (rwb_enabled(RQWB(rqos)))
return;
/* Queue not registered? Maybe shutting down... */
@@ -809,7 +807,7 @@ static struct rq_qos_ops wbt_rqos_ops = {
#endif
};
-static int wbt_alloc(struct request_queue *q)
+int wbt_alloc(struct request_queue *q)
{
struct rq_wb *rwb;
int i;
@@ -844,20 +842,13 @@ static int wbt_alloc(struct request_queue *q)
return 0;
}
-int wbt_init(struct request_queue *q)
+void wbt_init(struct request_queue *q)
{
- int ret = wbt_alloc(q);
- struct rq_wb *rwb;
-
- if (ret)
- return ret;
+ struct rq_wb *rwb = RQWB(wbt_rq_qos(q));
- rwb = RQWB(wbt_rq_qos(q));
rwb->enable_state = WBT_STATE_ON_DEFAULT;
rwb->min_lat_nsec = wbt_default_latency_nsec(q);
wbt_queue_depth_changed(&rwb->rqos);
wbt_set_write_cache(q, test_bit(QUEUE_FLAG_WC, &q->queue_flags));
-
- return 0;
}
diff --git a/block/blk-wbt.h b/block/blk-wbt.h
index 16bdc85b8df9..de07963f3544 100644
--- a/block/blk-wbt.h
+++ b/block/blk-wbt.h
@@ -87,7 +87,8 @@ static inline unsigned int wbt_inflight(struct rq_wb *rwb)
#ifdef CONFIG_BLK_WBT
-int wbt_init(struct request_queue *);
+int wbt_alloc(struct request_queue *q);
+void wbt_init(struct request_queue *);
void wbt_disable_default(struct request_queue *);
void wbt_enable_default(struct request_queue *);
@@ -103,9 +104,12 @@ u64 wbt_default_latency_nsec(struct request_queue *);
static inline void wbt_track(struct request *rq, enum wbt_flags flags)
{
}
-static inline int wbt_init(struct request_queue *q)
+static inline int wbt_alloc(struct request_queue *q)
+{
+ return 0;
+}
+static inline void wbt_init(struct request_queue *q)
{
- return -EINVAL;
}
static inline void wbt_disable_default(struct request_queue *q)
{
--
2.29.2
