Add support for setting CPU latency limits when a request is dispatched
to driver layer, and removing it once the device is idle. A delayed work
is scheduled from the first block layer activity, and the workqueue ticks
with the configurable timeout period, checking if there has been any
activity. After the initial kick of the workqueue, only the last activity
time is updated with the current jiffies value, minimizing overhead.
The feature is user configurable via sysfs knobs under each individual
block device.
Signed-off-by: Tero Kristo <tero.kristo@xxxxxxxxxxxxxxx>
---
v2:
* moved implementation back to block layer, to the request queue
dispatch section
v3:
* further optimization; fast path now only updates the jiffies value,
and kicks off the workqueue for handling the PM QoS activities if
not already active
* moved the fast path handling under individual request handling, to
avoid iterating the whole request queue
block/blk-mq.c | 51 ++++++++++++++++++++++++++++++++++++++++++
include/linux/blk-mq.h | 12 ++++++++++
2 files changed, 63 insertions(+)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 4b2c8e940f59..e8d82601471d 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -29,6 +29,7 @@
#include <linux/blk-crypto.h>
#include <linux/part_stat.h>
#include <linux/sched/isolation.h>
+#include <linux/pm_qos.h>
#include <trace/events/block.h>
@@ -1303,6 +1304,12 @@ static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
rq->rq_next = NULL;
rq_list_add(&plug->mq_list, rq);
plug->rq_count++;
+
+ if (rq->q->disk->cpu_lat_limit >= 0) {
+ rq->mq_hctx->last_active = jiffies;
+ if (!delayed_work_pending(&rq->mq_hctx->cpu_latency_work))
+ schedule_delayed_work(&rq->mq_hctx->cpu_latency_work, 0);
+ }
}
/**
@@ -2705,6 +2712,45 @@ static void __blk_mq_flush_plug_list(struct request_queue *q,
q->mq_ops->queue_rqs(&plug->mq_list);
}
+static void blk_mq_cpu_latency_work(struct work_struct *work)
+{
+ struct blk_mq_hw_ctx *hctx = container_of(work, struct blk_mq_hw_ctx,
+ cpu_latency_work.work);
+ int cpu;
+ bool add_req = false;
+ bool remove_req = false;
+ unsigned long timeout;
+
+ timeout = msecs_to_jiffies(hctx->queue->disk->cpu_lat_timeout);
+
+ if (time_after(jiffies, hctx->last_active + timeout)) {
+ remove_req = true;
+ hctx->cpu_lat_limit_active = false;
+ } else {
+ if (!hctx->cpu_lat_limit_active) {
+ hctx->cpu_lat_limit_active = true;
+ add_req = true;
+ }
+ schedule_delayed_work(&hctx->cpu_latency_work,
+ hctx->last_active + timeout - jiffies);
+ }
+
+ if (!add_req && !remove_req)
+ return;
+
+ for_each_cpu(cpu, hctx->cpumask) {
+ struct dev_pm_qos_request *qos;
+
+ qos = per_cpu_ptr(hctx->cpu_lat_qos, cpu);
+ if (add_req)
+ dev_pm_qos_add_request(get_cpu_device(cpu), qos,
+ DEV_PM_QOS_RESUME_LATENCY,
+ hctx->queue->disk->cpu_lat_limit);
+ else
+ dev_pm_qos_remove_request(qos);
+ }
+}
+
static void blk_mq_dispatch_plug_list(struct blk_plug *plug, bool from_sched)
{
struct blk_mq_hw_ctx *this_hctx = NULL;
@@ -3729,6 +3775,11 @@ static int blk_mq_init_hctx(struct request_queue *q,
if (xa_insert(&q->hctx_table, hctx_idx, hctx, GFP_KERNEL))
goto exit_flush_rq;
+ hctx->cpu_lat_qos = alloc_percpu(struct dev_pm_qos_request);
+ if (!hctx->cpu_lat_qos)
+ goto exit_flush_rq;
+ INIT_DELAYED_WORK(&hctx->cpu_latency_work, blk_mq_cpu_latency_work);
+
return 0;
exit_flush_rq:
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 4fecf46ef681..4442c18bf3d9 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -435,6 +435,18 @@ struct blk_mq_hw_ctx {
/** @kobj: Kernel object for sysfs. */
struct kobject kobj;
+ /** @cpu_latency_work: Work to handle CPU latency PM limits. */
+ struct delayed_work cpu_latency_work;
+
+ /** @cpu_lat_limit_active: If CPU latency limits are active or not. */
+ bool cpu_lat_limit_active;
+
+ /** @last_active: Jiffies value when the queue was last active. */
+ unsigned long last_active;
+
+ /** @cpu_lat_qos: PM QoS latency limits for individual CPUs. */
+ struct dev_pm_qos_request __percpu *cpu_lat_qos;
+
#ifdef CONFIG_BLK_DEBUG_FS
/**
* @debugfs_dir: debugfs directory for this hardware queue. Named
--
2.43.1
