From: Pavel Begunkov <asml.silence@xxxxxxxxx>
blk_rq_stat_sum() expects src argument (struct blk_rq_stat) to have
valid batch field and won't calculate it for dst. Thus, former dst
shouldn't be used as an src arg. iolatency_check_latencies() violates
that, making iolat->cur_stat.rqs.mean always to be 0 for non-ssd
devices.
Use 2 distinct functions instead: one to collect intermediate stats
(i.e. with valid batch), and the second one for merging already
accumulated stats (i.e. with valid mean).
Signed-off-by: Pavel Begunkov <asml.silence@xxxxxxxxx>
---
block/blk-iolatency.c | 21 ++++++++++++++++-----
block/blk-stat.c | 20 ++++++++++++++++++--
block/blk-stat.h | 3 ++-
3 files changed, 36 insertions(+), 8 deletions(-)
diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c
index 507212d75ee2..4010152ebeb2 100644
--- a/block/blk-iolatency.c
+++ b/block/blk-iolatency.c
@@ -198,7 +198,7 @@ static inline void latency_stat_init(struct iolatency_grp *iolat,
blk_rq_stat_init(&stat->rqs);
}
-static inline void latency_stat_sum(struct iolatency_grp *iolat,
+static inline void latency_stat_merge(struct iolatency_grp *iolat,
struct latency_stat *sum,
struct latency_stat *stat)
{
@@ -206,7 +206,18 @@ static inline void latency_stat_sum(struct iolatency_grp *iolat,
sum->ps.total += stat->ps.total;
sum->ps.missed += stat->ps.missed;
} else
- blk_rq_stat_sum(&sum->rqs, &stat->rqs);
+ blk_rq_stat_merge(&sum->rqs, &stat->rqs);
+}
+
+static inline void latency_stat_collect(struct iolatency_grp *iolat,
+ struct latency_stat *sum,
+ struct latency_stat *stat)
+{
+ if (iolat->ssd) {
+ sum->ps.total += stat->ps.total;
+ sum->ps.missed += stat->ps.missed;
+ } else
+ blk_rq_stat_collect(&sum->rqs, &stat->rqs);
}
static inline void latency_stat_record_time(struct iolatency_grp *iolat,
@@ -530,7 +541,7 @@ static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now)
for_each_online_cpu(cpu) {
struct latency_stat *s;
s = per_cpu_ptr(iolat->stats, cpu);
- latency_stat_sum(iolat, &stat, s);
+ latency_stat_collect(iolat, &stat, s);
latency_stat_init(iolat, s);
}
preempt_enable();
@@ -551,7 +562,7 @@ static void iolatency_check_latencies(struct iolatency_grp *iolat, u64 now)
/* Somebody beat us to the punch, just bail. */
spin_lock_irqsave(&lat_info->lock, flags);
- latency_stat_sum(iolat, &iolat->cur_stat, &stat);
+ latency_stat_merge(iolat, &iolat->cur_stat, &stat);
lat_info->nr_samples -= iolat->nr_samples;
lat_info->nr_samples += latency_stat_samples(iolat, &iolat->cur_stat);
iolat->nr_samples = latency_stat_samples(iolat, &iolat->cur_stat);
@@ -912,7 +923,7 @@ static size_t iolatency_ssd_stat(struct iolatency_grp *iolat, char *buf,
for_each_online_cpu(cpu) {
struct latency_stat *s;
s = per_cpu_ptr(iolat->stats, cpu);
- latency_stat_sum(iolat, &stat, s);
+ latency_stat_collect(iolat, &stat, s);
}
preempt_enable();
diff --git a/block/blk-stat.c b/block/blk-stat.c
index 696a04176e4d..a6da68af45db 100644
--- a/block/blk-stat.c
+++ b/block/blk-stat.c
@@ -25,7 +25,7 @@ void blk_rq_stat_init(struct blk_rq_stat *stat)
}
/* src is a per-cpu stat, mean isn't initialized */
-void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
+void blk_rq_stat_collect(struct blk_rq_stat *dst, struct blk_rq_stat *src)
{
if (!src->nr_samples)
return;
@@ -39,6 +39,21 @@ void blk_rq_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
dst->nr_samples += src->nr_samples;
}
+void blk_rq_stat_merge(struct blk_rq_stat *dst, struct blk_rq_stat *src)
+{
+ if (!src->nr_samples)
+ return;
+
+ dst->min = min(dst->min, src->min);
+ dst->max = max(dst->max, src->max);
+
+ dst->mean = div_u64(src->mean * src->nr_samples +
+ dst->mean * dst->nr_samples,
+ dst->nr_samples + src->nr_samples);
+
+ dst->nr_samples += src->nr_samples;
+}
+
void blk_rq_stat_add(struct blk_rq_stat *stat, u64 value)
{
stat->min = min(stat->min, value);
@@ -89,7 +104,8 @@ static void blk_stat_timer_fn(struct timer_list *t)
cpu_stat = per_cpu_ptr(cb->cpu_stat, cpu);
for (bucket = 0; bucket < cb->buckets; bucket++) {
- blk_rq_stat_sum(&cb->stat[bucket], &cpu_stat[bucket]);
+ blk_rq_stat_collect(&cb->stat[bucket],
+ &cpu_stat[bucket]);
blk_rq_stat_init(&cpu_stat[bucket]);
}
}
diff --git a/block/blk-stat.h b/block/blk-stat.h
index 17b47a86eefb..5597ecc34ef5 100644
--- a/block/blk-stat.h
+++ b/block/blk-stat.h
@@ -165,7 +165,8 @@ static inline void blk_stat_activate_msecs(struct blk_stat_callback *cb,
}
void blk_rq_stat_add(struct blk_rq_stat *, u64);
-void blk_rq_stat_sum(struct blk_rq_stat *, struct blk_rq_stat *);
+void blk_rq_stat_collect(struct blk_rq_stat *dst, struct blk_rq_stat *src);
+void blk_rq_stat_merge(struct blk_rq_stat *dst, struct blk_rq_stat *src);
void blk_rq_stat_init(struct blk_rq_stat *);
#endif
--
2.21.0
