Defer calling put_prev_task() on a CFS task_struct when there is a
pending RT task to run. Instead wait until the next
pick_next_task_fair() and do the work there.
The put_prev_task() call for a SCHED_OTHER task is currently a source
of non determinism in the latency of scheduling a SCHED_FIFO task.
This results in a priority inversion as the CFS scheduler is updating
load average and balancing the rq rbtree while the SCHED_FIFO task is
waiting to run.
Instrumented results on a quad core ARM A57. This is measured just
across the put_prev_task() in pick_next_task_rt().
before patch
cpu: 0
max_instr: 1114
min_instr: 9
avg_instr: 11
min_time: 32ns
max_time: 2528ns
avg_time: 64ns
cpu: 1
max_instr: 1122
min_instr: 9
avg_instr: 11
min_time: 32ns
max_time: 3040ns
avg_time: 64ns
cpu: 2
max_instr: 1118
min_instr: 9
avg_instr: 11
min_time: 32ns
max_time: 3584ns
avg_time: 64ns
cpu: 3
max_instr: 1122
min_instr: 9
avg_instr: 11
min_time: 32ns
max_time: 3456ns
avg_time: 64ns
after patch
cpu: 0
max_instr: 23
min_instr: 9
avg_instr: 11
min_time: 64ns
max_time: 480ns
avg_time: 64ns
cpu: 1
max_instr: 23
min_instr: 9
avg_instr: 11
min_time: 64ns
max_time: 224ns
avg_time: 64ns
cpu: 2
max_instr: 12
min_instr: 9
avg_instr: 10
min_time: 64ns
max_time: 224ns
avg_time: 64ns
cpu: 3
max_instr: 44
min_instr: 9
avg_instr: 11
min_time: 64ns
max_time: 160ns
avg_time: 64ns
Signed-off-by: Allen Martin <amartin@xxxxxxxxxx>
---
include/linux/sched.h | 4 ++++
kernel/sched/core.c | 4 ++++
kernel/sched/fair.c | 41 +++++++++++++++++++++++++++++++++++++++++
kernel/sched/rt.c | 6 ++++++
kernel/sched/sched.h | 3 +++
5 files changed, 58 insertions(+)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e7ae9273a809..fdaf4ae2383e 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1126,6 +1126,10 @@ struct task_struct {
void *security;
#endif
+#ifdef CONFIG_PREEMPT_RT_FULL
+ int rt_preempt;
+#endif
+
/*
* New fields for task_struct should be added above here, so that
* they are included in the randomized portion of task_struct.
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3a9899fc26f7..5cd3e1d25238 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6016,6 +6016,10 @@ void __init sched_init(void)
INIT_LIST_HEAD(&rq->cfs_tasks);
+#ifdef CONFIG_PREEMPT_RT_FULL
+ rq->cfs_deferred_task = NULL;
+#endif
+
rq_attach_root(rq, &def_root_domain);
#ifdef CONFIG_NO_HZ_COMMON
rq->last_load_update_tick = jiffies;
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index a1a7ea8662c4..79cc22cb9698 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4858,6 +4858,13 @@ static inline void hrtick_update(struct rq *rq)
}
#endif
+#ifdef CONFIG_PREEMPT_RT_FULL
+static int is_task_deferred(struct rq *rq, struct task_struct *prev)
+{
+ return rq->cfs_deferred_task == prev;
+}
+#endif
+
/*
* The enqueue_task method is called before nr_running is
* increased. Here we update the fair scheduling stats and
@@ -4926,6 +4933,15 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags)
struct sched_entity *se = &p->se;
int task_sleep = flags & DEQUEUE_SLEEP;
+#ifdef CONFIG_PREEMPT_RT_FULL
+ /* see if this was the deferred task */
+ if (is_task_deferred(rq, p)) {
+ /* reinsert it and then go through normal dequeue path */
+ rq->cfs_deferred_task = NULL;
+ put_prev_task(rq, p);
+ }
+#endif
+
for_each_sched_entity(se) {
cfs_rq = cfs_rq_of(se);
dequeue_entity(cfs_rq, se, flags);
@@ -6186,6 +6202,14 @@ pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf
struct task_struct *p;
int new_tasks;
+#ifdef CONFIG_PREEMPT_RT_FULL
+ if (rq->cfs_deferred_task) {
+ p = rq->cfs_deferred_task;
+ rq->cfs_deferred_task = NULL;
+ put_prev_task(rq, p);
+ }
+#endif
+
again:
#ifdef CONFIG_FAIR_GROUP_SCHED
if (!cfs_rq->nr_running)
@@ -6310,6 +6334,13 @@ static void put_prev_task_fair(struct rq *rq, struct task_struct *prev)
struct sched_entity *se = &prev->se;
struct cfs_rq *cfs_rq;
+#ifdef CONFIG_PREEMPT_RT_FULL
+ if (unlikely(prev->rt_preempt && prev->on_rq)) {
+ rq->cfs_deferred_task = prev;
+ return;
+ }
+#endif
+
for_each_sched_entity(se) {
cfs_rq = cfs_rq_of(se);
put_prev_entity(cfs_rq, se);
@@ -9105,6 +9136,16 @@ static void detach_task_cfs_rq(struct task_struct *p)
{
struct sched_entity *se = &p->se;
struct cfs_rq *cfs_rq = cfs_rq_of(se);
+#ifdef CONFIG_PREEMPT_RT_FULL
+ struct rq *rq = rq_of(cfs_rq);
+
+ /* see if this was in the deferred list */
+ if (is_task_deferred(rq, p)) {
+ /* reinsert it and then go through normal detach path */
+ rq->cfs_deferred_task = NULL;
+ put_prev_task(rq, p);
+ }
+#endif
if (!vruntime_normalized(p)) {
/*
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 3249423cdabd..45c7ff1a30ac 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -1571,7 +1571,13 @@ pick_next_task_rt(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
if (!rt_rq->rt_queued)
return NULL;
+#ifdef CONFIG_PREEMPT_RT_FULL
+ prev->rt_preempt = true;
+#endif
put_prev_task(rq, prev);
+#ifdef CONFIG_PREEMPT_RT_FULL
+ prev->rt_preempt = false;
+#endif
p = _pick_next_task_rt(rq);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 72983b9066c5..0da3dd449676 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -742,6 +742,9 @@ struct rq {
int online;
struct list_head cfs_tasks;
+#ifdef CONFIG_PREEMPT_RT_FULL
+ struct task_struct *cfs_deferred_task;
+#endif
u64 rt_avg;
u64 age_stamp;
--
2.7.4
--
To unsubscribe from this list: send the line "unsubscribe linux-rt-users" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
