From: chen xiaoguang <xiaoggchen@xxxxxxxxxxx>
Signed-off-by: Newton Gao <newtongao@xxxxxxxxxxx>
Signed-off-by: Shook Liu <shookliu@xxxxxxxxxxx>
Signed-off-by: Zhiguang Peng <zgpeng@xxxxxxxxxxx>
Signed-off-by: Xiaoguang Chen <xiaoggchen@xxxxxxxxxxx>
---
fs/proc/base.c | 3 ++-
kernel/delayacct.c | 2 +-
kernel/exit.c | 3 ++-
kernel/sched/core.c | 30 +++++++++++++++++++++++-------
kernel/sched/cputime.c | 7 ++++---
kernel/sched/fair.c | 27 +++++++++++++++++----------
kernel/sched/loadavg.c | 5 +++--
kernel/time/posix-cpu-timers.c | 2 +-
8 files changed, 53 insertions(+), 26 deletions(-)
diff --git a/fs/proc/base.c b/fs/proc/base.c
index 9c8ca6c..356f2e8 100644
--- a/fs/proc/base.c
+++ b/fs/proc/base.c
@@ -459,7 +459,8 @@ static int proc_pid_schedstat(struct seq_file *m, struct pid_namespace *ns,
seq_puts(m, "0 0 0\n");
else
seq_printf(m, "%llu %llu %lu\n",
- (unsigned long long)task->se.sum_exec_runtime,
+ (unsigned long long)task->se.sum_exec_runtime +
+ (unsigned long long)task->bt.sum_exec_runtime,
(unsigned long long)task->sched_info.run_delay,
task->sched_info.pcount);
diff --git a/kernel/delayacct.c b/kernel/delayacct.c
index 27725754..c8bb8f6 100644
--- a/kernel/delayacct.c
+++ b/kernel/delayacct.c
@@ -106,7 +106,7 @@ int __delayacct_add_tsk(struct taskstats *d, struct task_struct *tsk)
*/
t1 = tsk->sched_info.pcount;
t2 = tsk->sched_info.run_delay;
- t3 = tsk->se.sum_exec_runtime;
+ t3 = tsk->se.sum_exec_runtime + tsk->bt.sum_exec_runtime;
d->cpu_count += t1;
diff --git a/kernel/exit.c b/kernel/exit.c
index 1803efb..ee5ef46 100644
--- a/kernel/exit.c
+++ b/kernel/exit.c
@@ -152,7 +152,8 @@ static void __exit_signal(struct task_struct *tsk)
sig->inblock += task_io_get_inblock(tsk);
sig->oublock += task_io_get_oublock(tsk);
task_io_accounting_add(&sig->ioac, &tsk->ioac);
- sig->sum_sched_runtime += tsk->se.sum_exec_runtime;
+ sig->sum_sched_runtime += (tsk->se.sum_exec_runtime +
+ tsk->bt.sum_exec_runtime);
sig->nr_threads--;
__unhash_process(tsk, group_dead);
write_sequnlock(&sig->stats_lock);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index b542c17..e8fbd9e 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -789,9 +789,13 @@ static inline void dequeue_task(struct rq *rq, struct task_struct *p, int flags)
void activate_task(struct rq *rq, struct task_struct *p, int flags)
{
- if (task_contributes_to_load(p))
+ if (task_contributes_to_load(p)) {
rq->nr_uninterruptible--;
+ if (unlikely(p->sched_class == &bt_sched_class))
+ rq->bt.nr_uninterruptible--;
+ }
+
enqueue_task(rq, p, flags);
p->on_rq = TASK_ON_RQ_QUEUED;
@@ -801,9 +805,13 @@ void deactivate_task(struct rq *rq, struct task_struct *p, int flags)
{
p->on_rq = (flags & DEQUEUE_SLEEP) ? 0 : TASK_ON_RQ_MIGRATING;
- if (task_contributes_to_load(p))
+ if (task_contributes_to_load(p)) {
rq->nr_uninterruptible++;
+ if (unlikely(p->sched_class == &bt_sched_class))
+ rq->bt.nr_uninterruptible++;
+ }
+
dequeue_task(rq, p, flags);
}
@@ -1727,9 +1735,13 @@ static void ttwu_do_wakeup(struct rq *rq, struct task_struct *p, int wake_flags,
lockdep_assert_held(&rq->lock);
#ifdef CONFIG_SMP
- if (p->sched_contributes_to_load)
+ if (p->sched_contributes_to_load) {
rq->nr_uninterruptible--;
+ if (unlikely(p->sched_class == &bt_sched_class))
+ rq->bt.nr_uninterruptible--;
+ }
+
if (wake_flags & WF_MIGRATED)
en_flags |= ENQUEUE_MIGRATED;
#endif
@@ -3007,7 +3019,7 @@ unsigned long long task_sched_runtime(struct task_struct *p)
* been accounted, so we're correct here as well.
*/
if (!p->on_cpu || !task_on_rq_queued(p))
- return p->se.sum_exec_runtime;
+ return p->se.sum_exec_runtime + p->bt.sum_exec_runtime;
#endif
rq = task_rq_lock(p, &rf);
@@ -3021,7 +3033,7 @@ unsigned long long task_sched_runtime(struct task_struct *p)
update_rq_clock(rq);
p->sched_class->update_curr(rq);
}
- ns = p->se.sum_exec_runtime;
+ ns = p->se.sum_exec_runtime + p->bt.sum_exec_runtime;
task_rq_unlock(rq, p, &rf);
return ns;
@@ -4003,7 +4015,7 @@ int idle_cpu(int cpu)
if (rq->curr != rq->idle && !bt_prio(rq->curr->prio))
return 0;
- if (rq->nr_running)
+ if (rq->nr_running - rq->bt.bt_nr_running)
return 0;
#ifdef CONFIG_SMP
@@ -5383,9 +5395,13 @@ void init_idle(struct task_struct *idle, int cpu)
__sched_fork(0, idle);
idle->state = TASK_RUNNING;
- idle->se.exec_start = sched_clock();
+ idle->se.exec_start = idle->bt.exec_start = sched_clock();
idle->flags |= PF_IDLE;
+#if defined(CONFIG_SCHEDSTATS) || defined(CONFIG_LATENCYTOP)
+ idle->bt.statistics = &idle->se.statistics;
+#endif
+
kasan_unpoison_task_stack(idle);
#ifdef CONFIG_SMP
diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c
index ca26a04..4cba3f2 100644
--- a/kernel/sched/cputime.c
+++ b/kernel/sched/cputime.c
@@ -283,7 +283,7 @@ static inline u64 account_other_time(u64 max)
#ifdef CONFIG_64BIT
static inline u64 read_sum_exec_runtime(struct task_struct *t)
{
- return t->se.sum_exec_runtime;
+ return t->se.sum_exec_runtime + t->bt.sum_exec_runtime;
}
#else
static u64 read_sum_exec_runtime(struct task_struct *t)
@@ -293,7 +293,7 @@ static u64 read_sum_exec_runtime(struct task_struct *t)
struct rq *rq;
rq = task_rq_lock(t, &rf);
- ns = t->se.sum_exec_runtime;
+ ns = t->se.sum_exec_runtime + t->bt.sum_exec_runtime;
task_rq_unlock(rq, t, &rf);
return ns;
@@ -677,7 +677,8 @@ void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
{
struct task_cputime cputime = {
- .sum_exec_runtime = p->se.sum_exec_runtime,
+ .sum_exec_runtime = p->se.sum_exec_runtime +
+ p->bt.sum_exec_runtime,
};
task_cputime(p, &cputime.utime, &cputime.stime);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 5299ce8..ca08107 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -7137,7 +7137,7 @@ static void yield_task_fair(struct rq *rq)
/*
* Are we the only task in the tree?
*/
- if (unlikely(rq->nr_running == 1))
+ if (unlikely((rq->nr_running - rq->bt.bt_nr_running) == 1))
return;
clear_buddies(cfs_rq, se);
@@ -7357,7 +7357,9 @@ static int task_hot(struct task_struct *p, struct lb_env *env)
/*
* Buddy candidates are cache hot:
*/
- if (sched_feat(CACHE_HOT_BUDDY) && env->dst_rq->nr_running &&
+ if (sched_feat(CACHE_HOT_BUDDY) &&
+ (env->dst_rq->nr_running -
+ env->dst_rq->bt.bt_nr_running) &&
(&p->se == cfs_rq_of(&p->se)->next ||
&p->se == cfs_rq_of(&p->se)->last))
return 1;
@@ -7398,7 +7400,8 @@ static int migrate_degrades_locality(struct task_struct *p, struct lb_env *env)
/* Migrating away from the preferred node is always bad. */
if (src_nid == p->numa_preferred_nid) {
- if (env->src_rq->nr_running > env->src_rq->nr_preferred_running)
+ if ((env->src_rq->nr_running - env->src_rq->bt.bt_nr_running) >
+ env->src_rq->nr_preferred_running)
return 1;
else
return -1;
@@ -7580,7 +7583,9 @@ static int detach_tasks(struct lb_env *env)
* We don't want to steal all, otherwise we may be treated likewise,
* which could at worst lead to a livelock crash.
*/
- if (env->idle != CPU_NOT_IDLE && env->src_rq->nr_running <= 1)
+ if (env->idle != CPU_NOT_IDLE &&
+ (env->src_rq->nr_running -
+ env->src_rq->bt.bt_nr_running) <= 1)
break;
p = list_last_entry(tasks, struct task_struct, se.group_node);
@@ -8272,7 +8277,7 @@ static inline void update_sg_lb_stats(struct lb_env *env,
sgs->group_util += cpu_util(i);
sgs->sum_nr_running += rq->cfs.h_nr_running;
- nr_running = rq->nr_running;
+ nr_running = rq->nr_running - rq->bt.bt_nr_running;
if (nr_running > 1)
*sg_status |= SG_OVERLOAD;
@@ -9080,7 +9085,7 @@ static int load_balance(int this_cpu, struct rq *this_rq,
env.src_rq = busiest;
ld_moved = 0;
- if (busiest->nr_running > 1) {
+ if (busiest->nr_running - busiest->bt.bt_nr_running > 1) {
/*
* Attempt to move tasks. If find_busiest_group has found
* an imbalance but busiest->nr_running <= 1, the group is
@@ -9088,7 +9093,8 @@ static int load_balance(int this_cpu, struct rq *this_rq,
* correctly treated as an imbalance.
*/
env.flags |= LBF_ALL_PINNED;
- env.loop_max = min(sysctl_sched_nr_migrate, busiest->nr_running);
+ env.loop_max = min(sysctl_sched_nr_migrate,
+ busiest->nr_running - busiest->bt.bt_nr_running);
more_balance:
rq_lock_irqsave(busiest, &rf);
@@ -9359,7 +9365,7 @@ static int active_load_balance_cpu_stop(void *data)
goto out_unlock;
/* Is there any task to move? */
- if (busiest_rq->nr_running <= 1)
+ if (busiest_rq->nr_running - busiest_rq->bt.bt_nr_running <= 1)
goto out_unlock;
/*
@@ -9628,7 +9634,7 @@ static void nohz_balancer_kick(struct rq *rq)
if (time_before(now, nohz.next_balance))
goto out;
- if (rq->nr_running >= 2) {
+ if (rq->nr_running - rq->bt.bt_nr_running >= 2) {
flags = NOHZ_KICK_MASK;
goto out;
}
@@ -10075,7 +10081,8 @@ static int idle_balance(struct rq *this_rq, struct rq_flags *rf)
* Stop searching for tasks to pull if there are
* now runnable tasks on this rq.
*/
- if (pulled_task || this_rq->nr_running > 0)
+ if (pulled_task || this_rq->nr_running -
+ this_rq->bt.bt_nr_running > 0)
break;
}
rcu_read_unlock();
diff --git a/kernel/sched/loadavg.c b/kernel/sched/loadavg.c
index 28a5165..6810e59 100644
--- a/kernel/sched/loadavg.c
+++ b/kernel/sched/loadavg.c
@@ -80,8 +80,9 @@ long calc_load_fold_active(struct rq *this_rq, long adjust)
{
long nr_active, delta = 0;
- nr_active = this_rq->nr_running - adjust;
- nr_active += (long)this_rq->nr_uninterruptible;
+ nr_active = this_rq->nr_running - this_rq->bt.bt_nr_running - adjust;
+ nr_active += (long)this_rq->nr_uninterruptible -
+ (long)this_rq->bt.nr_uninterruptible;
if (nr_active != this_rq->calc_load_active) {
delta = nr_active - this_rq->calc_load_active;
diff --git a/kernel/time/posix-cpu-timers.c b/kernel/time/posix-cpu-timers.c
index 0a426f4..db54c82 100644
--- a/kernel/time/posix-cpu-timers.c
+++ b/kernel/time/posix-cpu-timers.c
@@ -829,7 +829,7 @@ static void check_thread_timers(struct task_struct *tsk,
tsk_expires->virt_exp = expires;
tsk_expires->sched_exp = check_timers_list(++timers, firing,
- tsk->se.sum_exec_runtime);
+ tsk->se.sum_exec_runtime + tsk->bt.sum_exec_runtime);
/*
* Check for the special case thread timers.
--
1.8.3.1
