While migrating a task, check if moving it improves consolidation.
However make sure that such a movement doesnt offset fairness or create
imbalance in the nodes.
Signed-off-by: Srikar Dronamraju <srikar@xxxxxxxxxxxxxxxxxx>
---
include/linux/sched.h | 1 +
kernel/sched/core.c | 1 +
kernel/sched/fair.c | 80 ++++++++++++++++++++++++++++++++++++++++++------
3 files changed, 72 insertions(+), 10 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e692a02..a77c3cd 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -815,6 +815,7 @@ enum cpu_idle_type {
#define SD_ASYM_PACKING 0x0800 /* Place busy groups earlier in the domain */
#define SD_PREFER_SIBLING 0x1000 /* Prefer to place tasks in a sibling domain */
#define SD_OVERLAP 0x2000 /* sched_domains of this level overlap */
+#define SD_NUMA 0x4000 /* cross-node balancing */
extern int __weak arch_sd_sibiling_asym_packing(void);
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 67d0465..e792312 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6136,6 +6136,7 @@ sd_numa_init(struct sched_domain_topology_level *tl, int cpu)
| 0*SD_SHARE_PKG_RESOURCES
| 1*SD_SERIALIZE
| 0*SD_PREFER_SIBLING
+ | 1*SD_NUMA
| sd_local_flags(level)
,
.last_balance = jiffies,
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 8a2b5aa..3df7f76 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3326,6 +3326,58 @@ static int select_idle_sibling(struct task_struct *p, int target)
return target;
}
+#ifdef CONFIG_NUMA_BALANCING
+/* Move a task if
+ * Return -1 if the task has no numa affinity
+ * Return 1 if the task has numa affinity and moving the destination
+ * runqueue is not already loaded.
+ * else return 0
+ */
+static int
+can_numa_migrate_task(struct task_struct *p, struct rq *dst_rq, struct rq *src_rq)
+{
+ struct mm_struct *mm;
+ int dst_node, src_node;
+ int dst_running, src_running;
+
+ mm = p->mm;
+ if (!mm || !mm->numa_weights)
+ return -1;
+
+ if (atomic_read(&mm->numa_weights[nr_node_ids]) < 2)
+ return -1;
+
+ dst_node = cpu_to_node(cpu_of(dst_rq));
+ src_node = cpu_to_node(cpu_of(src_rq));
+ dst_running = atomic_read(&mm->numa_weights[dst_node]);
+ src_running = atomic_read(&mm->numa_weights[src_node]);
+
+ if (dst_rq->nr_running <= src_rq->nr_running) {
+ if (dst_running * src_rq->nr_running >= src_running * dst_rq->nr_running)
+ return 1;
+ }
+
+ return 0;
+}
+#else
+static int
+can_numa_migrate_task(struct task_struct *p, struct rq *dst_rq, struct rq *src_rq)
+{
+ return -1;
+}
+#endif
+/*
+ * Dont move a task if the source runq has more numa affinity.
+ */
+static bool
+check_numa_affinity(struct task_struct *p, int cpu, int prev_cpu)
+{
+ struct rq *src_rq = cpu_rq(prev_cpu);
+ struct rq *dst_rq = cpu_rq(cpu);
+
+ return (can_numa_migrate_task(p, dst_rq, src_rq) != 0);
+}
+
/*
* sched_balance_self: balance the current task (running on cpu) in domains
* that have the 'flag' flag set. In practice, this is SD_BALANCE_FORK and
@@ -3351,7 +3403,8 @@ select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
return prev_cpu;
if (sd_flag & SD_BALANCE_WAKE) {
- if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)))
+ if (cpumask_test_cpu(cpu, tsk_cpus_allowed(p)) &&
+ check_numa_affinity(p, cpu, prev_cpu))
want_affine = 1;
new_cpu = prev_cpu;
}
@@ -3899,6 +3952,17 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd)
return delta < (s64)sysctl_sched_migration_cost;
}
+static bool force_migrate(struct lb_env *env, struct task_struct *p)
+{
+ if (env->sd->nr_balance_failed > env->sd->cache_nice_tries)
+ return true;
+
+ if (!(env->sd->flags & SD_NUMA))
+ return false;
+
+ return (can_numa_migrate_task(p, env->dst_rq, env->src_rq) == 1);
+}
+
/*
* can_migrate_task - may task p from runqueue rq be migrated to this_cpu?
*/
@@ -3949,21 +4013,17 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env)
* Aggressive migration if:
* 1) task is cache cold, or
* 2) too many balance attempts have failed.
+ * 3) has numa affinity
*/
-
tsk_cache_hot = task_hot(p, env->src_rq->clock_task, env->sd);
- if (!tsk_cache_hot ||
- env->sd->nr_balance_failed > env->sd->cache_nice_tries) {
+ if (tsk_cache_hot) {
+ if (force_migrate(env, p)) {
#ifdef CONFIG_SCHEDSTATS
- if (tsk_cache_hot) {
schedstat_inc(env->sd, lb_hot_gained[env->idle]);
schedstat_inc(p, se.statistics.nr_forced_migrations);
- }
#endif
- return 1;
- }
-
- if (tsk_cache_hot) {
+ return 1;
+ }
schedstat_inc(p, se.statistics.nr_failed_migrations_hot);
return 0;
}
--
1.7.1
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>