From: Alex Belits <abelits@xxxxxxxxxxx>
The current implementation of cpumask_local_spread() does not
respect the isolated CPUs, i.e., even if a CPU has been isolated
for Real-Time task, it will return it to the caller for pinning
of its IRQ threads. Having these unwanted IRQ threads on an
isolated CPU adds up to a latency overhead.
This patch restricts the CPUs that are returned for spreading
IRQs only to the available housekeeping CPUs.
Signed-off-by: Alex Belits <abelits@xxxxxxxxxxx>
Signed-off-by: Nitesh Narayan Lal <nitesh@xxxxxxxxxx>
---
lib/cpumask.c | 43 +++++++++++++++++++++++++------------------
1 file changed, 25 insertions(+), 18 deletions(-)
diff --git a/lib/cpumask.c b/lib/cpumask.c
index fb22fb266f93..cc4311a8c079 100644
--- a/lib/cpumask.c
+++ b/lib/cpumask.c
@@ -6,6 +6,7 @@
#include <linux/export.h>
#include <linux/memblock.h>
#include <linux/numa.h>
+#include <linux/sched/isolation.h>
/**
* cpumask_next - get the next cpu in a cpumask
@@ -205,28 +206,34 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask)
*/
unsigned int cpumask_local_spread(unsigned int i, int node)
{
- int cpu;
+ int cpu, m, n, hk_flags;
+ const struct cpumask *mask;
+ hk_flags = HK_FLAG_DOMAIN | HK_FLAG_WQ;
+ mask = housekeeping_cpumask(hk_flags);
+ m = cpumask_weight(mask);
/* Wrap: we always want a cpu. */
- i %= num_online_cpus();
+ n = i % m;
+ while (m-- > 0) {
+ if (node == NUMA_NO_NODE) {
+ for_each_cpu(cpu, mask)
+ if (n-- == 0)
+ return cpu;
+ } else {
+ /* NUMA first. */
+ for_each_cpu_and(cpu, cpumask_of_node(node), mask)
+ if (n-- == 0)
+ return cpu;
- if (node == NUMA_NO_NODE) {
- for_each_cpu(cpu, cpu_online_mask)
- if (i-- == 0)
- return cpu;
- } else {
- /* NUMA first. */
- for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask)
- if (i-- == 0)
- return cpu;
+ for_each_cpu(cpu, mask) {
+ /* Skip NUMA nodes, done above. */
+ if (cpumask_test_cpu(cpu,
+ cpumask_of_node(node)))
+ continue;
- for_each_cpu(cpu, cpu_online_mask) {
- /* Skip NUMA nodes, done above. */
- if (cpumask_test_cpu(cpu, cpumask_of_node(node)))
- continue;
-
- if (i-- == 0)
- return cpu;
+ if (n-- == 0)
+ return cpu;
+ }
}
}
BUG();
--
2.18.4
