The housekeeping CPU masks, set up by the "isolcpus" and "nohz_full"
boot command line options, are used at boot time to exclude selected CPUs
from running some kernel background processes to minimize disturbance
to latency sensitive userspace applications. Some of housekeeping CPU
masks are also checked at run time to avoid using those isolated CPUs.
The cpuset subsystem is now able to dynamically create a set of isolated
CPUs to be used in isolated cpuset partitions. The long term goal is
to make the degree of isolation as close as possible to what can be
done statically using those boot command line options.
This patch is a step in that direction by making the housekeeping CPU
mask APIs exclude the dynamically isolated CPUs when they are called
at run time. The housekeeping CPU masks will fall back to the bootup
default when all the dynamically isolated CPUs are released.
A new housekeeping_exlude_isolcpus() function is added which is to be
called by the cpuset subsystem to provide a list of isolated CPUs to
be excluded.
Signed-off-by: Waiman Long <longman@xxxxxxxxxx>
---
include/linux/sched/isolation.h | 8 +++
kernel/sched/isolation.c | 112 +++++++++++++++++++++++++++++++-
2 files changed, 119 insertions(+), 1 deletion(-)
diff --git a/include/linux/sched/isolation.h b/include/linux/sched/isolation.h
index 2b461129d1fa..d64fa4e60138 100644
--- a/include/linux/sched/isolation.h
+++ b/include/linux/sched/isolation.h
@@ -27,6 +27,8 @@ extern bool housekeeping_enabled(enum hk_type type);
extern void housekeeping_affine(struct task_struct *t, enum hk_type type);
extern bool housekeeping_test_cpu(int cpu, enum hk_type type);
extern void __init housekeeping_init(void);
+extern int housekeeping_exlude_isolcpus(const struct cpumask *isolcpus,
+ unsigned long flags);
#else
@@ -54,6 +56,12 @@ static inline bool housekeeping_test_cpu(int cpu, enum hk_type type)
}
static inline void housekeeping_init(void) { }
+
+static inline int housekeeping_exlude_isolcpus(struct cpumask *isolcpus,
+ unsigned long flags)
+{
+ return -EOPNOTSUPP;
+}
#endif /* CONFIG_CPU_ISOLATION */
static inline bool housekeeping_cpu(int cpu, enum hk_type type)
diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c
index 5891e715f00d..3018ba81eb65 100644
--- a/kernel/sched/isolation.c
+++ b/kernel/sched/isolation.c
@@ -28,7 +28,16 @@ struct housekeeping {
unsigned long flags;
};
-static struct housekeeping housekeeping;
+static struct housekeeping housekeeping __read_mostly;
+
+/*
+ * Boot time housekeeping cpumask and flags
+ *
+ * If more than one of nohz_full or isolcpus are specified, the cpumask must
+ * be the same or the setup will fail.
+ */
+static cpumask_var_t boot_hk_cpumask;
+static unsigned long boot_hk_flags;
bool housekeeping_enabled(enum hk_type type)
{
@@ -253,3 +262,104 @@ static int __init housekeeping_isolcpus_setup(char *str)
return housekeeping_setup(str, flags);
}
__setup("isolcpus=", housekeeping_isolcpus_setup);
+
+/*
+ * Save bootup housekeeping cpumask and flags
+ */
+static int housekeeping_save(void)
+{
+ enum hk_type type;
+
+ boot_hk_flags = housekeeping.flags;
+ for_each_set_bit(type, &housekeeping.flags, HK_TYPE_MAX) {
+ if (!alloc_cpumask_var(&boot_hk_cpumask, GFP_KERNEL))
+ return -ENOMEM;
+ cpumask_copy(boot_hk_cpumask, housekeeping.cpumasks[type]);
+ break;
+ }
+ return 0;
+}
+
+/*
+ * Exclude the given dynamically isolated CPUs from the housekeeping CPUs
+ * External synchronization is required to make sure that concurrent call to
+ * this function will not happen.
+ *
+ * [TODO] The housekeeping cpumasks and flags at bootup time are currently
+ * preserved as cpuset dynamic CPU isolation isn't as good as boot time CPU
+ * isolation yet. Once dynamic CPU isolation is close to boot time isolation,
+ * we will not need to save the bootup values and will allow them to be
+ * overridden.
+ *
+ * Return: 0 if successful, an error code if not
+ */
+int housekeeping_exlude_isolcpus(const struct cpumask *isolcpus, unsigned long flags)
+{
+ static unsigned long alloc_flags;
+ static cpumask_var_t tmp_mask;
+ static bool excluded; /* @true if some CPUs have been excluded */
+ static bool inited; /* @true if called before */
+
+ bool isolate_none = !isolcpus || cpumask_empty(isolcpus);
+ enum hk_type type;
+
+ lockdep_assert_cpus_held();
+
+ if (isolate_none && (!inited || !excluded))
+ return 0;
+
+ if (unlikely(!inited)) {
+ if (!alloc_cpumask_var(&tmp_mask, GFP_KERNEL))
+ return -ENOMEM;
+ if (housekeeping.flags) {
+ int err = housekeeping_save();
+
+ if (err)
+ return err;
+ }
+ alloc_flags = housekeeping.flags;
+ inited = true;
+ }
+
+ if (isolate_none) {
+ excluded = false;
+
+ /*
+ * Reset housekeeping to bootup default
+ */
+ for_each_set_bit(type, &boot_hk_flags, HK_TYPE_MAX)
+ cpumask_copy(housekeeping.cpumasks[type], boot_hk_cpumask);
+
+ WRITE_ONCE(housekeeping.flags, boot_hk_flags);
+ if (!boot_hk_flags && static_key_enabled(&housekeeping_overridden))
+ static_key_disable_cpuslocked(&housekeeping_overridden.key);
+ return 0;
+ }
+
+ /*
+ * Setting up the new housekeeping cpumasks
+ */
+ for_each_set_bit(type, &flags, HK_TYPE_MAX) {
+ const struct cpumask *src_mask;
+
+ if (!(BIT(type) & alloc_flags)) {
+ if (!alloc_cpumask_var(&housekeeping.cpumasks[type], GFP_KERNEL))
+ return -ENOMEM;
+ alloc_flags |= BIT(type);
+ }
+ src_mask = (BIT(type) & boot_hk_flags)
+ ? boot_hk_cpumask : cpu_possible_mask;
+ /*
+ * Make sure there is at least one online housekeeping CPU
+ */
+ cpumask_andnot(tmp_mask, src_mask, isolcpus);
+ if (!cpumask_intersects(tmp_mask, cpu_online_mask))
+ return -EINVAL; /* Invalid isolated CPUs */
+ cpumask_copy(housekeeping.cpumasks[type], tmp_mask);
+ }
+ WRITE_ONCE(housekeeping.flags, boot_hk_flags | flags);
+ excluded = true;
+ if (!static_key_enabled(&housekeeping_overridden))
+ static_key_enable_cpuslocked(&housekeeping_overridden.key);
+ return 0;
+}
--
2.43.5
