Provide a function to raise a softirq vector and force the wakeup of
ksoftirqd along the way, irrespective of the current interrupt context.
This is going to be used by rcutiny to fix and optimize the triggering
of quiescent states from idle.
Fixes: cff9b2332ab7 ("kernel/sched: Modify initial boot task idle setup")
Cc: Liam R. Howlett <Liam.Howlett@xxxxxxxxxx>
Cc: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx>
Cc: Sebastian Siewior <bigeasy@xxxxxxxxxxxxx>
Cc: Thomas Gleixner <tglx@xxxxxxxxxxxxx>
Signed-off-by: Frederic Weisbecker <frederic@xxxxxxxxxx>
---
include/linux/interrupt.h | 1 +
kernel/softirq.c | 71 +++++++++++++++++++++++----------------
2 files changed, 43 insertions(+), 29 deletions(-)
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 558a1a329da9..301d2956e746 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -608,6 +608,7 @@ extern void raise_softirq_no_wake(unsigned int nr);
extern void raise_softirq_irqoff(unsigned int nr);
extern void raise_softirq(unsigned int nr);
+extern void raise_ksoftirqd_irqoff(unsigned int nr);
DECLARE_PER_CPU(struct task_struct *, ksoftirqd);
diff --git a/kernel/softirq.c b/kernel/softirq.c
index acfed6f3701d..9c29a8ced1c3 100644
--- a/kernel/softirq.c
+++ b/kernel/softirq.c
@@ -659,35 +659,6 @@ void irq_exit(void)
lockdep_hardirq_exit();
}
-/*
- * This function must run with irqs disabled!
- */
-inline void raise_softirq_irqoff(unsigned int nr)
-{
- raise_softirq_no_wake(nr);
-
- /*
- * If we're in an interrupt or softirq, we're done
- * (this also catches softirq-disabled code). We will
- * actually run the softirq once we return from
- * the irq or softirq.
- *
- * Otherwise we wake up ksoftirqd to make sure we
- * schedule the softirq soon.
- */
- if (!in_interrupt() && should_wake_ksoftirqd())
- wakeup_softirqd();
-}
-
-void raise_softirq(unsigned int nr)
-{
- unsigned long flags;
-
- local_irq_save(flags);
- raise_softirq_irqoff(nr);
- local_irq_restore(flags);
-}
-
void raise_softirq_no_wake(unsigned int nr)
{
lockdep_assert_irqs_disabled();
@@ -695,6 +666,48 @@ void raise_softirq_no_wake(unsigned int nr)
or_softirq_pending(1UL << nr);
}
+/*
+ * This function must run with irqs disabled!
+ */
+static inline void __raise_softirq_irqoff(unsigned int nr, bool threaded)
+{
+ raise_softirq_no_wake(nr);
+
+ if (threaded && should_wake_ksoftirqd())
+ wakeup_softirqd();
+}
+
+/*
+ * This function must run with irqs disabled!
+ */
+inline void raise_softirq_irqoff(unsigned int nr)
+{
+ bool threaded;
+ /*
+ * If in an interrupt or softirq (servicing or disabled
+ * section), the vector will be handled at the end of
+ * the interrupt or softirq servicing/disabled section.
+ * Otherwise the vector must rely on ksoftirqd.
+ */
+ threaded = !in_interrupt();
+
+ __raise_softirq_irqoff(nr, threaded);
+}
+
+void raise_softirq(unsigned int nr)
+{
+ unsigned long flags;
+
+ local_irq_save(flags);
+ raise_softirq_irqoff(nr);
+ local_irq_restore(flags);
+}
+
+void raise_ksoftirqd_irqoff(unsigned int nr)
+{
+ __raise_softirq_irqoff(nr, true);
+}
+
void open_softirq(int nr, void (*action)(struct softirq_action *))
{
softirq_vec[nr].action = action;
--
2.34.1
