On Wed, Nov 02, 2022 at 06:33:36PM +0000, Valentin Schneider wrote:
> The newly-introduced ipi_send_cpumask tracepoint has a "callback" parameter
> which so far has only been fed with NULL.
>
> While CSD_TYPE_SYNC/ASYNC and CSD_TYPE_IRQ_WORK share a similar backing
> struct layout (meaning their callback func can be accessed without caring
> about the actual CSD type), CSD_TYPE_TTWU doesn't even have a function
> attached to its struct. This means we need to check the type of a CSD
> before eventually dereferencing its associated callback.
>
> This isn't as trivial as it sounds: the CSD type is stored in
> __call_single_node.u_flags, which get cleared right before the callback is
> executed via csd_unlock(). This implies checking the CSD type before it is
> enqueued on the call_single_queue, as the target CPU's queue can be flushed
> before we get to sending an IPI.
>
> Furthermore, send_call_function_single_ipi() only has a CPU parameter, and
> would need to have an additional argument to trickle down the invoked
> function. This is somewhat silly, as the extra argument will always be
> pushed down to the function even when nothing is being traced, which is
> unnecessary overhead.
>
> Two options present themselves:
> a) Create copies of send_call_function_{single_ipi, ipi_mask}() that take
> an extra argument used for tracing, so that codepaths remain unchanged
> when tracing isn't in effect (a sort of manual -fipa-sra).
>
> b) Stash the CSD func in somewhere as a side effect that
> the portion of send_call_function_{single_ipi, ipi_mask}() under the
> tracepoint's static key can fetch.
>
> a) creates redundant code, and b) is quite fragile due to requiring extra
> care for "reentrant" functions (async SMP calls).
>
> This implements a).
>
> Signed-off-by: Valentin Schneider <vschneid@xxxxxxxxxx>
> ---
> kernel/irq_work.c | 2 ++
> kernel/sched/core.c | 35 ++++++++++++++++++++++++-----------
> kernel/sched/smp.h | 1 +
> kernel/smp.c | 42 ++++++++++++++++++++++++++++++++++++++----
> 4 files changed, 65 insertions(+), 15 deletions(-)
>
> diff --git a/kernel/irq_work.c b/kernel/irq_work.c
> index aec38c294ce68..fcfa75c4a5daf 100644
> --- a/kernel/irq_work.c
> +++ b/kernel/irq_work.c
> @@ -24,6 +24,8 @@
>
> #include <trace/events/ipi.h>
>
> +#include "sched/smp.h"
> +
> static DEFINE_PER_CPU(struct llist_head, raised_list);
> static DEFINE_PER_CPU(struct llist_head, lazy_list);
> static DEFINE_PER_CPU(struct task_struct *, irq_workd);
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 02181f8072b5f..41196ca67e913 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -3743,17 +3743,30 @@ void sched_ttwu_pending(void *arg)
> rq_unlock_irqrestore(rq, &rf);
> }
>
> -void send_call_function_single_ipi(int cpu)
> -{
> - struct rq *rq = cpu_rq(cpu);
> -
> - if (!set_nr_if_polling(rq->idle)) {
> - trace_ipi_send_cpumask(cpumask_of(cpu), _RET_IP_, NULL);
> - arch_send_call_function_single_ipi(cpu);
> - } else {
> - trace_sched_wake_idle_without_ipi(cpu);
> - }
> -}
> +/*
> + * We want a variant that traces the function causing the IPI to be sent, but
> + * we don't want the extra argument to cause unnecessary overhead when tracing
> + * isn't happening.
> + */
> +#define GEN_CFSI(suffix, IPI_EXP, ...) \
> +void send_call_function_single_ipi##suffix(__VA_ARGS__) \
> +{ \
> + struct rq *rq = cpu_rq(cpu); \
> + \
> + if (!set_nr_if_polling(rq->idle)) { \
> + IPI_EXP; \
> + arch_send_call_function_single_ipi(cpu); \
> + } else { \
> + trace_sched_wake_idle_without_ipi(cpu); \
> + } \
> +}
> +
> +GEN_CFSI(/* nop */,
> + /* nop */,
> + int cpu)
> +GEN_CFSI(_trace,
> + trace_ipi_send_cpumask(cpumask_of(cpu), _RET_IP_, func),
> + int cpu, smp_call_func_t func)
>
*yuck*
How about something like so?
---
--- a/kernel/irq_work.c
+++ b/kernel/irq_work.c
@@ -24,6 +24,8 @@
#include <trace/events/ipi.h>
+#include "sched/smp.h"
+
static DEFINE_PER_CPU(struct llist_head, raised_list);
static DEFINE_PER_CPU(struct llist_head, lazy_list);
static DEFINE_PER_CPU(struct task_struct *, irq_workd);
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3763,16 +3763,17 @@ void sched_ttwu_pending(void *arg)
rq_unlock_irqrestore(rq, &rf);
}
-void send_call_function_single_ipi(int cpu)
+bool send_call_function_single_ipi(int cpu)
{
struct rq *rq = cpu_rq(cpu);
if (!set_nr_if_polling(rq->idle)) {
- trace_ipi_send_cpumask(cpumask_of(cpu), _RET_IP_, NULL);
arch_send_call_function_single_ipi(cpu);
- } else {
- trace_sched_wake_idle_without_ipi(cpu);
+ return true;
}
+
+ trace_sched_wake_idle_without_ipi(cpu);
+ return false;
}
/*
--- a/kernel/sched/smp.h
+++ b/kernel/sched/smp.h
@@ -6,7 +6,7 @@
extern void sched_ttwu_pending(void *arg);
-extern void send_call_function_single_ipi(int cpu);
+extern bool send_call_function_single_ipi(int cpu);
#ifdef CONFIG_SMP
extern void flush_smp_call_function_queue(void);
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -163,7 +163,6 @@ void __init call_function_init(void)
static inline void
send_call_function_ipi_mask(const struct cpumask *mask)
{
- trace_ipi_send_cpumask(mask, _RET_IP_, func);
arch_send_call_function_ipi_mask(mask);
}
@@ -438,11 +437,16 @@ static void __smp_call_single_queue_debu
struct cfd_seq_local *seq = this_cpu_ptr(&cfd_seq_local);
struct call_function_data *cfd = this_cpu_ptr(&cfd_data);
struct cfd_percpu *pcpu = per_cpu_ptr(cfd->pcpu, cpu);
+ struct __call_single_data *csd;
+
+ csd = container_of(node, call_single_data_t, node.llist);
+ WARN_ON_ONCE(!(CSD_TYPE(csd) & (CSD_TYPE_SYNC | CSD_TYPE_ASYNC)));
cfd_seq_store(pcpu->seq_queue, this_cpu, cpu, CFD_SEQ_QUEUE);
if (llist_add(node, &per_cpu(call_single_queue, cpu))) {
cfd_seq_store(pcpu->seq_ipi, this_cpu, cpu, CFD_SEQ_IPI);
cfd_seq_store(seq->ping, this_cpu, cpu, CFD_SEQ_PING);
+ trace_ipi_send_cpumask(cpumask_of(cpu), _RET_IP_, csd->func);
send_call_function_single_ipi(cpu);
cfd_seq_store(seq->pinged, this_cpu, cpu, CFD_SEQ_PINGED);
} else {
@@ -487,6 +491,27 @@ static __always_inline void csd_unlock(s
static DEFINE_PER_CPU_SHARED_ALIGNED(call_single_data_t, csd_data);
+static __always_inline
+bool raw_smp_call_single_queue(int cpu, struct llist_node *node)
+{
+ /*
+ * The list addition should be visible to the target CPU when it pops
+ * the head of the list to pull the entry off it in the IPI handler
+ * because of normal cache coherency rules implied by the underlying
+ * llist ops.
+ *
+ * If IPIs can go out of order to the cache coherency protocol
+ * in an architecture, sufficient synchronisation should be added
+ * to arch code to make it appear to obey cache coherency WRT
+ * locking and barrier primitives. Generic code isn't really
+ * equipped to do the right thing...
+ */
+ if (llist_add(node, &per_cpu(call_single_queue, cpu)))
+ return send_call_function_single_ipi(cpu);
+
+ return false;
+}
+
void __smp_call_single_queue(int cpu, struct llist_node *node)
{
#ifdef CONFIG_CSD_LOCK_WAIT_DEBUG
@@ -503,19 +528,28 @@ void __smp_call_single_queue(int cpu, st
#endif
/*
- * The list addition should be visible to the target CPU when it pops
- * the head of the list to pull the entry off it in the IPI handler
- * because of normal cache coherency rules implied by the underlying
- * llist ops.
- *
- * If IPIs can go out of order to the cache coherency protocol
- * in an architecture, sufficient synchronisation should be added
- * to arch code to make it appear to obey cache coherency WRT
- * locking and barrier primitives. Generic code isn't really
- * equipped to do the right thing...
- */
- if (llist_add(node, &per_cpu(call_single_queue, cpu)))
- send_call_function_single_ipi(cpu);
+ * We have to check the type of the CSD before queueing it, because
+ * once queued it can have its flags cleared by
+ * flush_smp_call_function_queue()
+ * even if we haven't sent the smp_call IPI yet (e.g. the stopper
+ * executes migration_cpu_stop() on the remote CPU).
+ */
+ if (trace_ipi_send_cpumask_enabled()) {
+ call_single_data_t *csd;
+ smp_call_func_t func;
+
+ csd = container_of(node, call_single_data_t, node.llist);
+
+ func = sched_ttwu_pending;
+ if (CSD_TYPE(csd) != CSD_TYPE_TTWU)
+ func = csd->func;
+
+ if (raw_smp_call_single_queue(cpu, node))
+ trace_ipi_send_cpumask(cpumask_of(cpu), _RET_IP_, func);
+ return;
+ }
+
+ raw_smp_call_single_queue(cpu, node);
}
/*
@@ -983,10 +1017,13 @@ static void smp_call_function_many_cond(
* number of CPUs might be zero due to concurrent changes to the
* provided mask.
*/
- if (nr_cpus == 1)
+ if (nr_cpus == 1) {
+ trace_ipi_send_cpumask(cpumask_of(last_cpu), _RET_IP_, func);
send_call_function_single_ipi(last_cpu);
- else if (likely(nr_cpus > 1))
- send_call_function_ipi_mask(cfd->cpumask_ipi);
+ } else if (likely(nr_cpus > 1)) {
+ trace_ipi_send_cpumask(mask, _RET_IP_, func);
+ send_call_function_ipi_mask(mask);
+ }
cfd_seq_store(this_cpu_ptr(&cfd_seq_local)->pinged, this_cpu, CFD_SEQ_NOCPU, CFD_SEQ_PINGED);
}
