On Tue, Feb 25, 2025 at 10:00:43PM -0500, Rik van Riel wrote:
> Context switch and TLB flush support for processes that use a global
> ASID & PCID across all CPUs.
>
> At both context switch time and TLB flush time, we need to check
> whether a task is switching to a global ASID, and reload the TLB
> with the new ASID as appropriate.
>
> In both code paths, we also short-circuit the TLB flush if we
> are using a global ASID, because the global ASIDs are always
> kept up to date across CPUs, even while the process is not
> running on a CPU.
>
> Signed-off-by: Rik van Riel <riel@xxxxxxxxxxx>
> ---
> arch/x86/include/asm/tlbflush.h | 18 ++++++++
> arch/x86/mm/tlb.c | 77 ++++++++++++++++++++++++++++++---
> 2 files changed, 88 insertions(+), 7 deletions(-)
Some touchups:
--- /tmp/current.patch 2025-03-02 08:54:44.821408308 +0100
+++ /tmp/0001-x86-mm-Handle-global-ASID-context-switch-and-TLB-flu.patch 2025-03-02 08:55:27.029190935 +0100
@@ -1,18 +1,23 @@
+From a92847ac925d2849708d036d8bb4920d9b6f2a59 Mon Sep 17 00:00:00 2001
From: Rik van Riel <riel@xxxxxxxxxxx>
Date: Tue, 25 Feb 2025 22:00:43 -0500
-Subject: x86/mm: Global ASID context switch & TLB flush handling
+Subject: [PATCH] x86/mm: Handle global ASID context switch and TLB flush
-Context switch and TLB flush support for processes that use a global
-ASID & PCID across all CPUs.
+Do context switch and TLB flush support for processes that use a global
+ASID and PCID across all CPUs.
-At both context switch time and TLB flush time, we need to check
-whether a task is switching to a global ASID, and reload the TLB
-with the new ASID as appropriate.
-
-In both code paths, we also short-circuit the TLB flush if we
-are using a global ASID, because the global ASIDs are always
-kept up to date across CPUs, even while the process is not
-running on a CPU.
+At both context switch time and TLB flush time, it needs to be checked whether
+a task is switching to a global ASID, and, if so, reload the TLB with the new
+ASID as appropriate.
+
+In both code paths, the TLB flush is avoided if a global ASID is used, because
+the global ASIDs are always kept up to date across CPUs, even when the
+process is not running on a CPU.
+
+ [ bp:
+ - Massage
+ - :%s/\<static_cpu_has\>/cpu_feature_enabled/cgi
+ ]
Signed-off-by: Rik van Riel <riel@xxxxxxxxxxx>
Signed-off-by: Borislav Petkov (AMD) <bp@xxxxxxxxx>
@@ -66,7 +71,7 @@ index 8e7df0ed7005..37b735dcf025 100644
#ifdef CONFIG_PARAVIRT
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
-index 9b1652c02452..b7d461db1b08 100644
+index d79ebdf095e1..cb43ab08ea4a 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -227,6 +227,20 @@ static void choose_new_asid(struct mm_struct *next, u64 next_tlb_gen,
@@ -77,7 +82,7 @@ index 9b1652c02452..b7d461db1b08 100644
+ * TLB consistency for global ASIDs is maintained with hardware assisted
+ * remote TLB flushing. Global ASIDs are always up to date.
+ */
-+ if (static_cpu_has(X86_FEATURE_INVLPGB)) {
++ if (cpu_feature_enabled(X86_FEATURE_INVLPGB)) {
+ u16 global_asid = mm_global_asid(next);
+
+ if (global_asid) {
@@ -90,22 +95,22 @@ index 9b1652c02452..b7d461db1b08 100644
if (this_cpu_read(cpu_tlbstate.invalidate_other))
clear_asid_other();
-@@ -391,6 +405,23 @@ void mm_free_global_asid(struct mm_struct *mm)
+@@ -396,6 +410,23 @@ void mm_free_global_asid(struct mm_struct *mm)
#endif
}
+/*
+ * Is the mm transitioning from a CPU-local ASID to a global ASID?
+ */
-+static bool needs_global_asid_reload(struct mm_struct *next, u16 prev_asid)
++static bool mm_needs_global_asid(struct mm_struct *mm, u16 asid)
+{
-+ u16 global_asid = mm_global_asid(next);
++ u16 global_asid = mm_global_asid(mm);
+
-+ if (!static_cpu_has(X86_FEATURE_INVLPGB))
++ if (!cpu_feature_enabled(X86_FEATURE_INVLPGB))
+ return false;
+
+ /* Process is transitioning to a global ASID */
-+ if (global_asid && prev_asid != global_asid)
++ if (global_asid && asid != global_asid)
+ return true;
+
+ return false;
@@ -124,19 +129,19 @@ index 9b1652c02452..b7d461db1b08 100644
next->context.ctx_id);
/*
-@@ -713,6 +745,20 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
+@@ -718,6 +750,20 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
!cpumask_test_cpu(cpu, mm_cpumask(next))))
cpumask_set_cpu(cpu, mm_cpumask(next));
+ /* Check if the current mm is transitioning to a global ASID */
-+ if (needs_global_asid_reload(next, prev_asid)) {
++ if (mm_needs_global_asid(next, prev_asid)) {
+ next_tlb_gen = atomic64_read(&next->context.tlb_gen);
+ choose_new_asid(next, next_tlb_gen, &new_asid, &need_flush);
+ goto reload_tlb;
+ }
+
+ /*
-+ * Broadcast TLB invalidation keeps this PCID up to date
++ * Broadcast TLB invalidation keeps this ASID up to date
+ * all the time.
+ */
+ if (is_global_asid(prev_asid))
@@ -145,13 +150,13 @@ index 9b1652c02452..b7d461db1b08 100644
/*
* If the CPU is not in lazy TLB mode, we are just switching
* from one thread in a process to another thread in the same
-@@ -746,6 +792,13 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
+@@ -751,6 +797,13 @@ void switch_mm_irqs_off(struct mm_struct *unused, struct mm_struct *next,
*/
cond_mitigation(tsk);
+ /*
+ * Let nmi_uaccess_okay() and finish_asid_transition()
-+ * know that we're changing CR3.
++ * know that CR3 is changing.
+ */
+ this_cpu_write(cpu_tlbstate.loaded_mm, LOADED_MM_SWITCHING);
+ barrier();
@@ -185,12 +190,12 @@ index 9b1652c02452..b7d461db1b08 100644
bool local = smp_processor_id() == f->initiating_cpu;
unsigned long nr_invalidate = 0;
u64 mm_tlb_gen;
-@@ -909,6 +960,16 @@ static void flush_tlb_func(void *info)
+@@ -914,6 +965,16 @@ static void flush_tlb_func(void *info)
if (unlikely(loaded_mm == &init_mm))
return;
+ /* Reload the ASID if transitioning into or out of a global ASID */
-+ if (needs_global_asid_reload(loaded_mm, loaded_mm_asid)) {
++ if (mm_needs_global_asid(loaded_mm, loaded_mm_asid)) {
+ switch_mm_irqs_off(NULL, loaded_mm, NULL);
+ loaded_mm_asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
+ }
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
