From: Junaid Shahid <junaids@xxxxxxxxxx>
When ASI is active, __get_current_cr3_fast() adjusts the returned CR3
value accordingly to reflect the actual ASI CR3.
Signed-off-by: Junaid Shahid <junaids@xxxxxxxxxx>
Signed-off-by: Brendan Jackman <jackmanb@xxxxxxxxxx>
---
arch/x86/mm/tlb.c | 27 +++++++++++++++++++++++----
1 file changed, 23 insertions(+), 4 deletions(-)
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 34d61b56d33f..02f73a71d4ea 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -18,6 +18,7 @@
#include <asm/cache.h>
#include <asm/cacheflush.h>
#include <asm/apic.h>
+#include <asm/asi.h>
#include <asm/perf_event.h>
#include "mm_internal.h"
@@ -1125,14 +1126,32 @@ void flush_tlb_kernel_range(unsigned long start, unsigned long end)
*/
inline_or_noinstr unsigned long __get_current_cr3_fast(void)
{
- unsigned long cr3 =
- build_cr3(this_cpu_read(cpu_tlbstate.loaded_mm)->pgd,
- this_cpu_read(cpu_tlbstate.loaded_mm_asid),
- tlbstate_lam_cr3_mask());
+ unsigned long cr3;
+ pgd_t *pgd;
+ u16 asid = this_cpu_read(cpu_tlbstate.loaded_mm_asid);
+ struct asi *asi = asi_get_current();
+ u16 pcid;
+
+ if (asi) {
+ pgd = asi_pgd(asi);
+ pcid = asi_pcid(asi, asid);
+ } else {
+ pgd = this_cpu_read(cpu_tlbstate.loaded_mm)->pgd;
+ pcid = kern_pcid(asid);
+ }
+
+ cr3 = build_cr3_pcid(pgd, pcid, tlbstate_lam_cr3_mask(), false);
/* For now, be very restrictive about when this can be called. */
VM_WARN_ON(in_nmi() || preemptible());
+ /*
+ * Outside of the ASI critical section, an ASI-restricted CR3 is
+ * unstable because an interrupt (including an inner interrupt, if we're
+ * already in one) could cause a persistent asi_exit.
+ */
+ VM_WARN_ON_ONCE(asi && (asi_is_relaxed() || asi_intr_nest_depth()));
+
VM_BUG_ON(cr3 != __read_cr3());
return cr3;
}
--
2.45.2.993.g49e7a77208-goog
