Currently RESTORE_CR3 does an unconditional flush (SAVE_AND_SWITCH_TO_KERNEL_CR3 does not set bit 63 on \save_reg). When restoring to a user ASID, check the user_asid_flush_mask to see if we can avoid the flush. For kernel ASIDs we can unconditionaly avoid the flush, since we do explicit flushes for them. Signed-off-by: Peter Zijlstra (Intel) <peterz@xxxxxxxxxxxxx> --- arch/x86/entry/calling.h | 29 +++++++++++++++++++++++++++-- arch/x86/entry/entry_64.S | 4 ++-- 2 files changed, 29 insertions(+), 4 deletions(-) --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -263,8 +263,33 @@ For 32-bit we have the following convent .Ldone_\@: .endm -.macro RESTORE_CR3 save_reg:req +.macro RESTORE_CR3 scratch_reg:req save_reg:req STATIC_JUMP_IF_FALSE .Lend_\@, kaiser_enabled_key, def=1 + + /* ASID bit 11 is for user */ + bt $11, \save_reg + /* + * KERNEL pages can always resume with NOFLUSH as we do + * explicit flushes. + */ + jnc .Lnoflush_\@ + + /* + * Check if there's a pending flush for the user ASID we're + * about to set. + */ + movq \save_reg, \scratch_reg + andq $(0x7FF), \scratch_reg + bt \scratch_reg, PER_CPU_VAR(user_asid_flush_mask) + jnc .Lnoflush_\@ + + btr \scratch_reg, PER_CPU_VAR(user_asid_flush_mask) + jmp .Ldo_\@ + +.Lnoflush_\@: + ALTERNATIVE "", "bts $63, \save_reg", X86_FEATURE_PCID + +.Ldo_\@: /* * The CR3 write could be avoided when not changing its value, * but would require a CR3 read *and* a scratch register. @@ -281,7 +306,7 @@ For 32-bit we have the following convent .endm .macro SAVE_AND_SWITCH_TO_KERNEL_CR3 scratch_reg:req save_reg:req .endm -.macro RESTORE_CR3 save_reg:req +.macro RESTORE_CR3 scratch_reg:req save_reg:req .endm #endif --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -1278,7 +1278,7 @@ ENTRY(paranoid_exit) testl %ebx, %ebx /* swapgs needed? */ jnz .Lparanoid_exit_no_swapgs TRACE_IRQS_IRETQ - RESTORE_CR3 save_reg=%r14 + RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 SWAPGS_UNSAFE_STACK jmp .Lparanoid_exit_restore .Lparanoid_exit_no_swapgs: @@ -1720,7 +1720,7 @@ ENTRY(nmi) movq $-1, %rsi call do_nmi - RESTORE_CR3 save_reg=%r14 + RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 testl %ebx, %ebx /* swapgs needed? */ jnz nmi_restore -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@xxxxxxxxx. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>
