The previous patch changed the do_user_addr_fault page fault handler to
invoke BPF's fixup routines (by searching exception tables and calling
ex_handler_bpf). This would only occur when SMAP is enabled, such that
any user address access from BPF programs running in kernel mode would
reach this path and invoke the fixup routines.
Relying on this behavior, disable any bounds checking instrumentation in
the BPF JIT for x86 when X86_FEATURE_SMAP is available. All BPF
programs execute with SMAP enabled, therefore when this feature is
available, we can assume that SMAP will be enabled during program
execution at runtime.
This optimizes PROBE_MEM loads down to a normal unchecked load
instruction. Any page faults for user or kernel addresses will be
handled using the fixup routines, and the generation exception table
entries for such load instructions.
All in all, this ensures that PROBE_MEM loads will now incur no runtime
overhead, and become practically free.
Signed-off-by: Kumar Kartikeya Dwivedi <memxor@xxxxxxxxx>
---
arch/x86/net/bpf_jit_comp.c | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 5159c7a22922..f8a39189cddc 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -1864,8 +1864,8 @@ st: if (is_imm8(insn->off))
case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
insn_off = insn->off;
- if (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
- BPF_MODE(insn->code) == BPF_PROBE_MEMSX) {
+ if ((BPF_MODE(insn->code) == BPF_PROBE_MEM ||
+ BPF_MODE(insn->code) == BPF_PROBE_MEMSX) && !cpu_feature_enabled(X86_FEATURE_SMAP)) {
/* Conservatively check that src_reg + insn->off is a kernel address:
* src_reg + insn->off > TASK_SIZE_MAX + PAGE_SIZE
* and
@@ -1912,6 +1912,9 @@ st: if (is_imm8(insn->off))
/* populate jmp_offset for JAE above to jump to start_of_ldx */
start_of_ldx = prog;
end_of_jmp[-1] = start_of_ldx - end_of_jmp;
+ } else if ((BPF_MODE(insn->code) == BPF_PROBE_MEM ||
+ BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) {
+ start_of_ldx = prog;
}
if (BPF_MODE(insn->code) == BPF_PROBE_MEMSX ||
BPF_MODE(insn->code) == BPF_MEMSX)
@@ -1924,9 +1927,13 @@ st: if (is_imm8(insn->off))
u8 *_insn = image + proglen + (start_of_ldx - temp);
s64 delta;
+ if (cpu_feature_enabled(X86_FEATURE_SMAP))
+ goto extable_fixup;
+
/* populate jmp_offset for JMP above */
start_of_ldx[-1] = prog - start_of_ldx;
+ extable_fixup:
if (!bpf_prog->aux->extable)
break;
--
2.43.0
