Use recorded bpf_capable flag of a BPF program in
bpf_jit_charge_modmem(), if possible. If BPF program is unavailable,
fallback to system-level bpf_capable() check. This is currently the case
for BPF trampoline update code, which might not be associated with
a particular instance of BPF program.
All the other users of bpf_jit_charge_modmem() do work within a context
of a specific BPF program, so can trivially use prog->aux->bpf_capable.
Signed-off-by: Andrii Nakryiko <andrii@xxxxxxxxxx>
---
arch/arm/net/bpf_jit_32.c | 2 +-
arch/arm64/net/bpf_jit_comp.c | 2 +-
arch/loongarch/net/bpf_jit.c | 2 +-
arch/mips/net/bpf_jit_comp.c | 2 +-
arch/powerpc/net/bpf_jit_comp.c | 2 +-
arch/riscv/net/bpf_jit_core.c | 3 ++-
arch/s390/net/bpf_jit_comp.c | 3 ++-
arch/sparc/net/bpf_jit_comp_64.c | 2 +-
arch/x86/net/bpf_jit_comp.c | 3 ++-
arch/x86/net/bpf_jit_comp32.c | 2 +-
include/linux/bpf.h | 2 +-
include/linux/filter.h | 6 ++++--
kernel/bpf/core.c | 20 +++++++++++---------
kernel/bpf/trampoline.c | 2 +-
14 files changed, 30 insertions(+), 23 deletions(-)
diff --git a/arch/arm/net/bpf_jit_32.c b/arch/arm/net/bpf_jit_32.c
index 6a1c9fca5260..82dae8e4d6b1 100644
--- a/arch/arm/net/bpf_jit_32.c
+++ b/arch/arm/net/bpf_jit_32.c
@@ -1963,7 +1963,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
image_size = sizeof(u32) * ctx.idx;
/* Now we know the size of the structure to make */
- header = bpf_jit_binary_alloc(image_size, &image_ptr,
+ header = bpf_jit_binary_alloc(prog, image_size, &image_ptr,
sizeof(u32), jit_fill_hole);
/* Not able to allocate memory for the structure then
* we must fall back to the interpretation
diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index b26da8efa616..5c60d9922a03 100644
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -1533,7 +1533,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
/* also allocate space for plt target */
extable_offset = round_up(prog_size + PLT_TARGET_SIZE, extable_align);
image_size = extable_offset + extable_size;
- header = bpf_jit_binary_alloc(image_size, &image_ptr,
+ header = bpf_jit_binary_alloc(prog, image_size, &image_ptr,
sizeof(u32), jit_fill_hole);
if (header == NULL) {
prog = orig_prog;
diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c
index db9342b2d0e6..c803581e87db 100644
--- a/arch/loongarch/net/bpf_jit.c
+++ b/arch/loongarch/net/bpf_jit.c
@@ -1172,7 +1172,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
prog_size = sizeof(u32) * ctx.idx;
image_size = prog_size + extable_size;
/* Now we know the size of the structure to make */
- header = bpf_jit_binary_alloc(image_size, &image_ptr,
+ header = bpf_jit_binary_alloc(prog, image_size, &image_ptr,
sizeof(u32), jit_fill_hole);
if (header == NULL) {
prog = orig_prog;
diff --git a/arch/mips/net/bpf_jit_comp.c b/arch/mips/net/bpf_jit_comp.c
index a40d926b6513..365849adc238 100644
--- a/arch/mips/net/bpf_jit_comp.c
+++ b/arch/mips/net/bpf_jit_comp.c
@@ -985,7 +985,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
/* Now we know the size of the structure to make */
image_size = sizeof(u32) * ctx.jit_index;
- header = bpf_jit_binary_alloc(image_size, &image_ptr,
+ header = bpf_jit_binary_alloc(prog, image_size, &image_ptr,
sizeof(u32), jit_fill_hole);
/*
* Not able to allocate memory for the structure then
diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c
index e93aefcfb83f..fb7732fe2e7e 100644
--- a/arch/powerpc/net/bpf_jit_comp.c
+++ b/arch/powerpc/net/bpf_jit_comp.c
@@ -154,7 +154,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
proglen = cgctx.idx * 4;
alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len;
- bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns);
+ bpf_hdr = bpf_jit_binary_alloc(fp, alloclen, &image, 4, bpf_jit_fill_ill_insns);
if (!bpf_hdr) {
fp = org_fp;
goto out_addrs;
diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c
index 737baf8715da..570567e02dc7 100644
--- a/arch/riscv/net/bpf_jit_core.c
+++ b/arch/riscv/net/bpf_jit_core.c
@@ -109,7 +109,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
prog_size = sizeof(*ctx->insns) * ctx->ninsns;
jit_data->header =
- bpf_jit_binary_alloc(prog_size + extable_size,
+ bpf_jit_binary_alloc(prog,
+ prog_size + extable_size,
&jit_data->image,
sizeof(u32),
bpf_fill_ill_insns);
diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index f95d7e401b96..11627295e695 100644
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -1878,7 +1878,8 @@ static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
__alignof__(struct exception_table_entry));
extable_size = fp->aux->num_exentries *
sizeof(struct exception_table_entry);
- header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf,
+ header = bpf_jit_binary_alloc(fp,
+ code_size + extable_size, &jit->prg_buf,
8, jit_fill_hole);
if (!header)
return NULL;
diff --git a/arch/sparc/net/bpf_jit_comp_64.c b/arch/sparc/net/bpf_jit_comp_64.c
index fa0759bfe498..bff669698e86 100644
--- a/arch/sparc/net/bpf_jit_comp_64.c
+++ b/arch/sparc/net/bpf_jit_comp_64.c
@@ -1567,7 +1567,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
/* Now we know the actual image size. */
image_size = sizeof(u32) * ctx.idx;
- header = bpf_jit_binary_alloc(image_size, &image_ptr,
+ header = bpf_jit_binary_alloc(prog, image_size, &image_ptr,
sizeof(u32), jit_fill_hole);
if (header == NULL) {
prog = orig_prog;
diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 1056bbf55b17..593c9daad167 100644
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -2556,7 +2556,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
sizeof(struct exception_table_entry);
/* allocate module memory for x86 insns and extable */
- header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,
+ header = bpf_jit_binary_pack_alloc(prog,
+ roundup(proglen, align) + extable_size,
&image, align, &rw_header, &rw_image,
jit_fill_hole);
if (!header) {
diff --git a/arch/x86/net/bpf_jit_comp32.c b/arch/x86/net/bpf_jit_comp32.c
index 429a89c5468b..e59ff8935b12 100644
--- a/arch/x86/net/bpf_jit_comp32.c
+++ b/arch/x86/net/bpf_jit_comp32.c
@@ -2586,7 +2586,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
break;
}
if (proglen == oldproglen) {
- header = bpf_jit_binary_alloc(proglen, &image,
+ header = bpf_jit_binary_alloc(prog, proglen, &image,
1, jit_fill_hole);
if (!header) {
prog = orig_prog;
diff --git a/include/linux/bpf.h b/include/linux/bpf.h
index a5832a69f24e..785b720358f5 100644
--- a/include/linux/bpf.h
+++ b/include/linux/bpf.h
@@ -1277,7 +1277,7 @@ void bpf_image_ksym_add(void *data, struct bpf_ksym *ksym);
void bpf_image_ksym_del(struct bpf_ksym *ksym);
void bpf_ksym_add(struct bpf_ksym *ksym);
void bpf_ksym_del(struct bpf_ksym *ksym);
-int bpf_jit_charge_modmem(u32 size);
+int bpf_jit_charge_modmem(u32 size, const struct bpf_prog *prog);
void bpf_jit_uncharge_modmem(u32 size);
bool bpf_prog_has_trampoline(const struct bpf_prog *prog);
#else
diff --git a/include/linux/filter.h b/include/linux/filter.h
index bbce89937fde..9c207d9848e9 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -1026,7 +1026,8 @@ typedef void (*bpf_jit_fill_hole_t)(void *area, unsigned int size);
void bpf_jit_fill_hole_with_zero(void *area, unsigned int size);
struct bpf_binary_header *
-bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
+bpf_jit_binary_alloc(const struct bpf_prog *prog,
+ unsigned int proglen, u8 **image_ptr,
unsigned int alignment,
bpf_jit_fill_hole_t bpf_fill_ill_insns);
void bpf_jit_binary_free(struct bpf_binary_header *hdr);
@@ -1047,7 +1048,8 @@ static inline bool bpf_prog_kallsyms_verify_off(const struct bpf_prog *fp)
}
struct bpf_binary_header *
-bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **ro_image,
+bpf_jit_binary_pack_alloc(const struct bpf_prog *prog,
+ unsigned int proglen, u8 **ro_image,
unsigned int alignment,
struct bpf_binary_header **rw_hdr,
u8 **rw_image,
diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
index 7421487422d4..4d057d39c286 100644
--- a/kernel/bpf/core.c
+++ b/kernel/bpf/core.c
@@ -978,13 +978,13 @@ static int __init bpf_jit_charge_init(void)
}
pure_initcall(bpf_jit_charge_init);
-int bpf_jit_charge_modmem(u32 size)
+int bpf_jit_charge_modmem(u32 size, const struct bpf_prog *prog)
{
if (atomic_long_add_return(size, &bpf_jit_current) > READ_ONCE(bpf_jit_limit)) {
- if (!bpf_capable()) {
- atomic_long_sub(size, &bpf_jit_current);
- return -EPERM;
- }
+ if (prog ? prog->aux->bpf_capable : bpf_capable())
+ return 0;
+ atomic_long_sub(size, &bpf_jit_current);
+ return -EPERM;
}
return 0;
@@ -1006,7 +1006,8 @@ void __weak bpf_jit_free_exec(void *addr)
}
struct bpf_binary_header *
-bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
+bpf_jit_binary_alloc(const struct bpf_prog *prog,
+ unsigned int proglen, u8 **image_ptr,
unsigned int alignment,
bpf_jit_fill_hole_t bpf_fill_ill_insns)
{
@@ -1022,7 +1023,7 @@ bpf_jit_binary_alloc(unsigned int proglen, u8 **image_ptr,
*/
size = round_up(proglen + sizeof(*hdr) + 128, PAGE_SIZE);
- if (bpf_jit_charge_modmem(size))
+ if (bpf_jit_charge_modmem(size, prog))
return NULL;
hdr = bpf_jit_alloc_exec(size);
if (!hdr) {
@@ -1061,7 +1062,8 @@ void bpf_jit_binary_free(struct bpf_binary_header *hdr)
* the JITed program to the RO memory.
*/
struct bpf_binary_header *
-bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr,
+bpf_jit_binary_pack_alloc(const struct bpf_prog *prog,
+ unsigned int proglen, u8 **image_ptr,
unsigned int alignment,
struct bpf_binary_header **rw_header,
u8 **rw_image,
@@ -1076,7 +1078,7 @@ bpf_jit_binary_pack_alloc(unsigned int proglen, u8 **image_ptr,
/* add 16 bytes for a random section of illegal instructions */
size = round_up(proglen + sizeof(*ro_header) + 16, BPF_PROG_CHUNK_SIZE);
- if (bpf_jit_charge_modmem(size))
+ if (bpf_jit_charge_modmem(size, prog))
return NULL;
ro_header = bpf_prog_pack_alloc(size, bpf_fill_ill_insns);
if (!ro_header) {
diff --git a/kernel/bpf/trampoline.c b/kernel/bpf/trampoline.c
index ac021bc43a66..b464807f4b62 100644
--- a/kernel/bpf/trampoline.c
+++ b/kernel/bpf/trampoline.c
@@ -355,7 +355,7 @@ static struct bpf_tramp_image *bpf_tramp_image_alloc(u64 key, u32 idx)
if (!im)
goto out;
- err = bpf_jit_charge_modmem(PAGE_SIZE);
+ err = bpf_jit_charge_modmem(PAGE_SIZE, NULL);
if (err)
goto out_free_im;
--
2.34.1
