On Wed, 2023-07-12 at 23:07 -0700, Yonghong Song wrote:
> Add interpreter/jit support for new signed div/mod insns.
> The new signed div/mod instructions are encoded with
> unsigned div/mod instructions plus insn->off == 1.
> Also add basic verifier support to ensure new insns get
> accepted.
>
> Signed-off-by: Yonghong Song <yhs@xxxxxx>
> ---
> arch/x86/net/bpf_jit_comp.c | 27 +++++++----
> kernel/bpf/core.c | 96 ++++++++++++++++++++++++++++++-------
> kernel/bpf/verifier.c | 6 ++-
> 3 files changed, 103 insertions(+), 26 deletions(-)
>
> diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
> index adda5e7626b4..3176b60d25c7 100644
> --- a/arch/x86/net/bpf_jit_comp.c
> +++ b/arch/x86/net/bpf_jit_comp.c
> @@ -1194,15 +1194,26 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image
> /* mov rax, dst_reg */
> emit_mov_reg(&prog, is64, BPF_REG_0, dst_reg);
>
> - /*
> - * xor edx, edx
> - * equivalent to 'xor rdx, rdx', but one byte less
> - */
> - EMIT2(0x31, 0xd2);
> + if (insn->off == 0) {
> + /*
> + * xor edx, edx
> + * equivalent to 'xor rdx, rdx', but one byte less
> + */
> + EMIT2(0x31, 0xd2);
>
> - /* div src_reg */
> - maybe_emit_1mod(&prog, src_reg, is64);
> - EMIT2(0xF7, add_1reg(0xF0, src_reg));
> + /* div src_reg */
> + maybe_emit_1mod(&prog, src_reg, is64);
> + EMIT2(0xF7, add_1reg(0xF0, src_reg));
> + } else {
> + if (BPF_CLASS(insn->code) == BPF_ALU)
> + EMIT1(0x99); /* cltd */
> + else
> + EMIT2(0x48, 0x99); /* cqto */
Nitpick: I can't find names cltd/cqto in the Intel instruction manual,
instead it uses names cdq/cqo for these instructions.
(See Vol. 2A pages 3-315 and 3-497)
> +
> + /* idiv src_reg */
> + maybe_emit_1mod(&prog, src_reg, is64);
> + EMIT2(0xF7, add_1reg(0xF8, src_reg));
> + }
>
> if (BPF_OP(insn->code) == BPF_MOD &&
> dst_reg != BPF_REG_3)
> diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c
> index 86bb412fee39..6f7134657935 100644
> --- a/kernel/bpf/core.c
> +++ b/kernel/bpf/core.c
> @@ -1789,36 +1789,100 @@ static u64 ___bpf_prog_run(u64 *regs, const struct bpf_insn *insn)
> (*(s64 *) &DST) >>= IMM;
> CONT;
> ALU64_MOD_X:
> - div64_u64_rem(DST, SRC, &AX);
> - DST = AX;
> + switch (OFF) {
> + case 0:
> + div64_u64_rem(DST, SRC, &AX);
> + DST = AX;
> + break;
> + case 1:
> + AX = div64_s64(DST, SRC);
> + DST = DST - AX * SRC;
> + break;
> + }
> CONT;
> ALU_MOD_X:
> - AX = (u32) DST;
> - DST = do_div(AX, (u32) SRC);
> + switch (OFF) {
> + case 0:
> + AX = (u32) DST;
> + DST = do_div(AX, (u32) SRC);
> + break;
> + case 1:
> + AX = (s32) DST;
> + DST = (u32)do_div(AX, (s32) SRC);
> + break;
> + }
> CONT;
> ALU64_MOD_K:
> - div64_u64_rem(DST, IMM, &AX);
> - DST = AX;
> + switch (OFF) {
> + case 0:
> + div64_u64_rem(DST, IMM, &AX);
> + DST = AX;
> + break;
> + case 1:
> + AX = div64_s64(DST, IMM);
> + DST = DST - AX * IMM;
> + break;
> + }
> CONT;
> ALU_MOD_K:
> - AX = (u32) DST;
> - DST = do_div(AX, (u32) IMM);
> + switch (OFF) {
> + case 0:
> + AX = (u32) DST;
> + DST = do_div(AX, (u32) IMM);
> + break;
> + case 1:
> + AX = (s32) DST;
> + DST = (u32)do_div(AX, (s32) IMM);
> + break;
> + }
> CONT;
> ALU64_DIV_X:
> - DST = div64_u64(DST, SRC);
> + switch (OFF) {
> + case 0:
> + DST = div64_u64(DST, SRC);
> + break;
> + case 1:
> + DST = div64_s64(DST, SRC);
> + break;
> + }
> CONT;
> ALU_DIV_X:
> - AX = (u32) DST;
> - do_div(AX, (u32) SRC);
> - DST = (u32) AX;
> + switch (OFF) {
> + case 0:
> + AX = (u32) DST;
> + do_div(AX, (u32) SRC);
> + DST = (u32) AX;
> + break;
> + case 1:
> + AX = (s32) DST;
> + do_div(AX, (s32) SRC);
> + DST = (u32) AX;
> + break;
> + }
> CONT;
> ALU64_DIV_K:
> - DST = div64_u64(DST, IMM);
> + switch (OFF) {
> + case 0:
> + DST = div64_u64(DST, IMM);
> + break;
> + case 1:
> + DST = div64_s64(DST, IMM);
> + break;
> + }
> CONT;
> ALU_DIV_K:
> - AX = (u32) DST;
> - do_div(AX, (u32) IMM);
> - DST = (u32) AX;
> + switch (OFF) {
> + case 0:
> + AX = (u32) DST;
> + do_div(AX, (u32) IMM);
> + DST = (u32) AX;
> + break;
> + case 1:
> + AX = (s32) DST;
> + do_div(AX, (s32) IMM);
> + DST = (u32) AX;
> + break;
> + }
> CONT;
> ALU_END_TO_BE:
> switch (IMM) {
> diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c
> index 22ba0744547b..b606c8ed5470 100644
> --- a/kernel/bpf/verifier.c
> +++ b/kernel/bpf/verifier.c
> @@ -13192,7 +13192,8 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
> } else { /* all other ALU ops: and, sub, xor, add, ... */
>
> if (BPF_SRC(insn->code) == BPF_X) {
> - if (insn->imm != 0 || insn->off != 0) {
> + if (insn->imm != 0 || insn->off > 1 ||
> + (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
> verbose(env, "BPF_ALU uses reserved fields\n");
> return -EINVAL;
> }
> @@ -13201,7 +13202,8 @@ static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
> if (err)
> return err;
> } else {
> - if (insn->src_reg != BPF_REG_0 || insn->off != 0) {
> + if (insn->src_reg != BPF_REG_0 || insn->off > 1 ||
> + (insn->off == 1 && opcode != BPF_MOD && opcode != BPF_DIV)) {
> verbose(env, "BPF_ALU uses reserved fields\n");
> return -EINVAL;
> }
