Re: [PATCH bpf-next 2/4] bpf: inline bpf_get_smp_processor_id() helper

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



Hi Andrii,

kernel test robot noticed the following build warnings:

[auto build test WARNING on bpf-next/master]

url:    https://github.com/intel-lab-lkp/linux/commits/Andrii-Nakryiko/bpf-add-internal-only-per-CPU-LDX-instructions/20240330-025035
base:   https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git master
patch link:    https://lore.kernel.org/r/20240329184740.4084786-3-andrii%40kernel.org
patch subject: [PATCH bpf-next 2/4] bpf: inline bpf_get_smp_processor_id() helper
config: x86_64-randconfig-123-20240330 (https://download.01.org/0day-ci/archive/20240331/202403310434.Sx0Qe1lY-lkp@xxxxxxxxx/config)
compiler: clang version 17.0.6 (https://github.com/llvm/llvm-project 6009708b4367171ccdbf4b5905cb6a803753fe18)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240331/202403310434.Sx0Qe1lY-lkp@xxxxxxxxx/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@xxxxxxxxx>
| Closes: https://lore.kernel.org/oe-kbuild-all/202403310434.Sx0Qe1lY-lkp@xxxxxxxxx/

sparse warnings: (new ones prefixed by >>)
>> kernel/bpf/verifier.c:20078:39: sparse: sparse: cast removes address space '__percpu' of expression
   kernel/bpf/verifier.c:20203:38: sparse: sparse: subtraction of functions? Share your drugs
   kernel/bpf/verifier.c: note: in included file (through include/linux/bpf.h, include/linux/bpf-cgroup.h):
   include/linux/bpfptr.h:65:40: sparse: sparse: cast to non-scalar
   include/linux/bpfptr.h:65:40: sparse: sparse: cast from non-scalar
   include/linux/bpfptr.h:65:40: sparse: sparse: cast to non-scalar
   include/linux/bpfptr.h:65:40: sparse: sparse: cast from non-scalar
   include/linux/bpfptr.h:65:40: sparse: sparse: cast to non-scalar
   include/linux/bpfptr.h:65:40: sparse: sparse: cast from non-scalar
   include/linux/bpfptr.h:65:40: sparse: sparse: cast to non-scalar
   include/linux/bpfptr.h:65:40: sparse: sparse: cast from non-scalar

vim +/__percpu +20078 kernel/bpf/verifier.c

 19587	
 19588	/* Do various post-verification rewrites in a single program pass.
 19589	 * These rewrites simplify JIT and interpreter implementations.
 19590	 */
 19591	static int do_misc_fixups(struct bpf_verifier_env *env)
 19592	{
 19593		struct bpf_prog *prog = env->prog;
 19594		enum bpf_attach_type eatype = prog->expected_attach_type;
 19595		enum bpf_prog_type prog_type = resolve_prog_type(prog);
 19596		struct bpf_insn *insn = prog->insnsi;
 19597		const struct bpf_func_proto *fn;
 19598		const int insn_cnt = prog->len;
 19599		const struct bpf_map_ops *ops;
 19600		struct bpf_insn_aux_data *aux;
 19601		struct bpf_insn insn_buf[16];
 19602		struct bpf_prog *new_prog;
 19603		struct bpf_map *map_ptr;
 19604		int i, ret, cnt, delta = 0, cur_subprog = 0;
 19605		struct bpf_subprog_info *subprogs = env->subprog_info;
 19606		u16 stack_depth = subprogs[cur_subprog].stack_depth;
 19607		u16 stack_depth_extra = 0;
 19608	
 19609		if (env->seen_exception && !env->exception_callback_subprog) {
 19610			struct bpf_insn patch[] = {
 19611				env->prog->insnsi[insn_cnt - 1],
 19612				BPF_MOV64_REG(BPF_REG_0, BPF_REG_1),
 19613				BPF_EXIT_INSN(),
 19614			};
 19615	
 19616			ret = add_hidden_subprog(env, patch, ARRAY_SIZE(patch));
 19617			if (ret < 0)
 19618				return ret;
 19619			prog = env->prog;
 19620			insn = prog->insnsi;
 19621	
 19622			env->exception_callback_subprog = env->subprog_cnt - 1;
 19623			/* Don't update insn_cnt, as add_hidden_subprog always appends insns */
 19624			mark_subprog_exc_cb(env, env->exception_callback_subprog);
 19625		}
 19626	
 19627		for (i = 0; i < insn_cnt;) {
 19628			if (insn->code == (BPF_ALU64 | BPF_MOV | BPF_X) && insn->imm) {
 19629				if ((insn->off == BPF_ADDR_SPACE_CAST && insn->imm == 1) ||
 19630				    (((struct bpf_map *)env->prog->aux->arena)->map_flags & BPF_F_NO_USER_CONV)) {
 19631					/* convert to 32-bit mov that clears upper 32-bit */
 19632					insn->code = BPF_ALU | BPF_MOV | BPF_X;
 19633					/* clear off and imm, so it's a normal 'wX = wY' from JIT pov */
 19634					insn->off = 0;
 19635					insn->imm = 0;
 19636				} /* cast from as(0) to as(1) should be handled by JIT */
 19637				goto next_insn;
 19638			}
 19639	
 19640			if (env->insn_aux_data[i + delta].needs_zext)
 19641				/* Convert BPF_CLASS(insn->code) == BPF_ALU64 to 32-bit ALU */
 19642				insn->code = BPF_ALU | BPF_OP(insn->code) | BPF_SRC(insn->code);
 19643	
 19644			/* Make divide-by-zero exceptions impossible. */
 19645			if (insn->code == (BPF_ALU64 | BPF_MOD | BPF_X) ||
 19646			    insn->code == (BPF_ALU64 | BPF_DIV | BPF_X) ||
 19647			    insn->code == (BPF_ALU | BPF_MOD | BPF_X) ||
 19648			    insn->code == (BPF_ALU | BPF_DIV | BPF_X)) {
 19649				bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
 19650				bool isdiv = BPF_OP(insn->code) == BPF_DIV;
 19651				struct bpf_insn *patchlet;
 19652				struct bpf_insn chk_and_div[] = {
 19653					/* [R,W]x div 0 -> 0 */
 19654					BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
 19655						     BPF_JNE | BPF_K, insn->src_reg,
 19656						     0, 2, 0),
 19657					BPF_ALU32_REG(BPF_XOR, insn->dst_reg, insn->dst_reg),
 19658					BPF_JMP_IMM(BPF_JA, 0, 0, 1),
 19659					*insn,
 19660				};
 19661				struct bpf_insn chk_and_mod[] = {
 19662					/* [R,W]x mod 0 -> [R,W]x */
 19663					BPF_RAW_INSN((is64 ? BPF_JMP : BPF_JMP32) |
 19664						     BPF_JEQ | BPF_K, insn->src_reg,
 19665						     0, 1 + (is64 ? 0 : 1), 0),
 19666					*insn,
 19667					BPF_JMP_IMM(BPF_JA, 0, 0, 1),
 19668					BPF_MOV32_REG(insn->dst_reg, insn->dst_reg),
 19669				};
 19670	
 19671				patchlet = isdiv ? chk_and_div : chk_and_mod;
 19672				cnt = isdiv ? ARRAY_SIZE(chk_and_div) :
 19673					      ARRAY_SIZE(chk_and_mod) - (is64 ? 2 : 0);
 19674	
 19675				new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
 19676				if (!new_prog)
 19677					return -ENOMEM;
 19678	
 19679				delta    += cnt - 1;
 19680				env->prog = prog = new_prog;
 19681				insn      = new_prog->insnsi + i + delta;
 19682				goto next_insn;
 19683			}
 19684	
 19685			/* Implement LD_ABS and LD_IND with a rewrite, if supported by the program type. */
 19686			if (BPF_CLASS(insn->code) == BPF_LD &&
 19687			    (BPF_MODE(insn->code) == BPF_ABS ||
 19688			     BPF_MODE(insn->code) == BPF_IND)) {
 19689				cnt = env->ops->gen_ld_abs(insn, insn_buf);
 19690				if (cnt == 0 || cnt >= ARRAY_SIZE(insn_buf)) {
 19691					verbose(env, "bpf verifier is misconfigured\n");
 19692					return -EINVAL;
 19693				}
 19694	
 19695				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 19696				if (!new_prog)
 19697					return -ENOMEM;
 19698	
 19699				delta    += cnt - 1;
 19700				env->prog = prog = new_prog;
 19701				insn      = new_prog->insnsi + i + delta;
 19702				goto next_insn;
 19703			}
 19704	
 19705			/* Rewrite pointer arithmetic to mitigate speculation attacks. */
 19706			if (insn->code == (BPF_ALU64 | BPF_ADD | BPF_X) ||
 19707			    insn->code == (BPF_ALU64 | BPF_SUB | BPF_X)) {
 19708				const u8 code_add = BPF_ALU64 | BPF_ADD | BPF_X;
 19709				const u8 code_sub = BPF_ALU64 | BPF_SUB | BPF_X;
 19710				struct bpf_insn *patch = &insn_buf[0];
 19711				bool issrc, isneg, isimm;
 19712				u32 off_reg;
 19713	
 19714				aux = &env->insn_aux_data[i + delta];
 19715				if (!aux->alu_state ||
 19716				    aux->alu_state == BPF_ALU_NON_POINTER)
 19717					goto next_insn;
 19718	
 19719				isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
 19720				issrc = (aux->alu_state & BPF_ALU_SANITIZE) ==
 19721					BPF_ALU_SANITIZE_SRC;
 19722				isimm = aux->alu_state & BPF_ALU_IMMEDIATE;
 19723	
 19724				off_reg = issrc ? insn->src_reg : insn->dst_reg;
 19725				if (isimm) {
 19726					*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
 19727				} else {
 19728					if (isneg)
 19729						*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
 19730					*patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
 19731					*patch++ = BPF_ALU64_REG(BPF_SUB, BPF_REG_AX, off_reg);
 19732					*patch++ = BPF_ALU64_REG(BPF_OR, BPF_REG_AX, off_reg);
 19733					*patch++ = BPF_ALU64_IMM(BPF_NEG, BPF_REG_AX, 0);
 19734					*patch++ = BPF_ALU64_IMM(BPF_ARSH, BPF_REG_AX, 63);
 19735					*patch++ = BPF_ALU64_REG(BPF_AND, BPF_REG_AX, off_reg);
 19736				}
 19737				if (!issrc)
 19738					*patch++ = BPF_MOV64_REG(insn->dst_reg, insn->src_reg);
 19739				insn->src_reg = BPF_REG_AX;
 19740				if (isneg)
 19741					insn->code = insn->code == code_add ?
 19742						     code_sub : code_add;
 19743				*patch++ = *insn;
 19744				if (issrc && isneg && !isimm)
 19745					*patch++ = BPF_ALU64_IMM(BPF_MUL, off_reg, -1);
 19746				cnt = patch - insn_buf;
 19747	
 19748				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 19749				if (!new_prog)
 19750					return -ENOMEM;
 19751	
 19752				delta    += cnt - 1;
 19753				env->prog = prog = new_prog;
 19754				insn      = new_prog->insnsi + i + delta;
 19755				goto next_insn;
 19756			}
 19757	
 19758			if (is_may_goto_insn(insn)) {
 19759				int stack_off = -stack_depth - 8;
 19760	
 19761				stack_depth_extra = 8;
 19762				insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_AX, BPF_REG_10, stack_off);
 19763				insn_buf[1] = BPF_JMP_IMM(BPF_JEQ, BPF_REG_AX, 0, insn->off + 2);
 19764				insn_buf[2] = BPF_ALU64_IMM(BPF_SUB, BPF_REG_AX, 1);
 19765				insn_buf[3] = BPF_STX_MEM(BPF_DW, BPF_REG_10, BPF_REG_AX, stack_off);
 19766				cnt = 4;
 19767	
 19768				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 19769				if (!new_prog)
 19770					return -ENOMEM;
 19771	
 19772				delta += cnt - 1;
 19773				env->prog = prog = new_prog;
 19774				insn = new_prog->insnsi + i + delta;
 19775				goto next_insn;
 19776			}
 19777	
 19778			if (insn->code != (BPF_JMP | BPF_CALL))
 19779				goto next_insn;
 19780			if (insn->src_reg == BPF_PSEUDO_CALL)
 19781				goto next_insn;
 19782			if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) {
 19783				ret = fixup_kfunc_call(env, insn, insn_buf, i + delta, &cnt);
 19784				if (ret)
 19785					return ret;
 19786				if (cnt == 0)
 19787					goto next_insn;
 19788	
 19789				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 19790				if (!new_prog)
 19791					return -ENOMEM;
 19792	
 19793				delta	 += cnt - 1;
 19794				env->prog = prog = new_prog;
 19795				insn	  = new_prog->insnsi + i + delta;
 19796				goto next_insn;
 19797			}
 19798	
 19799			if (insn->imm == BPF_FUNC_get_route_realm)
 19800				prog->dst_needed = 1;
 19801			if (insn->imm == BPF_FUNC_get_prandom_u32)
 19802				bpf_user_rnd_init_once();
 19803			if (insn->imm == BPF_FUNC_override_return)
 19804				prog->kprobe_override = 1;
 19805			if (insn->imm == BPF_FUNC_tail_call) {
 19806				/* If we tail call into other programs, we
 19807				 * cannot make any assumptions since they can
 19808				 * be replaced dynamically during runtime in
 19809				 * the program array.
 19810				 */
 19811				prog->cb_access = 1;
 19812				if (!allow_tail_call_in_subprogs(env))
 19813					prog->aux->stack_depth = MAX_BPF_STACK;
 19814				prog->aux->max_pkt_offset = MAX_PACKET_OFF;
 19815	
 19816				/* mark bpf_tail_call as different opcode to avoid
 19817				 * conditional branch in the interpreter for every normal
 19818				 * call and to prevent accidental JITing by JIT compiler
 19819				 * that doesn't support bpf_tail_call yet
 19820				 */
 19821				insn->imm = 0;
 19822				insn->code = BPF_JMP | BPF_TAIL_CALL;
 19823	
 19824				aux = &env->insn_aux_data[i + delta];
 19825				if (env->bpf_capable && !prog->blinding_requested &&
 19826				    prog->jit_requested &&
 19827				    !bpf_map_key_poisoned(aux) &&
 19828				    !bpf_map_ptr_poisoned(aux) &&
 19829				    !bpf_map_ptr_unpriv(aux)) {
 19830					struct bpf_jit_poke_descriptor desc = {
 19831						.reason = BPF_POKE_REASON_TAIL_CALL,
 19832						.tail_call.map = BPF_MAP_PTR(aux->map_ptr_state),
 19833						.tail_call.key = bpf_map_key_immediate(aux),
 19834						.insn_idx = i + delta,
 19835					};
 19836	
 19837					ret = bpf_jit_add_poke_descriptor(prog, &desc);
 19838					if (ret < 0) {
 19839						verbose(env, "adding tail call poke descriptor failed\n");
 19840						return ret;
 19841					}
 19842	
 19843					insn->imm = ret + 1;
 19844					goto next_insn;
 19845				}
 19846	
 19847				if (!bpf_map_ptr_unpriv(aux))
 19848					goto next_insn;
 19849	
 19850				/* instead of changing every JIT dealing with tail_call
 19851				 * emit two extra insns:
 19852				 * if (index >= max_entries) goto out;
 19853				 * index &= array->index_mask;
 19854				 * to avoid out-of-bounds cpu speculation
 19855				 */
 19856				if (bpf_map_ptr_poisoned(aux)) {
 19857					verbose(env, "tail_call abusing map_ptr\n");
 19858					return -EINVAL;
 19859				}
 19860	
 19861				map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
 19862				insn_buf[0] = BPF_JMP_IMM(BPF_JGE, BPF_REG_3,
 19863							  map_ptr->max_entries, 2);
 19864				insn_buf[1] = BPF_ALU32_IMM(BPF_AND, BPF_REG_3,
 19865							    container_of(map_ptr,
 19866									 struct bpf_array,
 19867									 map)->index_mask);
 19868				insn_buf[2] = *insn;
 19869				cnt = 3;
 19870				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 19871				if (!new_prog)
 19872					return -ENOMEM;
 19873	
 19874				delta    += cnt - 1;
 19875				env->prog = prog = new_prog;
 19876				insn      = new_prog->insnsi + i + delta;
 19877				goto next_insn;
 19878			}
 19879	
 19880			if (insn->imm == BPF_FUNC_timer_set_callback) {
 19881				/* The verifier will process callback_fn as many times as necessary
 19882				 * with different maps and the register states prepared by
 19883				 * set_timer_callback_state will be accurate.
 19884				 *
 19885				 * The following use case is valid:
 19886				 *   map1 is shared by prog1, prog2, prog3.
 19887				 *   prog1 calls bpf_timer_init for some map1 elements
 19888				 *   prog2 calls bpf_timer_set_callback for some map1 elements.
 19889				 *     Those that were not bpf_timer_init-ed will return -EINVAL.
 19890				 *   prog3 calls bpf_timer_start for some map1 elements.
 19891				 *     Those that were not both bpf_timer_init-ed and
 19892				 *     bpf_timer_set_callback-ed will return -EINVAL.
 19893				 */
 19894				struct bpf_insn ld_addrs[2] = {
 19895					BPF_LD_IMM64(BPF_REG_3, (long)prog->aux),
 19896				};
 19897	
 19898				insn_buf[0] = ld_addrs[0];
 19899				insn_buf[1] = ld_addrs[1];
 19900				insn_buf[2] = *insn;
 19901				cnt = 3;
 19902	
 19903				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 19904				if (!new_prog)
 19905					return -ENOMEM;
 19906	
 19907				delta    += cnt - 1;
 19908				env->prog = prog = new_prog;
 19909				insn      = new_prog->insnsi + i + delta;
 19910				goto patch_call_imm;
 19911			}
 19912	
 19913			if (is_storage_get_function(insn->imm)) {
 19914				if (!in_sleepable(env) ||
 19915				    env->insn_aux_data[i + delta].storage_get_func_atomic)
 19916					insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_ATOMIC);
 19917				else
 19918					insn_buf[0] = BPF_MOV64_IMM(BPF_REG_5, (__force __s32)GFP_KERNEL);
 19919				insn_buf[1] = *insn;
 19920				cnt = 2;
 19921	
 19922				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 19923				if (!new_prog)
 19924					return -ENOMEM;
 19925	
 19926				delta += cnt - 1;
 19927				env->prog = prog = new_prog;
 19928				insn = new_prog->insnsi + i + delta;
 19929				goto patch_call_imm;
 19930			}
 19931	
 19932			/* bpf_per_cpu_ptr() and bpf_this_cpu_ptr() */
 19933			if (env->insn_aux_data[i + delta].call_with_percpu_alloc_ptr) {
 19934				/* patch with 'r1 = *(u64 *)(r1 + 0)' since for percpu data,
 19935				 * bpf_mem_alloc() returns a ptr to the percpu data ptr.
 19936				 */
 19937				insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_1, BPF_REG_1, 0);
 19938				insn_buf[1] = *insn;
 19939				cnt = 2;
 19940	
 19941				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 19942				if (!new_prog)
 19943					return -ENOMEM;
 19944	
 19945				delta += cnt - 1;
 19946				env->prog = prog = new_prog;
 19947				insn = new_prog->insnsi + i + delta;
 19948				goto patch_call_imm;
 19949			}
 19950	
 19951			/* BPF_EMIT_CALL() assumptions in some of the map_gen_lookup
 19952			 * and other inlining handlers are currently limited to 64 bit
 19953			 * only.
 19954			 */
 19955			if (prog->jit_requested && BITS_PER_LONG == 64 &&
 19956			    (insn->imm == BPF_FUNC_map_lookup_elem ||
 19957			     insn->imm == BPF_FUNC_map_update_elem ||
 19958			     insn->imm == BPF_FUNC_map_delete_elem ||
 19959			     insn->imm == BPF_FUNC_map_push_elem   ||
 19960			     insn->imm == BPF_FUNC_map_pop_elem    ||
 19961			     insn->imm == BPF_FUNC_map_peek_elem   ||
 19962			     insn->imm == BPF_FUNC_redirect_map    ||
 19963			     insn->imm == BPF_FUNC_for_each_map_elem ||
 19964			     insn->imm == BPF_FUNC_map_lookup_percpu_elem)) {
 19965				aux = &env->insn_aux_data[i + delta];
 19966				if (bpf_map_ptr_poisoned(aux))
 19967					goto patch_call_imm;
 19968	
 19969				map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
 19970				ops = map_ptr->ops;
 19971				if (insn->imm == BPF_FUNC_map_lookup_elem &&
 19972				    ops->map_gen_lookup) {
 19973					cnt = ops->map_gen_lookup(map_ptr, insn_buf);
 19974					if (cnt == -EOPNOTSUPP)
 19975						goto patch_map_ops_generic;
 19976					if (cnt <= 0 || cnt >= ARRAY_SIZE(insn_buf)) {
 19977						verbose(env, "bpf verifier is misconfigured\n");
 19978						return -EINVAL;
 19979					}
 19980	
 19981					new_prog = bpf_patch_insn_data(env, i + delta,
 19982								       insn_buf, cnt);
 19983					if (!new_prog)
 19984						return -ENOMEM;
 19985	
 19986					delta    += cnt - 1;
 19987					env->prog = prog = new_prog;
 19988					insn      = new_prog->insnsi + i + delta;
 19989					goto next_insn;
 19990				}
 19991	
 19992				BUILD_BUG_ON(!__same_type(ops->map_lookup_elem,
 19993					     (void *(*)(struct bpf_map *map, void *key))NULL));
 19994				BUILD_BUG_ON(!__same_type(ops->map_delete_elem,
 19995					     (long (*)(struct bpf_map *map, void *key))NULL));
 19996				BUILD_BUG_ON(!__same_type(ops->map_update_elem,
 19997					     (long (*)(struct bpf_map *map, void *key, void *value,
 19998						      u64 flags))NULL));
 19999				BUILD_BUG_ON(!__same_type(ops->map_push_elem,
 20000					     (long (*)(struct bpf_map *map, void *value,
 20001						      u64 flags))NULL));
 20002				BUILD_BUG_ON(!__same_type(ops->map_pop_elem,
 20003					     (long (*)(struct bpf_map *map, void *value))NULL));
 20004				BUILD_BUG_ON(!__same_type(ops->map_peek_elem,
 20005					     (long (*)(struct bpf_map *map, void *value))NULL));
 20006				BUILD_BUG_ON(!__same_type(ops->map_redirect,
 20007					     (long (*)(struct bpf_map *map, u64 index, u64 flags))NULL));
 20008				BUILD_BUG_ON(!__same_type(ops->map_for_each_callback,
 20009					     (long (*)(struct bpf_map *map,
 20010						      bpf_callback_t callback_fn,
 20011						      void *callback_ctx,
 20012						      u64 flags))NULL));
 20013				BUILD_BUG_ON(!__same_type(ops->map_lookup_percpu_elem,
 20014					     (void *(*)(struct bpf_map *map, void *key, u32 cpu))NULL));
 20015	
 20016	patch_map_ops_generic:
 20017				switch (insn->imm) {
 20018				case BPF_FUNC_map_lookup_elem:
 20019					insn->imm = BPF_CALL_IMM(ops->map_lookup_elem);
 20020					goto next_insn;
 20021				case BPF_FUNC_map_update_elem:
 20022					insn->imm = BPF_CALL_IMM(ops->map_update_elem);
 20023					goto next_insn;
 20024				case BPF_FUNC_map_delete_elem:
 20025					insn->imm = BPF_CALL_IMM(ops->map_delete_elem);
 20026					goto next_insn;
 20027				case BPF_FUNC_map_push_elem:
 20028					insn->imm = BPF_CALL_IMM(ops->map_push_elem);
 20029					goto next_insn;
 20030				case BPF_FUNC_map_pop_elem:
 20031					insn->imm = BPF_CALL_IMM(ops->map_pop_elem);
 20032					goto next_insn;
 20033				case BPF_FUNC_map_peek_elem:
 20034					insn->imm = BPF_CALL_IMM(ops->map_peek_elem);
 20035					goto next_insn;
 20036				case BPF_FUNC_redirect_map:
 20037					insn->imm = BPF_CALL_IMM(ops->map_redirect);
 20038					goto next_insn;
 20039				case BPF_FUNC_for_each_map_elem:
 20040					insn->imm = BPF_CALL_IMM(ops->map_for_each_callback);
 20041					goto next_insn;
 20042				case BPF_FUNC_map_lookup_percpu_elem:
 20043					insn->imm = BPF_CALL_IMM(ops->map_lookup_percpu_elem);
 20044					goto next_insn;
 20045				}
 20046	
 20047				goto patch_call_imm;
 20048			}
 20049	
 20050			/* Implement bpf_jiffies64 inline. */
 20051			if (prog->jit_requested && BITS_PER_LONG == 64 &&
 20052			    insn->imm == BPF_FUNC_jiffies64) {
 20053				struct bpf_insn ld_jiffies_addr[2] = {
 20054					BPF_LD_IMM64(BPF_REG_0,
 20055						     (unsigned long)&jiffies),
 20056				};
 20057	
 20058				insn_buf[0] = ld_jiffies_addr[0];
 20059				insn_buf[1] = ld_jiffies_addr[1];
 20060				insn_buf[2] = BPF_LDX_MEM(BPF_DW, BPF_REG_0,
 20061							  BPF_REG_0, 0);
 20062				cnt = 3;
 20063	
 20064				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf,
 20065							       cnt);
 20066				if (!new_prog)
 20067					return -ENOMEM;
 20068	
 20069				delta    += cnt - 1;
 20070				env->prog = prog = new_prog;
 20071				insn      = new_prog->insnsi + i + delta;
 20072				goto next_insn;
 20073			}
 20074	
 20075			/* Implement bpf_get_smp_processor_id() inline. */
 20076			if (insn->imm == BPF_FUNC_get_smp_processor_id &&
 20077			    prog->jit_requested && bpf_jit_supports_percpu_insns()) {
 20078				insn_buf[0] = BPF_MOV32_IMM(BPF_REG_0, (u32)(long)&pcpu_hot.cpu_number);
 20079				insn_buf[1] = BPF_LDX_MEM_PERCPU(BPF_W, BPF_REG_0, BPF_REG_0, 0);
 20080				cnt = 2;
 20081	
 20082				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 20083				if (!new_prog)
 20084					return -ENOMEM;
 20085	
 20086				delta    += cnt - 1;
 20087				env->prog = prog = new_prog;
 20088				insn      = new_prog->insnsi + i + delta;
 20089				goto next_insn;
 20090			}
 20091	
 20092			/* Implement bpf_get_func_arg inline. */
 20093			if (prog_type == BPF_PROG_TYPE_TRACING &&
 20094			    insn->imm == BPF_FUNC_get_func_arg) {
 20095				/* Load nr_args from ctx - 8 */
 20096				insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
 20097				insn_buf[1] = BPF_JMP32_REG(BPF_JGE, BPF_REG_2, BPF_REG_0, 6);
 20098				insn_buf[2] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_2, 3);
 20099				insn_buf[3] = BPF_ALU64_REG(BPF_ADD, BPF_REG_2, BPF_REG_1);
 20100				insn_buf[4] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_2, 0);
 20101				insn_buf[5] = BPF_STX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
 20102				insn_buf[6] = BPF_MOV64_IMM(BPF_REG_0, 0);
 20103				insn_buf[7] = BPF_JMP_A(1);
 20104				insn_buf[8] = BPF_MOV64_IMM(BPF_REG_0, -EINVAL);
 20105				cnt = 9;
 20106	
 20107				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 20108				if (!new_prog)
 20109					return -ENOMEM;
 20110	
 20111				delta    += cnt - 1;
 20112				env->prog = prog = new_prog;
 20113				insn      = new_prog->insnsi + i + delta;
 20114				goto next_insn;
 20115			}
 20116	
 20117			/* Implement bpf_get_func_ret inline. */
 20118			if (prog_type == BPF_PROG_TYPE_TRACING &&
 20119			    insn->imm == BPF_FUNC_get_func_ret) {
 20120				if (eatype == BPF_TRACE_FEXIT ||
 20121				    eatype == BPF_MODIFY_RETURN) {
 20122					/* Load nr_args from ctx - 8 */
 20123					insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
 20124					insn_buf[1] = BPF_ALU64_IMM(BPF_LSH, BPF_REG_0, 3);
 20125					insn_buf[2] = BPF_ALU64_REG(BPF_ADD, BPF_REG_0, BPF_REG_1);
 20126					insn_buf[3] = BPF_LDX_MEM(BPF_DW, BPF_REG_3, BPF_REG_0, 0);
 20127					insn_buf[4] = BPF_STX_MEM(BPF_DW, BPF_REG_2, BPF_REG_3, 0);
 20128					insn_buf[5] = BPF_MOV64_IMM(BPF_REG_0, 0);
 20129					cnt = 6;
 20130				} else {
 20131					insn_buf[0] = BPF_MOV64_IMM(BPF_REG_0, -EOPNOTSUPP);
 20132					cnt = 1;
 20133				}
 20134	
 20135				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 20136				if (!new_prog)
 20137					return -ENOMEM;
 20138	
 20139				delta    += cnt - 1;
 20140				env->prog = prog = new_prog;
 20141				insn      = new_prog->insnsi + i + delta;
 20142				goto next_insn;
 20143			}
 20144	
 20145			/* Implement get_func_arg_cnt inline. */
 20146			if (prog_type == BPF_PROG_TYPE_TRACING &&
 20147			    insn->imm == BPF_FUNC_get_func_arg_cnt) {
 20148				/* Load nr_args from ctx - 8 */
 20149				insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -8);
 20150	
 20151				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
 20152				if (!new_prog)
 20153					return -ENOMEM;
 20154	
 20155				env->prog = prog = new_prog;
 20156				insn      = new_prog->insnsi + i + delta;
 20157				goto next_insn;
 20158			}
 20159	
 20160			/* Implement bpf_get_func_ip inline. */
 20161			if (prog_type == BPF_PROG_TYPE_TRACING &&
 20162			    insn->imm == BPF_FUNC_get_func_ip) {
 20163				/* Load IP address from ctx - 16 */
 20164				insn_buf[0] = BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, -16);
 20165	
 20166				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, 1);
 20167				if (!new_prog)
 20168					return -ENOMEM;
 20169	
 20170				env->prog = prog = new_prog;
 20171				insn      = new_prog->insnsi + i + delta;
 20172				goto next_insn;
 20173			}
 20174	
 20175			/* Implement bpf_kptr_xchg inline */
 20176			if (prog->jit_requested && BITS_PER_LONG == 64 &&
 20177			    insn->imm == BPF_FUNC_kptr_xchg &&
 20178			    bpf_jit_supports_ptr_xchg()) {
 20179				insn_buf[0] = BPF_MOV64_REG(BPF_REG_0, BPF_REG_2);
 20180				insn_buf[1] = BPF_ATOMIC_OP(BPF_DW, BPF_XCHG, BPF_REG_1, BPF_REG_0, 0);
 20181				cnt = 2;
 20182	
 20183				new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
 20184				if (!new_prog)
 20185					return -ENOMEM;
 20186	
 20187				delta    += cnt - 1;
 20188				env->prog = prog = new_prog;
 20189				insn      = new_prog->insnsi + i + delta;
 20190				goto next_insn;
 20191			}
 20192	patch_call_imm:
 20193			fn = env->ops->get_func_proto(insn->imm, env->prog);
 20194			/* all functions that have prototype and verifier allowed
 20195			 * programs to call them, must be real in-kernel functions
 20196			 */
 20197			if (!fn->func) {
 20198				verbose(env,
 20199					"kernel subsystem misconfigured func %s#%d\n",
 20200					func_id_name(insn->imm), insn->imm);
 20201				return -EFAULT;
 20202			}
 20203			insn->imm = fn->func - __bpf_call_base;
 20204	next_insn:
 20205			if (subprogs[cur_subprog + 1].start == i + delta + 1) {
 20206				subprogs[cur_subprog].stack_depth += stack_depth_extra;
 20207				subprogs[cur_subprog].stack_extra = stack_depth_extra;
 20208				cur_subprog++;
 20209				stack_depth = subprogs[cur_subprog].stack_depth;
 20210				stack_depth_extra = 0;
 20211			}
 20212			i++;
 20213			insn++;
 20214		}
 20215	
 20216		env->prog->aux->stack_depth = subprogs[0].stack_depth;
 20217		for (i = 0; i < env->subprog_cnt; i++) {
 20218			int subprog_start = subprogs[i].start;
 20219			int stack_slots = subprogs[i].stack_extra / 8;
 20220	
 20221			if (!stack_slots)
 20222				continue;
 20223			if (stack_slots > 1) {
 20224				verbose(env, "verifier bug: stack_slots supports may_goto only\n");
 20225				return -EFAULT;
 20226			}
 20227	
 20228			/* Add ST insn to subprog prologue to init extra stack */
 20229			insn_buf[0] = BPF_ST_MEM(BPF_DW, BPF_REG_FP,
 20230						 -subprogs[i].stack_depth, BPF_MAX_LOOPS);
 20231			/* Copy first actual insn to preserve it */
 20232			insn_buf[1] = env->prog->insnsi[subprog_start];
 20233	
 20234			new_prog = bpf_patch_insn_data(env, subprog_start, insn_buf, 2);
 20235			if (!new_prog)
 20236				return -ENOMEM;
 20237			env->prog = prog = new_prog;
 20238		}
 20239	
 20240		/* Since poke tab is now finalized, publish aux to tracker. */
 20241		for (i = 0; i < prog->aux->size_poke_tab; i++) {
 20242			map_ptr = prog->aux->poke_tab[i].tail_call.map;
 20243			if (!map_ptr->ops->map_poke_track ||
 20244			    !map_ptr->ops->map_poke_untrack ||
 20245			    !map_ptr->ops->map_poke_run) {
 20246				verbose(env, "bpf verifier is misconfigured\n");
 20247				return -EINVAL;
 20248			}
 20249	
 20250			ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
 20251			if (ret < 0) {
 20252				verbose(env, "tracking tail call prog failed\n");
 20253				return ret;
 20254			}
 20255		}
 20256	
 20257		sort_kfunc_descs_by_imm_off(env->prog);
 20258	
 20259		return 0;
 20260	}
 20261	

-- 
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki




[Index of Archives]     [Linux Samsung SoC]     [Linux Rockchip SoC]     [Linux Actions SoC]     [Linux for Synopsys ARC Processors]     [Linux NFS]     [Linux NILFS]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]


  Powered by Linux