Lines Matching defs:aux
200 static bool bpf_map_ptr_poisoned(const struct bpf_insn_aux_data *aux)
202 return BPF_MAP_PTR(aux->map_ptr_state) == BPF_MAP_PTR_POISON;
205 static bool bpf_map_ptr_unpriv(const struct bpf_insn_aux_data *aux)
207 return aux->map_ptr_state & BPF_MAP_PTR_UNPRIV;
210 static void bpf_map_ptr_store(struct bpf_insn_aux_data *aux, const struct bpf_map *map, bool unpriv)
213 unpriv |= bpf_map_ptr_unpriv(aux);
214 aux->map_ptr_state = (unsigned long)map | (unpriv ? BPF_MAP_PTR_UNPRIV : 0UL);
217 static bool bpf_map_key_poisoned(const struct bpf_insn_aux_data *aux)
219 return aux->map_key_state & BPF_MAP_KEY_POISON;
222 static bool bpf_map_key_unseen(const struct bpf_insn_aux_data *aux)
224 return !(aux->map_key_state & BPF_MAP_KEY_SEEN);
227 static u64 bpf_map_key_immediate(const struct bpf_insn_aux_data *aux)
229 return aux->map_key_state & ~(BPF_MAP_KEY_SEEN | BPF_MAP_KEY_POISON);
232 static void bpf_map_key_store(struct bpf_insn_aux_data *aux, u64 state)
234 bool poisoned = bpf_map_key_poisoned(aux);
236 aux->map_key_state = state | BPF_MAP_KEY_SEEN | (poisoned ? BPF_MAP_KEY_POISON : 0ULL);
264 nr_linfo = prog->aux->nr_linfo;
270 linfo = prog->aux->linfo;
391 verbose(env, "%s\n", ltrim(btf_name_by_offset(env->prog->aux->btf, linfo->line_off)));
2895 return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type;
2972 env->prog->aux->max_pkt_offset = max_t(u32, env->prog->aux->max_pkt_offset, off + reg->umax_value + size - 1);
3002 if (env->prog->aux->max_ctx_offset < off + size) {
3003 env->prog->aux->max_ctx_offset = off + size;
3317 env->prog->aux->tail_call_reachable = true;
3406 if (off + size > env->prog->aux->max_tp_access) {
3407 env->prog->aux->max_tp_access = off + size;
3873 max_access = &env->prog->aux->max_rdonly_access;
3876 max_access = &env->prog->aux->max_rdwr_access;
4099 max_access = &env->prog->aux->max_rdonly_access;
4102 max_access = &env->prog->aux->max_rdwr_access;
5088 func_info_aux = env->prog->aux->func_info_aux;
5224 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
5249 if (!BPF_MAP_PTR(aux->map_ptr_state)) {
5250 bpf_map_ptr_store(aux, meta->map_ptr, !meta->map_ptr->bypass_spec_v1);
5251 } else if (BPF_MAP_PTR(aux->map_ptr_state) != meta->map_ptr) {
5252 bpf_map_ptr_store(aux, BPF_MAP_PTR_POISON, !meta->map_ptr->bypass_spec_v1);
5259 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
5278 bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
5287 if (bpf_map_key_unseen(aux)) {
5288 bpf_map_key_store(aux, val);
5289 } else if (!bpf_map_key_poisoned(aux) && bpf_map_key_immediate(aux) != val) {
5290 bpf_map_key_store(aux, BPF_MAP_KEY_POISON);
5675 static int update_alu_sanitation_state(struct bpf_insn_aux_data *aux, u32 alu_state, u32 alu_limit)
5680 if (aux->alu_state && (aux->alu_state != alu_state || aux->alu_limit != alu_limit)) {
5685 aux->alu_state = alu_state;
5686 aux->alu_limit = alu_limit;
5692 struct bpf_insn_aux_data *aux = cur_aux(env);
5698 return update_alu_sanitation_state(aux, BPF_ALU_NON_POINTER, 0);
5707 struct bpf_insn_aux_data aux;
5734 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
5749 /* We already marked aux for masking from non-speculative
5774 alu_state = info->aux.alu_state;
5775 alu_limit = abs(info->aux.alu_limit - alu_limit);
5789 err = update_alu_sanitation_state(aux, alu_state, alu_limit);
7984 struct bpf_insn_aux_data *aux = cur_aux(env);
8020 dst_reg->type = aux->btf_var.reg_type;
8023 dst_reg->mem_size = aux->btf_var.mem_size;
8027 dst_reg->btf_id = aux->btf_var.btf_id;
8036 map = env->used_maps[aux->map_index];
8041 dst_reg->off = aux->map_off;
8172 !prog->aux->attach_func_proto->type) {
8226 if (!env->prog->aux->attach_btf_id) {
8586 btf = prog->aux->btf;
8666 prog->aux->func_info = krecord;
8667 prog->aux->func_info_cnt = nfuncs;
8668 prog->aux->func_info_aux = info_aux;
8679 struct bpf_prog_aux *aux = env->prog->aux;
8682 if (!aux->func_info) {
8687 aux->func_info[i].insn_off = env->subprog_info[i].start;
8727 btf = prog->aux->btf;
8802 prog->aux->linfo = linfo;
8803 prog->aux->nr_linfo = nr_linfo;
8828 env->prog->aux->btf = btf;
9728 if (bpf_prog_is_dev_bound(env->prog->aux)) {
9959 static int check_pseudo_btf_id(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *aux)
10017 aux->btf_var.reg_type = PTR_TO_PERCPU_BTF_ID;
10018 aux->btf_var.btf_id = type;
10031 aux->btf_var.reg_type = PTR_TO_MEM | MEM_RDONLY;
10032 aux->btf_var.mem_size = tsize;
10034 aux->btf_var.reg_type = PTR_TO_BTF_ID;
10035 aux->btf_var.btf_id = type;
10112 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && !bpf_offload_prog_map_match(prog, map)) {
10122 if (prog->aux->sleepable) {
10177 struct bpf_insn_aux_data *aux;
10194 aux = &env->insn_aux_data[i];
10195 err = check_pseudo_btf_id(env, insn, aux);
10224 aux = &env->insn_aux_data[i];
10249 aux->map_off = off;
10259 aux->map_index = j;
10277 aux->map_index = env->used_map_cnt;
10280 if (bpf_map_is_cgroup_storage(map) && bpf_cgroup_storage_assign(env->prog->aux, map)) {
10310 __bpf_free_used_maps(env->prog->aux, env->used_maps, env->used_map_cnt);
10340 /* aux info at OFF always needs adjustment, no matter fast path
10380 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
10381 int i, sz = prog->aux->size_poke_tab;
10444 struct bpf_prog_aux *aux = env->prog->aux;
10454 if (aux->func_info) {
10455 move = aux->func_info_cnt - j;
10457 memmove(aux->func_info + i, aux->func_info + j, sizeof(*aux->func_info) * move);
10458 aux->func_info_cnt -= j - i;
10484 nr_linfo = prog->aux->nr_linfo;
10489 linfo = prog->aux->linfo;
10521 prog->aux->nr_linfo -= l_cnt;
10522 nr_linfo = prog->aux->nr_linfo;
10553 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10642 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10703 struct bpf_insn_aux_data *aux = env->insn_aux_data;
10719 if (!aux[adj_idx].zext_dst) {
10745 if (class == BPF_LDX && aux[adj_idx].ptr_type == PTR_TO_CTX) {
10774 aux = env->insn_aux_data;
10817 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10883 env->prog->aux->num_exentries++;
11021 * func[i]->aux->stats will never be accessed and stays NULL
11034 func[i]->aux->func_idx = i;
11035 /* Below members will be freed only at prog->aux */
11036 func[i]->aux->btf = prog->aux->btf;
11037 func[i]->aux->func_info = prog->aux->func_info;
11038 func[i]->aux->poke_tab = prog->aux->poke_tab;
11039 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;
11041 for (j = 0; j < prog->aux->size_poke_tab; j++) {
11044 poke = &prog->aux->poke_tab[j];
11046 poke->aux = func[i]->aux;
11053 func[i]->aux->name[0] = 'F';
11054 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
11056 func[i]->aux->linfo = prog->aux->linfo;
11057 func[i]->aux->nr_linfo = prog->aux->nr_linfo;
11058 func[i]->aux->jited_linfo = prog->aux->jited_linfo;
11059 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
11067 func[i]->aux->num_exentries = num_exentries;
11068 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
11091 /* we use the aux data to keep a list of the start addresses
11102 func[i]->aux->func = func;
11103 func[i]->aux->func_cnt = env->subprog_cnt;
11139 prog->aux->func = func;
11140 prog->aux->func_cnt = env->subprog_cnt;
11148 for (i = 0; i < prog->aux->size_poke_tab; i++) {
11149 map_ptr = prog->aux->poke_tab[i].tail_call.map;
11150 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
11160 func[i]->aux->poke_tab = NULL;
11187 if (env->prog->jit_requested && !bpf_prog_is_dev_bound(env->prog->aux)) {
11197 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
11232 struct bpf_insn_aux_data *aux;
11299 aux = &env->insn_aux_data[i + delta];
11300 if (!aux->alu_state || aux->alu_state == BPF_ALU_NON_POINTER) {
11304 isneg = aux->alu_state & BPF_ALU_NEG_VALUE;
11305 issrc = (aux->alu_state & BPF_ALU_SANITIZE) == BPF_ALU_SANITIZE_SRC;
11306 isimm = aux->alu_state & BPF_ALU_IMMEDIATE;
11310 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
11315 *patch++ = BPF_MOV32_IMM(BPF_REG_AX, aux->alu_limit);
11370 prog->aux->stack_depth = MAX_BPF_STACK;
11372 prog->aux->max_pkt_offset = MAX_PACKET_OFF;
11382 aux = &env->insn_aux_data[i + delta];
11383 if (env->bpf_capable && !expect_blinding && prog->jit_requested && !bpf_map_key_poisoned(aux) &&
11384 !bpf_map_ptr_poisoned(aux) && !bpf_map_ptr_unpriv(aux)) {
11387 .tail_call.map = BPF_MAP_PTR(aux->map_ptr_state),
11388 .tail_call.key = bpf_map_key_immediate(aux),
11402 if (!bpf_map_ptr_unpriv(aux)) {
11412 if (bpf_map_ptr_poisoned(aux)) {
11417 map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
11441 aux = &env->insn_aux_data[i + delta];
11442 if (bpf_map_ptr_poisoned(aux)) {
11446 map_ptr = BPF_MAP_PTR(aux->map_ptr_state);
11537 /* Since poke tab is now finalized, publish aux to tracker. */
11538 for (i = 0; i < prog->aux->size_poke_tab; i++) {
11539 map_ptr = prog->aux->poke_tab[i].tail_call.map;
11545 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
11681 struct bpf_prog_aux *aux = env->prog->aux;
11684 if (!aux->func_info) {
11689 if (aux->func_info_aux[i].linkage != BTF_FUNC_GLOBAL) {
11711 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
11754 btf_id = prog->aux->attach_btf_id;
11784 prog->aux->attach_func_proto = func_proto;
11785 prog->aux->attach_func_name = mname;
11847 btf = tgt_prog ? tgt_prog->aux->btf : btf_vmlinux;
11863 struct bpf_prog_aux *aux = tgt_prog->aux;
11865 for (i = 0; i < aux->func_info_cnt; i++) {
11866 if (aux->func_info[i].type_id == btf_id) {
11875 conservative = aux->func_info_aux[subprog].unreliable;
11987 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
11988 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
11989 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) {
12006 addr = (long)tgt_prog->aux->func[subprog]->bpf_func;
12016 if (prog->aux->sleepable) {
12065 struct bpf_prog *tgt_prog = prog->aux->dst_prog;
12067 u32 btf_id = prog->aux->attach_btf_id;
12072 if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && prog->type != BPF_PROG_TYPE_LSM) {
12100 prog->aux->attach_func_proto = tgt_info.tgt_type;
12101 prog->aux->attach_func_name = tgt_info.tgt_name;
12104 prog->aux->saved_dst_prog_type = tgt_prog->type;
12105 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
12109 prog->aux->attach_btf_trace = true;
12131 prog->aux->dst_trampoline = tr;
12256 if (bpf_prog_is_dev_bound(env->prog->aux)) {
12271 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) {
12311 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) {
12313 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret : false;
12333 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, sizeof(env->used_maps[0]), GFP_KERNEL);
12335 if (!env->prog->aux->used_maps) {
12340 memcpy(env->prog->aux->used_maps, env->used_maps, sizeof(env->used_maps[0]) * env->used_map_cnt);
12341 env->prog->aux->used_map_cnt = env->used_map_cnt;
12354 if (!env->prog->aux->used_maps) {