Lines Matching defs:env
257 static const struct bpf_line_info *find_linfo(const struct bpf_verifier_env *env, u32 insn_off)
263 prog = env->prog;
320 __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env, const char *fmt, ...)
324 if (!bpf_verifier_log_needed(&env->log)) {
329 bpf_verifier_vlog(&env->log, fmt, args);
336 struct bpf_verifier_env *env = private_data;
339 if (!bpf_verifier_log_needed(&env->log)) {
344 bpf_verifier_vlog(&env->log, fmt, args);
370 __printf(3, 4) static void verbose_linfo(struct bpf_verifier_env *env, u32 insn_off, const char *prefix_fmt, ...)
374 if (!bpf_verifier_log_needed(&env->log)) {
378 linfo = find_linfo(env, insn_off);
379 if (!linfo || linfo == env->prev_linfo) {
387 bpf_verifier_vlog(&env->log, prefix_fmt, args);
391 verbose(env, "%s\n", ltrim(btf_name_by_offset(env->prog->aux->btf, linfo->line_off)));
393 env->prev_linfo = linfo;
481 static const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type)
521 (void)snprintf(env->type_str_buf, TYPE_STR_BUF_LEN, "%s%s%s", prefix, str[base_type(type)], postfix);
522 return env->type_str_buf;
532 static void print_liveness(struct bpf_verifier_env *env, enum bpf_reg_liveness live)
535 verbose(env, "_");
538 verbose(env, "r");
541 verbose(env, "w");
544 verbose(env, "D");
548 static struct bpf_func_state *func(struct bpf_verifier_env *env, const struct bpf_reg_state *reg)
550 struct bpf_verifier_state *cur = env->cur_state;
560 static void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_func_state *state)
567 verbose(env, " frame%d:", state->frameno);
575 verbose(env, " R%d", i);
576 print_liveness(env, reg->live);
577 verbose(env, "=%s", reg_type_str(env, t));
579 verbose(env, "P");
583 verbose(env, "%lld", reg->var_off.value + reg->off);
586 verbose(env, "%s", kernel_type_name(reg->btf_id));
588 verbose(env, "(id=%d", reg->id);
590 verbose(env, ",ref_obj_id=%d", reg->ref_obj_id);
593 verbose(env, ",off=%d", reg->off);
596 verbose(env, ",r=%d", reg->range);
598 verbose(env, ",ks=%d,vs=%d", reg->map_ptr->key_size, reg->map_ptr->value_size);
605 verbose(env, ",imm=%llx", reg->var_off.value);
608 verbose(env, ",smin_value=%lld", (long long)reg->smin_value);
611 verbose(env, ",smax_value=%lld", (long long)reg->smax_value);
614 verbose(env, ",umin_value=%llu", (unsigned long long)reg->umin_value);
617 verbose(env, ",umax_value=%llu", (unsigned long long)reg->umax_value);
623 verbose(env, ",var_off=%s", tn_buf);
626 verbose(env, ",s32_min_value=%d", (int)(reg->s32_min_value));
629 verbose(env, ",s32_max_value=%d", (int)(reg->s32_max_value));
632 verbose(env, ",u32_min_value=%d", (int)(reg->u32_min_value));
635 verbose(env, ",u32_max_value=%d", (int)(reg->u32_max_value));
638 verbose(env, ")");
656 verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
657 print_liveness(env, state->stack[i].spilled_ptr.live);
661 verbose(env, "=%s", reg_type_str(env, t));
663 verbose(env, "P");
666 verbose(env, "%lld", reg->var_off.value + reg->off);
669 verbose(env, "=%s", types_buf);
673 verbose(env, " refs=%d", state->refs[0].id);
676 verbose(env, ",%d", state->refs[i].id);
680 verbose(env, "\n");
754 /* Acquire a pointer id from the env and update the state->refs to include
759 static int acquire_reference_state(struct bpf_verifier_env *env, int insn_idx)
761 struct bpf_func_state *state = cur_func(env);
769 id = ++env->id_gen;
903 static void update_branch_counts(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
919 static int pop_stack(struct bpf_verifier_env *env, int *prev_insn_idx, int *insn_idx, bool pop_log)
921 struct bpf_verifier_state *cur = env->cur_state;
922 struct bpf_verifier_stack_elem *elem, *head = env->head;
925 if (env->head == NULL) {
936 bpf_vlog_reset(&env->log, head->log_pos);
947 env->head = elem;
948 env->stack_size--;
952 static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env, int insn_idx, int prev_insn_idx,
955 struct bpf_verifier_state *cur = env->cur_state;
966 elem->next = env->head;
967 elem->log_pos = env->log.len_used;
968 env->head = elem;
969 env->stack_size++;
975 if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
976 verbose(env, "The sequence of %d jumps is too complex.\n", env->stack_size);
993 free_verifier_state(env->cur_state, true);
994 env->cur_state = NULL;
996 while (!pop_stack(env, NULL, NULL, false)) {
1005 static void verifier_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg);
1055 static void mark_reg_known_zero(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno)
1058 verbose(env, "mark_reg_known_zero(regs, %u)\n", regno);
1061 verifier_mark_reg_not_init(env, regs + regno);
1318 static void __mark_reg_unknown(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1328 reg->precise = env->subprog_cnt > 1 || !env->bpf_capable;
1332 static void mark_reg_unknown(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno)
1335 verbose(env, "mark_reg_unknown(regs, %u)\n", regno);
1338 verifier_mark_reg_not_init(env, regs + regno);
1342 __mark_reg_unknown(env, regs + regno);
1345 static void verifier_mark_reg_not_init(const struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1347 __mark_reg_unknown(env, reg);
1351 static void mark_reg_not_init(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno)
1354 verbose(env, "mark_reg_not_init(regs, %u)\n", regno);
1357 verifier_mark_reg_not_init(env, regs + regno);
1361 verifier_mark_reg_not_init(env, regs + regno);
1364 static void mark_btf_ld_reg(struct bpf_verifier_env *env, struct bpf_reg_state *regs, u32 regno,
1368 mark_reg_unknown(env, regs, regno);
1371 mark_reg_known_zero(env, regs, regno);
1377 static void init_reg_state(struct bpf_verifier_env *env, struct bpf_func_state *state)
1383 mark_reg_not_init(env, regs, i);
1391 mark_reg_known_zero(env, regs, BPF_REG_FP);
1396 static void init_func_state(struct bpf_verifier_env *env, struct bpf_func_state *state, int callsite, int frameno,
1402 init_reg_state(env, state);
1416 static int find_subprog(struct bpf_verifier_env *env, int off)
1420 p = bsearch(&off, env->subprog_info, env->subprog_cnt, sizeof(env->subprog_info[0]), cmp_subprogs);
1424 return p - env->subprog_info;
1427 static int add_subprog(struct bpf_verifier_env *env, int off)
1429 int insn_cnt = env->prog->len;
1433 verbose(env, "call to invalid destination\n");
1436 ret = find_subprog(env, off);
1440 if (env->subprog_cnt >= BPF_MAX_SUBPROGS) {
1441 verbose(env, "too many subprograms\n");
1444 env->subprog_info[env->subprog_cnt++].start = off;
1445 sort(env->subprog_info, env->subprog_cnt, sizeof(env->subprog_info[0]), cmp_subprogs, NULL);
1449 static int check_subprogs(struct bpf_verifier_env *env)
1452 struct bpf_subprog_info *subprog = env->subprog_info;
1453 struct bpf_insn *insn = env->prog->insnsi;
1454 int insn_cnt = env->prog->len;
1457 ret = add_subprog(env, 0);
1470 if (!env->bpf_capable) {
1471 verbose(env, "function calls to other bpf functions are allowed for CAP_BPF and CAP_SYS_ADMIN\n");
1474 ret = add_subprog(env, i + insn[i].imm + 1);
1483 subprog[env->subprog_cnt].start = insn_cnt;
1485 if (env->log.level & BPF_LOG_LEVEL2) {
1486 for (i = 0; i < env->subprog_cnt; i++) {
1487 verbose(env, "func#%d @%d\n", i, subprog[i].start);
1511 verbose(env, "jump out of range from insn %d to %d\n", i, off);
1521 verbose(env, "last insn is not an exit or jmp\n");
1526 if (cur_subprog < env->subprog_cnt) {
1537 static int mark_reg_read(struct bpf_verifier_env *env, const struct bpf_reg_state *state, struct bpf_reg_state *parent,
1549 verbose(env, "verifier BUG type %s var_off %lld off %d\n", reg_type_str(env, parent->type),
1580 if (env->longest_mark_read_walk < cnt) {
1581 env->longest_mark_read_walk = cnt;
1590 static bool is_reg64(struct bpf_verifier_env *env, struct bpf_insn *insn, u32 regno, struct bpf_reg_state *reg,
1689 static bool insn_has_def32(struct bpf_verifier_env *env, struct bpf_insn *insn)
1695 return !is_reg64(env, insn, insn->dst_reg, NULL, DST_OP);
1698 static void mark_insn_zext(struct bpf_verifier_env *env, struct bpf_reg_state *reg)
1706 env->insn_aux_data[def_idx - 1].zext_dst = true;
1711 static int check_reg_arg(struct bpf_verifier_env *env, u32 regno, enum reg_arg_type t)
1713 struct bpf_verifier_state *vstate = env->cur_state;
1715 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
1720 verbose(env, "R%d is invalid\n", regno);
1725 rw64 = is_reg64(env, insn, regno, reg, t);
1729 verbose(env, "R%d !read_ok\n", regno);
1738 mark_insn_zext(env, reg);
1741 return mark_reg_read(env, reg, reg->parent, rw64 ? REG_LIVE_READ64 : REG_LIVE_READ32);
1745 verbose(env, "frame pointer is read only\n");
1749 reg->subreg_def = rw64 ? DEF_NOT_SUBREG : env->insn_idx + 1;
1751 mark_reg_unknown(env, regs, regno);
1758 static int push_jmp_history(struct bpf_verifier_env *env, struct bpf_verifier_state *cur)
1768 p[cnt - 1].idx = env->insn_idx;
1769 p[cnt - 1].prev_idx = env->prev_insn_idx;
1795 static int backtrack_insn(struct bpf_verifier_env *env, int idx, u32 *reg_mask, u64 *stack_mask)
1799 .private_data = env,
1801 struct bpf_insn *insn = env->prog->insnsi + idx;
1812 if (env->log.level & BPF_LOG_LEVEL) {
1813 verbose(env, "regs=%x stack=%llx before ", *reg_mask, *stack_mask);
1814 verbose(env, "%d: ", idx);
1815 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
1876 verbose(env, "BUG spi %d\n", spi);
1898 verbose(env, "BUG spi %d\n", spi);
1920 verbose(env, "BUG regs %x\n", *reg_mask);
1996 static void mark_all_scalars_precise(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
2029 static int __mark_chain_precision(struct bpf_verifier_env *env, int regno, int spi)
2031 struct bpf_verifier_state *st = env->cur_state;
2033 int last_idx = env->insn_idx;
2042 if (!env->bpf_capable) {
2090 if (env->log.level & BPF_LOG_LEVEL) {
2091 verbose(env, "last_idx %d first_idx %d\n", last_idx, first_idx);
2098 err = backtrack_insn(env, i, ®_mask, &stack_mask);
2101 mark_all_scalars_precise(env, st);
2117 if (i >= env->prog->len) {
2124 verbose(env, "BUG backtracking idx %d\n", i);
2167 mark_all_scalars_precise(env, st);
2185 if (env->log.level & BPF_LOG_LEVEL) {
2186 print_verifier_state(env, func);
2187 verbose(env, "parent %s regs=%x stack=%llx marks\n", new_marks ? "didn't have" : "already had", reg_mask,
2204 static int mark_chain_precision(struct bpf_verifier_env *env, int regno)
2206 return __mark_chain_precision(env, regno, -1);
2209 static int mark_chain_precision_stack(struct bpf_verifier_env *env, int spi)
2211 return __mark_chain_precision(env, -1, spi);
2286 static int check_stack_write_fixed_off(struct bpf_verifier_env *env,
2292 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg;
2302 if (!env->allow_ptr_leaks && state->stack[spi].slot_type[0] == STACK_SPILL && size != BPF_REG_SIZE) {
2303 verbose(env, "attempt to corrupt spilled pointer on stack\n");
2307 cur = env->cur_state->frame[env->cur_state->curframe];
2311 if (!env->bypass_spec_v4) {
2322 env->insn_aux_data[insn_idx].sanitize_stack_spill = true;
2326 if (reg && size == BPF_REG_SIZE && register_is_bounded(reg) && !register_is_null(reg) && env->bpf_capable) {
2334 err = mark_chain_precision(env, value_regno);
2343 verbose_linfo(env, insn_idx, "; ");
2344 verbose(env, "invalid size of register spill\n");
2348 verbose(env, "cannot spill pointers to stack into stack frame of the caller\n");
2379 err = mark_chain_precision(env, value_regno);
2413 static int check_stack_write_var_off(struct bpf_verifier_env *env,
2428 cur = env->cur_state->frame[env->cur_state->curframe];
2453 if (!env->allow_ptr_leaks && *stype != NOT_INIT && *stype != SCALAR_VALUE) {
2460 verbose(env, "spilled ptr in range of var-offset stack write; insn %d, ptr off: %d", insn_idx, i);
2481 if (*stype == STACK_INVALID && !env->allow_uninit_stack) {
2482 verbose(env, "uninit stack in range of var-offset write prohibited for !root; insn %d, off: %d", insn_idx,
2490 err = mark_chain_precision(env, value_regno);
2506 static void mark_reg_stack_read(struct bpf_verifier_env *env,
2510 struct bpf_verifier_state *vstate = env->cur_state;
2543 mark_reg_unknown(env, state->regs, dst_regno);
2557 static int check_stack_read_fixed_off(struct bpf_verifier_env *env,
2561 struct bpf_verifier_state *vstate = env->cur_state;
2573 verbose_linfo(env, env->insn_idx, "; ");
2574 verbose(env, "invalid size of register fill\n");
2578 mark_reg_unknown(env, state->regs, dst_regno);
2581 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2586 verbose(env, "corrupted spill memory\n");
2599 } else if (__is_pointer_value(env->allow_ptr_leaks, reg)) {
2606 verbose(env, "leaking pointer from stack off %d\n", off);
2609 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2621 verbose(env, "invalid read from stack off %d+%d size %d\n", off, i, size);
2624 mark_reg_read(env, reg, reg->parent, REG_LIVE_READ64);
2626 mark_reg_stack_read(env, reg_state, off, off + size, dst_regno);
2637 static int check_stack_range_initialized(struct bpf_verifier_env *env, int regno, int off, int access_size,
2641 static struct bpf_reg_state *reg_state(struct bpf_verifier_env *env, int regno)
2643 return cur_regs(env) + regno;
2659 static int check_stack_read_var_off(struct bpf_verifier_env *env, int ptr_regno, int off, int size, int dst_regno)
2662 struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2663 struct bpf_func_state *ptr_state = func(env, reg);
2669 err = check_stack_range_initialized(env, ptr_regno, off, size, false, ACCESS_DIRECT, NULL);
2676 mark_reg_stack_read(env, ptr_state, min_off, max_off + size, dst_regno);
2689 static int check_stack_read(struct bpf_verifier_env *env, int ptr_regno, int off, int size, int dst_regno)
2691 struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2692 struct bpf_func_state *state = func(env, reg);
2704 verbose(env, "variable offset stack pointer cannot be passed into helper function; var_off=%s off=%d size=%d\n",
2712 if (!env->bypass_spec_v1 && var_off) {
2716 verbose(env, "R%d variable offset stack access prohibited for !root, var_off=%s\n", ptr_regno, tn_buf);
2722 err = check_stack_read_fixed_off(env, state, off, size, dst_regno);
2728 err = check_stack_read_var_off(env, ptr_regno, off, size, dst_regno);
2743 static int check_stack_write(struct bpf_verifier_env *env, int ptr_regno, int off, int size, int value_regno,
2746 struct bpf_reg_state *reg = reg_state(env, ptr_regno);
2747 struct bpf_func_state *state = func(env, reg);
2752 err = check_stack_write_fixed_off(env, state, off, size, value_regno, insn_idx);
2757 err = check_stack_write_var_off(env, state, ptr_regno, off, size, value_regno, insn_idx);
2762 static int check_map_access_type(struct bpf_verifier_env *env, u32 regno, int off, int size, enum bpf_access_type type)
2764 struct bpf_reg_state *regs = cur_regs(env);
2768 verbose(env, "write into map forbidden, value_size=%d off=%d size=%d\n", map->value_size, off, size);
2772 verbose(env, "read from map forbidden, value_size=%d off=%d size=%d\n", map->value_size, off, size);
2780 static int __check_mem_access(struct bpf_verifier_env *env, int regno, int off, int size, u32 mem_size,
2790 reg = &cur_regs(env)[regno];
2793 verbose(env, "invalid access to map value, value_size=%d off=%d size=%d\n", mem_size, off, size);
2798 verbose(env, "invalid access to packet, off=%d size=%d, R%d(id=%d,off=%d,r=%d)\n", off, size, regno,
2803 verbose(env, "invalid access to memory, mem_size=%u off=%d size=%d\n", mem_size, off, size);
2810 static int check_mem_region_access(struct bpf_verifier_env *env, u32 regno, int off, int size, u32 mem_size,
2813 struct bpf_verifier_state *vstate = env->cur_state;
2822 if (env->log.level & BPF_LOG_LEVEL) {
2823 print_verifier_state(env, state);
2835 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", regno);
2838 err = __check_mem_access(env, regno, reg->smin_value + off, size, mem_size, zero_size_allowed);
2840 verbose(env, "R%d min value is outside of the allowed memory range\n", regno);
2849 verbose(env, "R%d unbounded memory access, make sure to bounds check any such access\n", regno);
2852 err = __check_mem_access(env, regno, reg->umax_value + off, size, mem_size, zero_size_allowed);
2854 verbose(env, "R%d max value is outside of the allowed memory range\n", regno);
2862 static int check_map_access(struct bpf_verifier_env *env, u32 regno, int off, int size, bool zero_size_allowed)
2864 struct bpf_verifier_state *vstate = env->cur_state;
2870 err = check_mem_region_access(env, regno, off, size, map->value_size, zero_size_allowed);
2884 verbose(env, "bpf_spin_lock cannot be accessed directly by load/store\n");
2898 static bool may_access_direct_pkt_data(struct bpf_verifier_env *env, const struct bpf_call_arg_meta *meta,
2901 enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
2927 env->seen_direct_write = true;
2932 env->seen_direct_write = true;
2942 static int check_packet_access(struct bpf_verifier_env *env, u32 regno, int off, int size, bool zero_size_allowed)
2944 struct bpf_reg_state *regs = cur_regs(env);
2957 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", regno);
2960 err = __check_mem_access(env, regno, off, size, reg->range, zero_size_allowed);
2962 verbose(env, "R%d offset is outside of the packet\n", regno);
2972 env->prog->aux->max_pkt_offset = max_t(u32, env->prog->aux->max_pkt_offset, off + reg->umax_value + size - 1);
2978 static int check_ctx_access(struct bpf_verifier_env *env, int insn_idx, int off, int size, enum bpf_access_type t,
2983 .log = &env->log,
2986 if (env->ops->is_valid_access && env->ops->is_valid_access(off, size, t, env->prog, &info)) {
2999 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
3002 if (env->prog->aux->max_ctx_offset < off + size) {
3003 env->prog->aux->max_ctx_offset = off + size;
3008 verbose(env, "invalid bpf_context access off=%d size=%d\n", off, size);
3012 static int check_flow_keys_access(struct bpf_verifier_env *env, int off, int size)
3015 verbose(env, "invalid access to flow keys off=%d size=%d\n", off, size);
3021 static int check_sock_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, int off, int size,
3024 struct bpf_reg_state *regs = cur_regs(env);
3030 verbose(env, "R%d min value is negative, either use unsigned index or do a if (index >=0) check.\n", regno);
3052 env->insn_aux_data[insn_idx].ctx_field_size = info.ctx_field_size;
3056 verbose(env, "R%d invalid %s access off=%d size=%d\n", regno, reg_type_str(env, reg->type), off, size);
3061 static bool is_pointer_value(struct bpf_verifier_env *env, int regno)
3063 return __is_pointer_value(env->allow_ptr_leaks, reg_state(env, regno));
3066 static bool is_ctx_reg(struct bpf_verifier_env *env, int regno)
3068 const struct bpf_reg_state *reg = reg_state(env, regno);
3073 static bool is_sk_reg(struct bpf_verifier_env *env, int regno)
3075 const struct bpf_reg_state *reg = reg_state(env, regno);
3080 static bool is_pkt_reg(struct bpf_verifier_env *env, int regno)
3082 const struct bpf_reg_state *reg = reg_state(env, regno);
3087 static bool is_flow_key_reg(struct bpf_verifier_env *env, int regno)
3089 const struct bpf_reg_state *reg = reg_state(env, regno);
3095 static int check_pkt_ptr_alignment(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int off, int size,
3121 verbose(env, "misaligned packet access off %d+%s+%d+%d size %d\n", ip_align, tn_buf, reg->off, off, size);
3128 static int check_generic_ptr_alignment(struct bpf_verifier_env *env, const struct bpf_reg_state *reg,
3143 verbose(env, "misaligned %saccess off %s+%d+%d size %d\n", pointer_desc, tn_buf, reg->off, off, size);
3150 static int check_ptr_alignment(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int off, int size,
3153 bool strict = env->strict_alignment || strict_alignment_once;
3162 return check_pkt_ptr_alignment(env, reg, off, size, strict);
3195 return check_generic_ptr_alignment(env, reg, pointer_desc, off, size, strict);
3198 static int update_stack_depth(struct bpf_verifier_env *env, const struct bpf_func_state *func, int off)
3200 u16 stack = env->subprog_info[func->subprogno].stack_depth;
3207 env->subprog_info[func->subprogno].stack_depth = -off;
3217 static int check_max_stack_depth(struct bpf_verifier_env *env)
3220 struct bpf_subprog_info *subprog = env->subprog_info;
3221 struct bpf_insn *insn = env->prog->insnsi;
3251 verbose(env, "tail_calls are not allowed when call stack of previous frames is %d bytes. Too large\n",
3260 verbose(env, "combined stack size of %d calls is %d. Too large\n", frame + 1, depth);
3280 idx = find_subprog(env, i);
3292 verbose(env, "the call stack of %d frames is too deep !\n", frame);
3317 env->prog->aux->tail_call_reachable = true;
3336 static int get_callee_stack_depth(struct bpf_verifier_env *env, const struct bpf_insn *insn, int idx)
3340 subprog = find_subprog(env, start);
3345 return env->subprog_info[subprog].stack_depth;
3349 static int __check_ptr_off_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno,
3357 verbose(env, "dereference of modified %s ptr R%d off=%d disallowed\n", reg_type_str(env, reg->type), regno,
3366 verbose(env, "variable %s access var_off=%s disallowed\n", reg_type_str(env, reg->type), tn_buf);
3373 int check_ptr_off_reg(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno)
3375 return __check_ptr_off_reg(env, reg, regno, false);
3378 static int __check_buffer_access(struct bpf_verifier_env *env, const char *buf_info, const struct bpf_reg_state *reg,
3382 verbose(env, "R%d invalid %s buffer access: off=%d, size=%d\n", regno, buf_info, off, size);
3389 verbose(env, "R%d invalid variable buffer offset: off=%d, var_off=%s\n", regno, off, tn_buf);
3396 static int check_tp_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno, int off,
3401 err = __check_buffer_access(env, "tracepoint", reg, regno, off, size);
3406 if (off + size > env->prog->aux->max_tp_access) {
3407 env->prog->aux->max_tp_access = off + size;
3413 static int check_buffer_access(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, int regno, int off,
3418 err = __check_buffer_access(env, buf_info, reg, regno, off, size);
3505 static int check_ptr_to_btf_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, int regno, int off,
3515 verbose(env, "R%d is ptr_%s invalid negative access: off=%d\n", regno, tname, off);
3522 verbose(env, "R%d is ptr_%s invalid variable offset: off=%d, var_off=%s\n", regno, tname, off, tn_buf);
3526 if (env->ops->btf_struct_access) {
3527 ret = env->ops->btf_struct_access(&env->log, t, off, size, atype, &btf_id);
3530 verbose(env, "only read is supported\n");
3534 ret = btf_struct_access(&env->log, t, off, size, atype, &btf_id);
3542 mark_btf_ld_reg(env, regs, value_regno, ret, btf_id);
3548 static int check_ptr_to_map_access(struct bpf_verifier_env *env, struct bpf_reg_state *regs, int regno, int off,
3559 verbose(env, "map_ptr access not supported without CONFIG_DEBUG_INFO_BTF\n");
3564 verbose(env, "map_ptr access not supported for map type %d\n", map->map_type);
3571 if (!env->allow_ptr_to_map_access) {
3572 verbose(env, "%s access is allowed only to CAP_PERFMON and CAP_SYS_ADMIN\n", tname);
3577 verbose(env, "R%d is %s invalid negative access: off=%d\n", regno, tname, off);
3582 verbose(env, "only read from %s is supported\n", tname);
3586 ret = btf_struct_access(&env->log, t, off, size, atype, &btf_id);
3592 mark_btf_ld_reg(env, regs, value_regno, ret, btf_id);
3625 static int check_stack_access_within_bounds(struct bpf_verifier_env *env, int regno, int off, int access_size,
3628 struct bpf_reg_state *regs = cur_regs(env);
3630 struct bpf_func_state *state = func(env, reg);
3653 verbose(env, "invalid unbounded variable-offset%s stack R%d\n", err_extra, regno);
3671 verbose(env, "invalid%s stack R%d off=%d size=%d\n", err_extra, regno, off, access_size);
3676 verbose(env, "invalid variable-offset%s stack R%d var_off=%s size=%d\n", err_extra, regno, tn_buf,
3689 static int check_mem_access(struct bpf_verifier_env *env, int insn_idx, u32 regno, int off, int bpf_size,
3692 struct bpf_reg_state *regs = cur_regs(env);
3703 err = check_ptr_alignment(env, reg, off, size, strict_alignment_once);
3712 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) {
3713 verbose(env, "R%d leaks addr into map\n", value_regno);
3716 err = check_map_access_type(env, regno, off, size, t);
3720 err = check_map_access(env, regno, off, size, false);
3737 mark_reg_unknown(env, regs, value_regno);
3744 verbose(env, "R%d invalid mem access '%s'\n", regno, reg_type_str(env, reg->type));
3749 verbose(env, "R%d cannot write into %s\n", regno, reg_type_str(env, reg->type));
3753 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) {
3754 verbose(env, "R%d leaks addr into mem\n", value_regno);
3758 err = check_mem_region_access(env, regno, off, size, reg->mem_size, false);
3760 mark_reg_unknown(env, regs, value_regno);
3766 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) {
3767 verbose(env, "R%d leaks addr into ctx\n", value_regno);
3771 err = check_ptr_off_reg(env, reg, regno);
3776 err = check_ctx_access(env, insn_idx, off, size, t, ®_type, &btf_id);
3778 verbose_linfo(env, insn_idx, "; ");
3786 mark_reg_unknown(env, regs, value_regno);
3788 mark_reg_known_zero(env, regs, value_regno);
3790 regs[value_regno].id = ++env->id_gen;
3806 err = check_stack_access_within_bounds(env, regno, off, size, ACCESS_DIRECT, t);
3810 state = func(env, reg);
3811 err = update_stack_depth(env, state, off);
3817 err = check_stack_read(env, regno, off, size, value_regno);
3819 err = check_stack_write(env, regno, off, size, value_regno, insn_idx);
3822 if (t == BPF_WRITE && !may_access_direct_pkt_data(env, NULL, t)) {
3823 verbose(env, "cannot write into packet\n");
3826 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) {
3827 verbose(env, "R%d leaks addr into packet\n", value_regno);
3830 err = check_packet_access(env, regno, off, size, false);
3832 mark_reg_unknown(env, regs, value_regno);
3835 if (t == BPF_WRITE && value_regno >= 0 && is_pointer_value(env, value_regno)) {
3836 verbose(env, "R%d leaks addr into flow keys\n", value_regno);
3840 err = check_flow_keys_access(env, off, size);
3842 mark_reg_unknown(env, regs, value_regno);
3846 verbose(env, "R%d cannot write into %s\n", regno, reg_type_str(env, reg->type));
3849 err = check_sock_access(env, insn_idx, regno, off, size, t);
3851 mark_reg_unknown(env, regs, value_regno);
3854 err = check_tp_buffer_access(env, reg, regno, off, size);
3856 mark_reg_unknown(env, regs, value_regno);
3859 err = check_ptr_to_btf_access(env, regs, regno, off, size, t, value_regno);
3861 err = check_ptr_to_map_access(env, regs, regno, off, size, t, value_regno);
3869 verbose(env, "R%d cannot write into %s\n", regno, reg_type_str(env, reg->type));
3873 max_access = &env->prog->aux->max_rdonly_access;
3876 max_access = &env->prog->aux->max_rdwr_access;
3879 err = check_buffer_access(env, reg, regno, off, size, false, buf_info, max_access);
3881 mark_reg_unknown(env, regs, value_regno);
3884 verbose(env, "R%d invalid mem access '%s'\n", regno, reg_type_str(env, reg->type));
3895 static int check_xadd(struct bpf_verifier_env *env, int insn_idx, struct bpf_insn *insn)
3900 verbose(env, "BPF_XADD uses reserved fields\n");
3905 err = check_reg_arg(env, insn->src_reg, SRC_OP);
3911 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
3916 if (is_pointer_value(env, insn->src_reg)) {
3917 verbose(env, "R%d leaks addr into mem\n", insn->src_reg);
3921 if (is_ctx_reg(env, insn->dst_reg) || is_pkt_reg(env, insn->dst_reg) || is_flow_key_reg(env, insn->dst_reg) ||
3922 is_sk_reg(env, insn->dst_reg)) {
3923 verbose(env, "BPF_XADD stores into R%d %s is not allowed\n", insn->dst_reg,
3924 reg_type_str(env, reg_state(env, insn->dst_reg)->type));
3929 err = check_mem_access(env, insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_READ, -1, true);
3935 return check_mem_access(env, insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1, true);
3947 static int check_stack_range_initialized(struct bpf_verifier_env *env, int regno, int off, int access_size,
3951 struct bpf_reg_state *reg = reg_state(env, regno);
3952 struct bpf_func_state *state = func(env, reg);
3962 verbose(env, "invalid zero-sized read\n");
3976 err = check_stack_access_within_bounds(env, regno, off, access_size, type, bounds_check_type);
3989 if (!env->bypass_spec_v1) {
3993 verbose(env, "R%d%s variable offset stack access prohibited for !root, var_off=%s\n", regno, err_extra,
4042 (state->stack[spi].spilled_ptr.type == SCALAR_VALUE || env->allow_ptr_leaks)) {
4044 __mark_reg_unknown(env, &state->stack[spi].spilled_ptr);
4054 verbose(env, "invalid%s read from stack R%d off %d+%d size %d\n", err_extra, regno, min_off, i - min_off,
4060 verbose(env, "invalid%s read from stack R%d var_off %s+%d size %d\n", err_extra, regno, tn_buf, i - min_off,
4068 mark_reg_read(env, &state->stack[spi].spilled_ptr, state->stack[spi].spilled_ptr.parent, REG_LIVE_READ64);
4070 return update_stack_depth(env, state, min_off);
4073 static int check_helper_mem_access(struct bpf_verifier_env *env, int regno, int access_size, bool zero_size_allowed,
4076 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
4083 return check_packet_access(env, regno, reg->off, access_size, zero_size_allowed);
4085 if (check_map_access_type(env, regno, reg->off, access_size,
4089 return check_map_access(env, regno, reg->off, access_size, zero_size_allowed);
4091 return check_mem_region_access(env, regno, reg->off, access_size, reg->mem_size, zero_size_allowed);
4099 max_access = &env->prog->aux->max_rdonly_access;
4102 max_access = &env->prog->aux->max_rdwr_access;
4104 return check_buffer_access(env, reg, regno, reg->off, access_size, zero_size_allowed, buf_info, max_access);
4106 return check_stack_range_initialized(env, regno, reg->off, access_size, zero_size_allowed, ACCESS_HELPER,
4114 verbose(env, "R%d type=%s ", regno, reg_type_str(env, reg->type));
4115 verbose(env, "expected=%s\n", reg_type_str(env, PTR_TO_STACK));
4139 static int process_spin_lock(struct bpf_verifier_env *env, int regno, bool is_lock)
4141 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
4142 struct bpf_verifier_state *cur = env->cur_state;
4148 verbose(env, "R%d doesn't have constant offset. bpf_spin_lock has to be at the constant offset\n", regno);
4152 verbose(env, "map '%s' has to have BTF in order to use bpf_spin_lock\n", map->name);
4157 verbose(env, "map '%s' has more than one 'struct bpf_spin_lock'\n", map->name);
4159 verbose(env, "map '%s' doesn't have 'struct bpf_spin_lock'\n", map->name);
4161 verbose(env, "map '%s' is not a struct type or bpf_spin_lock is mangled\n", map->name);
4166 verbose(env, "off %lld doesn't point to 'struct bpf_spin_lock'\n", val + reg->off);
4171 verbose(env, "Locking two bpf_spin_locks are not allowed\n");
4177 verbose(env, "bpf_spin_unlock without taking a lock\n");
4181 verbose(env, "bpf_spin_unlock of different lock\n");
4220 static int resolve_map_arg_type(struct bpf_verifier_env *env, const struct bpf_call_arg_meta *meta,
4225 verbose(env, "invalid map_ptr to access map->type\n");
4235 verbose(env, "invalid arg_type for sockmap/sockhash\n");
4341 static int check_reg_type(struct bpf_verifier_env *env, u32 regno, enum bpf_arg_type arg_type, const u32 *arg_btf_id)
4343 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
4350 verbose(env, "verifier internal error: unsupported arg type %d\n", arg_type);
4382 verbose(env, "R%d type=%s expected=", regno, reg_type_str(env, reg->type));
4384 verbose(env, "%s, ", reg_type_str(env, compatible->types[j]));
4386 verbose(env, "%s\n", reg_type_str(env, compatible->types[j]));
4393 verbose(env, "verifier internal error: missing arg compatible BTF ID\n");
4399 if (!btf_struct_ids_match(&env->log, reg->off, reg->btf_id, *arg_btf_id)) {
4400 verbose(env, "R%d is of type %s but %s is expected\n", regno, kernel_type_name(reg->btf_id),
4409 static int check_func_arg(struct bpf_verifier_env *env, u32 arg, struct bpf_call_arg_meta *meta,
4413 struct bpf_reg_state *regs = cur_regs(env), *reg = ®s[regno];
4422 err = check_reg_arg(env, regno, SRC_OP);
4428 if (is_pointer_value(env, regno)) {
4429 verbose(env, "R%d leaks addr into helper function\n", regno);
4435 if (type_is_pkt_pointer(type) && !may_access_direct_pkt_data(env, meta, BPF_READ)) {
4436 verbose(env, "helper access to the packet is not allowed\n");
4441 err = resolve_map_arg_type(env, meta, &arg_type);
4454 err = check_reg_type(env, regno, arg_type, fn->arg_btf_id[arg]);
4481 err = __check_ptr_off_reg(env, reg, regno, type == PTR_TO_BTF_ID);
4491 verbose(env, "verifier internal error: more than one arg with ref_obj_id R%d %u %u\n", regno,
4512 verbose(env, "invalid map_ptr to access map->key\n");
4515 err = check_helper_mem_access(env, regno, meta->map_ptr->key_size, false, NULL);
4526 verbose(env, "invalid map_ptr to access map->value\n");
4530 err = check_helper_mem_access(env, regno, meta->map_ptr->value_size, false, meta);
4533 verbose(env, "Helper has invalid btf_id in R%d\n", regno);
4539 if (process_spin_lock(env, regno, true)) {
4543 if (process_spin_lock(env, regno, false)) {
4547 verbose(env, "verifier internal error\n");
4581 verbose(env, "R%d min value is negative, either use unsigned or 'var &= const'\n", regno);
4586 err = check_helper_mem_access(env, regno - 1, 0, zero_size_allowed, meta);
4593 verbose(env, "R%d unbounded memory access, use 'var &= const' or 'if (var < const)'\n", regno);
4596 err = check_helper_mem_access(env, regno - 1, reg->umax_value, zero_size_allowed, meta);
4598 err = mark_chain_precision(env, regno);
4602 verbose(env, "R%d unbounded size, use 'var &= const' or 'if (var < const)'\n", regno);
4609 err = check_helper_mem_access(env, regno, size, false, meta);
4613 err = check_ptr_alignment(env, reg, 0, size, true);
4619 static bool may_update_sockmap(struct bpf_verifier_env *env, int func_id)
4621 enum bpf_attach_type eatype = env->prog->expected_attach_type;
4622 enum bpf_prog_type type = resolve_prog_type(env->prog);
4649 verbose(env, "cannot update sockmap in this context\n");
4653 static bool allow_tail_call_in_subprogs(struct bpf_verifier_env *env)
4655 return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64);
4658 static int check_map_func_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, int func_id)
4729 !may_update_sockmap(env, func_id)) {
4737 !may_update_sockmap(env, func_id)) {
4773 if (env->subprog_cnt > 1 && !allow_tail_call_in_subprogs(env)) {
4774 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
4861 verbose(env, "cannot pass map_type %d into func %s#%d\n", map->map_type, func_id_name(func_id), func_id);
4976 static void __clear_all_pkt_pointers(struct bpf_verifier_env *env, struct bpf_func_state *state)
4983 mark_reg_unknown(env, regs, i);
4993 __mark_reg_unknown(env, reg);
4998 static void clear_all_pkt_pointers(struct bpf_verifier_env *env)
5000 struct bpf_verifier_state *vstate = env->cur_state;
5004 __clear_all_pkt_pointers(env, vstate->frame[i]);
5008 static void release_reg_references(struct bpf_verifier_env *env, struct bpf_func_state *state, int ref_obj_id)
5015 mark_reg_unknown(env, regs, i);
5025 __mark_reg_unknown(env, reg);
5033 static int release_reference(struct bpf_verifier_env *env, int ref_obj_id)
5035 struct bpf_verifier_state *vstate = env->cur_state;
5039 err = release_reference_state(cur_func(env), ref_obj_id);
5045 release_reg_references(env, vstate->frame[i], ref_obj_id);
5051 static void clear_caller_saved_regs(struct bpf_verifier_env *env, struct bpf_reg_state *regs)
5057 mark_reg_not_init(env, regs, caller_saved[i]);
5058 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
5062 static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx)
5064 struct bpf_verifier_state *state = env->cur_state;
5071 verbose(env, "the call stack of %d frames is too deep\n", state->curframe + 2);
5076 subprog = find_subprog(env, target_insn + 1);
5078 verbose(env, "verifier bug. No program starts at insn %d\n", target_insn + 1);
5084 verbose(env, "verifier bug. Frame %d already allocated\n", state->curframe + 1);
5088 func_info_aux = env->prog->aux->func_info_aux;
5092 err = btf_check_func_arg_match(env, subprog, caller->regs);
5098 verbose(env, "Caller passes invalid args into func#%d\n", subprog);
5101 if (env->log.level & BPF_LOG_LEVEL) {
5102 verbose(env, "Func#%d is global and valid. Skipping.\n", subprog);
5104 clear_caller_saved_regs(env, caller->regs);
5107 mark_reg_unknown(env, caller->regs, BPF_REG_0);
5125 init_func_state(env, callee,
5143 clear_caller_saved_regs(env, caller->regs);
5151 if (env->log.level & BPF_LOG_LEVEL) {
5152 verbose(env, "caller:\n");
5153 print_verifier_state(env, caller);
5154 verbose(env, "callee:\n");
5155 print_verifier_state(env, callee);
5160 static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx)
5162 struct bpf_verifier_state *state = env->cur_state;
5176 verbose(env, "cannot return stack pointer to the caller\n");
5192 if (env->log.level & BPF_LOG_LEVEL) {
5193 verbose(env, "returning from callee:\n");
5194 print_verifier_state(env, callee);
5195 verbose(env, "to caller at %d:\n", *insn_idx);
5196 print_verifier_state(env, caller);
5222 static int record_func_map(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int func_id, int insn_idx)
5224 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
5234 verbose(env, "kernel subsystem misconfigured verifier\n");
5245 verbose(env, "write into map forbidden\n");
5257 static int record_func_key(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, int func_id, int insn_idx)
5259 struct bpf_insn_aux_data *aux = &env->insn_aux_data[insn_idx];
5260 struct bpf_reg_state *regs = cur_regs(env), *reg;
5269 verbose(env, "kernel subsystem misconfigured verifier\n");
5282 err = mark_chain_precision(env, BPF_REG_3);
5295 static int check_reference_leak(struct bpf_verifier_env *env)
5297 struct bpf_func_state *state = cur_func(env);
5301 verbose(env, "Unreleased reference id=%d alloc_insn=%d\n", state->refs[i].id, state->refs[i].insn_idx);
5306 static int check_helper_call(struct bpf_verifier_env *env, int func_id, int insn_idx)
5318 verbose(env, "invalid func %s#%d\n", func_id_name(func_id), func_id);
5322 if (env->ops->get_func_proto) {
5323 fn = env->ops->get_func_proto(func_id, env->prog);
5326 verbose(env, "unknown func %s#%d\n", func_id_name(func_id), func_id);
5331 if (!env->prog->gpl_compatible && fn->gpl_only) {
5332 verbose(env, "cannot call GPL-restricted function from non-GPL compatible program\n");
5336 if (fn->allowed && !fn->allowed(env->prog)) {
5337 verbose(env, "helper call is not allowed in probe\n");
5344 verbose(env, "kernel subsystem misconfigured func %s#%d: r1 != ctx\n", func_id_name(func_id), func_id);
5353 verbose(env, "kernel subsystem misconfigured func %s#%d\n", func_id_name(func_id), func_id);
5360 err = check_func_arg(env, i, &meta, fn);
5366 err = record_func_map(env, &meta, func_id, insn_idx);
5371 err = record_func_key(env, &meta, func_id, insn_idx);
5380 err = check_mem_access(env, insn_idx, meta.regno, i, BPF_B, BPF_WRITE, -1, false);
5387 err = check_reference_leak(env);
5389 verbose(env, "tail_call would lead to reference leak\n");
5393 err = release_reference(env, meta.ref_obj_id);
5395 verbose(env, "func %s#%d reference has not been acquired before\n", func_id_name(func_id), func_id);
5400 regs = cur_regs(env);
5405 verbose(env, "get_local_storage() doesn't support non-zero flags\n");
5411 mark_reg_not_init(env, regs, caller_saved[i]);
5412 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
5423 mark_reg_unknown(env, regs, BPF_REG_0);
5428 mark_reg_known_zero(env, regs, BPF_REG_0);
5434 verbose(env, "kernel subsystem misconfigured verifier\n");
5440 regs[BPF_REG_0].id = ++env->id_gen;
5443 mark_reg_known_zero(env, regs, BPF_REG_0);
5446 mark_reg_known_zero(env, regs, BPF_REG_0);
5449 mark_reg_known_zero(env, regs, BPF_REG_0);
5452 mark_reg_known_zero(env, regs, BPF_REG_0);
5458 mark_reg_known_zero(env, regs, BPF_REG_0);
5469 verbose(env, "unable to resolve the size of type '%s': %ld\n", tname, PTR_ERR(ret));
5488 mark_reg_known_zero(env, regs, BPF_REG_0);
5492 verbose(env, "invalid return type %u of func %s#%d\n", base_type(ret_type), func_id_name(func_id), func_id);
5497 verbose(env, "unknown return type %u of func %s#%d\n", base_type(ret_type), func_id_name(func_id), func_id);
5502 regs[BPF_REG_0].id = ++env->id_gen;
5509 int id = acquire_reference_state(env, insn_idx);
5521 err = check_map_func_compatibility(env, meta.map_ptr, func_id);
5526 if ((func_id == BPF_FUNC_get_stack || func_id == BPF_FUNC_get_task_stack) && !env->prog->has_callchain_buf) {
5537 verbose(env, err_str, func_id_name(func_id), func_id);
5541 env->prog->has_callchain_buf = true;
5545 env->prog->call_get_stack = true;
5549 clear_all_pkt_pointers(env);
5598 static bool check_reg_sane_offset(struct bpf_verifier_env *env, const struct bpf_reg_state *reg, enum bpf_reg_type type)
5605 verbose(env, "math between %s pointer and %lld is not allowed\n", reg_type_str(env, type), val);
5610 verbose(env, "%s pointer offset %d is not allowed\n", reg_type_str(env, type), reg->off);
5615 verbose(env, "math between %s pointer and register with unbounded min value is not allowed\n",
5616 reg_type_str(env, type));
5621 verbose(env, "value %lld makes %s pointer be out of bounds\n", smin, reg_type_str(env, type));
5628 static struct bpf_insn_aux_data *cur_aux(struct bpf_verifier_env *env)
5630 return &env->insn_aux_data[env->insn_idx];
5670 static bool can_skip_alu_sanitation(const struct bpf_verifier_env *env, const struct bpf_insn *insn)
5672 return env->bypass_spec_v1 || BPF_SRC(insn->code) == BPF_K;
5690 static int sanitize_val_alu(struct bpf_verifier_env *env, struct bpf_insn *insn)
5692 struct bpf_insn_aux_data *aux = cur_aux(env);
5694 if (can_skip_alu_sanitation(env, insn)) {
5711 static struct bpf_verifier_state *sanitize_speculative_path(struct bpf_verifier_env *env, const struct bpf_insn *insn,
5717 branch = push_stack(env, next_idx, curr_idx, true);
5721 mark_reg_unknown(env, regs, insn->dst_reg);
5723 mark_reg_unknown(env, regs, insn->dst_reg);
5724 mark_reg_unknown(env, regs, insn->src_reg);
5730 static int sanitize_ptr_alu(struct bpf_verifier_env *env, struct bpf_insn *insn, const struct bpf_reg_state *ptr_reg,
5734 struct bpf_insn_aux_data *aux = commit_window ? cur_aux(env) : &info->aux;
5735 struct bpf_verifier_state *vstate = env->cur_state;
5745 if (can_skip_alu_sanitation(env, insn)) {
5785 env->explore_alu_limits = true;
5819 ret = sanitize_speculative_path(env, NULL, env->insn_idx + 1, env->insn_idx);
5826 static void sanitize_mark_insn_seen(struct bpf_verifier_env *env)
5828 struct bpf_verifier_state *vstate = env->cur_state;
5836 env->insn_aux_data[env->insn_idx].seen = env->pass_cnt;
5840 static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason,
5849 verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", off_reg == dst_reg ? dst : src, err);
5852 verbose(env, "R%d has pointer with unsupported alu operation, %s\n", off_reg == dst_reg ? src : dst, err);
5855 verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", dst, op, err);
5858 verbose(env, "R%d tried to %s beyond pointer bounds, %s\n", dst, op, err);
5861 verbose(env, "R%d could not be pushed for speculative verification, %s\n", dst, err);
5864 verbose(env, "verifier internal error: unknown reason (%d)\n", reason);
5881 static int check_stack_access_for_ptr_arithmetic(struct bpf_verifier_env *env, int regno,
5888 verbose(env, "R%d variable stack access prohibited for !root, var_off=%s off=%d\n", regno, tn_buf, off);
5893 verbose(env,
5903 static int sanitize_check_bounds(struct bpf_verifier_env *env, const struct bpf_insn *insn,
5911 if (env->bypass_spec_v1) {
5917 if (check_stack_access_for_ptr_arithmetic(env, dst, dst_reg, dst_reg->off + dst_reg->var_off.value)) {
5922 if (check_map_access(env, dst, dst_reg->off, 1, false)) {
5923 verbose(env,
5940 * scalar. So we only emit a diagnostic if !env->allow_ptr_leaks.
5942 static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn,
5945 struct bpf_verifier_state *vstate = env->cur_state;
5964 __mark_reg_unknown(env, dst_reg);
5970 if (opcode == BPF_SUB && env->allow_ptr_leaks) {
5971 __mark_reg_unknown(env, dst_reg);
5975 verbose(env, "R%d 32-bit pointer arithmetic prohibited\n", dst);
5980 verbose(env, "R%d pointer arithmetic on %s prohibited, null-check it first\n", dst,
5981 reg_type_str(env, ptr_reg->type));
5998 verbose(env, "R%d pointer arithmetic on %s prohibited\n", dst, reg_type_str(env, ptr_reg->type));
6013 if (!check_reg_sane_offset(env, off_reg, ptr_reg->type) || !check_reg_sane_offset(env, ptr_reg, ptr_reg->type)) {
6021 ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, &info, false);
6023 return sanitize_err(env, insn, ret, off_reg, dst_reg);
6070 dst_reg->id = ++env->id_gen;
6078 verbose(env, "R%d tried to subtract pointer from scalar\n", dst);
6086 verbose(env, "R%d subtraction from stack pointer prohibited\n", dst);
6125 dst_reg->id = ++env->id_gen;
6136 verbose(env, "R%d bitwise operator %s on pointer prohibited\n", dst, bpf_alu_string[opcode >> 0x4]);
6140 verbose(env, "R%d pointer arithmetic with %s operator prohibited\n", dst, bpf_alu_string[opcode >> 0x4]);
6144 if (!check_reg_sane_offset(env, dst_reg, ptr_reg->type)) {
6150 if (sanitize_check_bounds(env, insn, dst_reg) < 0) {
6154 ret = sanitize_ptr_alu(env, insn, dst_reg, off_reg, dst_reg, &info, true);
6156 return sanitize_err(env, insn, ret, off_reg, dst_reg);
6712 static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn,
6715 struct bpf_reg_state *regs = cur_regs(env);
6743 __mark_reg_unknown(env, dst_reg);
6753 __mark_reg_unknown(env, dst_reg);
6759 __mark_reg_unknown(env, dst_reg);
6764 ret = sanitize_val_alu(env, insn);
6766 return sanitize_err(env, insn, ret, NULL, NULL);
6820 mark_reg_unknown(env, regs, insn->dst_reg);
6834 mark_reg_unknown(env, regs, insn->dst_reg);
6848 mark_reg_unknown(env, regs, insn->dst_reg);
6858 mark_reg_unknown(env, regs, insn->dst_reg);
6874 static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, struct bpf_insn *insn)
6876 struct bpf_verifier_state *vstate = env->cur_state;
6901 if (opcode == BPF_SUB && env->allow_ptr_leaks) {
6902 mark_reg_unknown(env, regs, insn->dst_reg);
6905 verbose(env, "R%d pointer %s pointer prohibited\n", insn->dst_reg,
6913 err = mark_chain_precision(env, insn->dst_reg);
6917 return adjust_ptr_min_max_vals(env, insn, src_reg, dst_reg);
6921 err = mark_chain_precision(env, insn->src_reg);
6925 return adjust_ptr_min_max_vals(env, insn, dst_reg, src_reg);
6935 return adjust_ptr_min_max_vals(env, insn, ptr_reg, src_reg);
6941 print_verifier_state(env, state);
6942 verbose(env, "verifier internal error: unexpected ptr_reg\n");
6946 print_verifier_state(env, state);
6947 verbose(env, "verifier internal error: no src_reg\n");
6950 return adjust_scalar_min_max_vals(env, insn, dst_reg, *src_reg);
6954 static int check_alu_op(struct bpf_verifier_env *env, struct bpf_insn *insn)
6956 struct bpf_reg_state *regs = cur_regs(env);
6963 verbose(env, "BPF_NEG uses reserved fields\n");
6970 verbose(env, "BPF_END uses reserved fields\n");
6976 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
6981 if (is_pointer_value(env, insn->dst_reg)) {
6982 verbose(env, "R%d pointer arithmetic prohibited\n", insn->dst_reg);
6987 err = check_reg_arg(env, insn->dst_reg, DST_OP);
6994 verbose(env, "BPF_MOV uses reserved fields\n");
6998 err = check_reg_arg(env, insn->src_reg, SRC_OP);
7004 verbose(env, "BPF_MOV uses reserved fields\n");
7010 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
7028 src_reg->id = ++env->id_gen;
7035 if (is_pointer_value(env, insn->src_reg)) {
7036 verbose(env, "R%d partial copy of pointer\n", insn->src_reg);
7046 dst_reg->subreg_def = env->insn_idx + 1;
7048 mark_reg_unknown(env, regs, insn->dst_reg);
7059 mark_reg_unknown(env, regs, insn->dst_reg);
7068 verbose(env, "invalid BPF_ALU opcode %x\n", opcode);
7073 verbose(env, "BPF_ALU uses reserved fields\n");
7077 err = check_reg_arg(env, insn->src_reg, SRC_OP);
7083 verbose(env, "BPF_ALU uses reserved fields\n");
7089 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
7095 verbose(env, "div by zero\n");
7102 verbose(env, "invalid shift %d\n", insn->imm);
7107 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
7111 return adjust_reg_min_max_vals(env, insn);
7822 static int check_cond_jmp_op(struct bpf_verifier_env *env, struct bpf_insn *insn, int *insn_idx)
7824 struct bpf_verifier_state *this_branch = env->cur_state;
7835 verbose(env, "invalid BPF_JMP/JMP32 opcode %x\n", opcode);
7841 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
7846 err = check_reg_arg(env, insn->src_reg, SRC_OP);
7851 if (is_pointer_value(env, insn->src_reg)) {
7852 verbose(env, "R%d pointer comparison prohibited\n", insn->src_reg);
7858 verbose(env, "BPF_JMP/JMP32 uses reserved fields\n");
7864 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
7885 err = mark_chain_precision(env, insn->dst_reg);
7888 err = mark_chain_precision(env, insn->src_reg);
7900 if (!env->bypass_spec_v1 && !sanitize_speculative_path(env, insn, *insn_idx + 1, *insn_idx)) {
7910 if (!env->bypass_spec_v1 && !sanitize_speculative_path(env, insn, *insn_idx + insn->off + 1, *insn_idx)) {
7916 other_branch = push_stack(env, *insn_idx + insn->off + 1, *insn_idx, false);
7971 is_pointer_value(env, insn->dst_reg)) {
7972 verbose(env, "R%d pointer comparison prohibited\n", insn->dst_reg);
7975 if (env->log.level & BPF_LOG_LEVEL) {
7976 print_verifier_state(env, this_branch->frame[this_branch->curframe]);
7982 static int check_ld_imm(struct bpf_verifier_env *env, struct bpf_insn *insn)
7984 struct bpf_insn_aux_data *aux = cur_aux(env);
7985 struct bpf_reg_state *regs = cur_regs(env);
7991 verbose(env, "invalid BPF_LD_IMM insn\n");
7995 verbose(env, "BPF_LD_IMM64 uses reserved fields\n");
7999 err = check_reg_arg(env, insn->dst_reg, DST_OP);
8017 mark_reg_known_zero(env, regs, insn->dst_reg);
8030 verbose(env, "bpf verifier is misconfigured\n");
8036 map = env->used_maps[aux->map_index];
8043 dst_reg->id = ++env->id_gen;
8048 verbose(env, "bpf verifier is misconfigured\n");
8082 static int check_ld_abs(struct bpf_verifier_env *env, struct bpf_insn *insn)
8084 struct bpf_reg_state *regs = cur_regs(env);
8089 if (!may_access_skb(resolve_prog_type(env->prog))) {
8090 verbose(env, "BPF_LD_[ABS|IND] instructions not allowed for this program type\n");
8094 if (!env->ops->gen_ld_abs) {
8095 verbose(env, "bpf verifier is misconfigured\n");
8101 verbose(env, "BPF_LD_[ABS|IND] uses reserved fields\n");
8106 err = check_reg_arg(env, ctx_reg, SRC_OP);
8115 err = check_reference_leak(env);
8117 verbose(env, "BPF_LD_[ABS|IND] cannot be mixed with socket references\n");
8121 if (env->cur_state->active_spin_lock) {
8122 verbose(env, "BPF_LD_[ABS|IND] cannot be used inside bpf_spin_lock-ed region\n");
8127 verbose(env, "at the time of BPF_LD_ABS|IND R6 != pointer to skb\n");
8133 err = check_reg_arg(env, insn->src_reg, SRC_OP);
8139 err = check_ptr_off_reg(env, ®s[ctx_reg], ctx_reg);
8146 mark_reg_not_init(env, regs, caller_saved[i]);
8147 check_reg_arg(env, caller_saved[i], DST_OP_NO_MARK);
8154 mark_reg_unknown(env, regs, BPF_REG_0);
8156 regs[BPF_REG_0].subreg_def = env->insn_idx + 1;
8160 static int check_return_code(struct bpf_verifier_env *env)
8163 const struct bpf_prog *prog = env->prog;
8166 enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
8168 const bool is_subprog = env->cur_state->frame[0]->subprogno;
8182 err = check_reg_arg(env, BPF_REG_0, SRC_OP);
8187 if (is_pointer_value(env, BPF_REG_0)) {
8188 verbose(env, "R0 leaks addr as return value\n");
8192 reg = cur_regs(env) + BPF_REG_0;
8195 verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n",
8196 reg_type_str(env, reg->type));
8204 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
8205 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
8206 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
8207 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
8208 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
8209 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) {
8214 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
8226 if (!env->prog->aux->attach_btf_id) {
8232 switch (env->prog->expected_attach_type) {
8258 verbose(env, "At program exit the register R0 is not a known value (%s)\n", reg_type_str(env, reg->type));
8265 verbose(env, "At program exit the register R0 ");
8268 verbose(env, "has value %s", tn_buf);
8270 verbose(env, "has unknown scalar value");
8273 verbose(env, " should have been in %s\n", tn_buf);
8278 env->prog->enforce_expected_attach_type = 1;
8323 static u32 state_htab_size(struct bpf_verifier_env *env)
8325 return env->prog->len;
8328 static struct bpf_verifier_state_list **explored_state(struct bpf_verifier_env *env, int idx)
8330 struct bpf_verifier_state *cur = env->cur_state;
8333 return &env->explored_states[(idx ^ state->callsite) % state_htab_size(env)];
8336 static void init_explored_state(struct bpf_verifier_env *env, int idx)
8338 env->insn_aux_data[idx].prune_point = true;
8346 static int push_insn(int t, int w, int e, struct bpf_verifier_env *env, bool loop_ok)
8348 int *insn_stack = env->cfg.insn_stack;
8349 int *insn_state = env->cfg.insn_state;
8359 if (w < 0 || w >= env->prog->len) {
8360 verbose_linfo(env, t, "%d: ", t);
8361 verbose(env, "jump out of range from insn %d to %d\n", t, w);
8367 init_explored_state(env, w);
8374 if (env->cfg.cur_stack >= env->prog->len) {
8377 insn_stack[env->cfg.cur_stack++] = w;
8380 if (loop_ok && env->bpf_capable) {
8383 verbose_linfo(env, t, "%d: ", t);
8384 verbose_linfo(env, w, "%d: ", w);
8385 verbose(env, "back-edge from insn %d to %d\n", t, w);
8391 verbose(env, "insn state internal bug\n");
8400 static int check_cfg(struct bpf_verifier_env *env)
8402 struct bpf_insn *insns = env->prog->insnsi;
8403 int insn_cnt = env->prog->len;
8408 insn_state = env->cfg.insn_state = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
8413 insn_stack = env->cfg.insn_stack = kvcalloc(insn_cnt, sizeof(int), GFP_KERNEL);
8421 env->cfg.cur_stack = 1;
8424 if (env->cfg.cur_stack == 0) {
8427 t = insn_stack[env->cfg.cur_stack - 1];
8434 ret = push_insn(t, t + 1, FALLTHROUGH, env, false);
8441 init_explored_state(env, t + 1);
8444 init_explored_state(env, t);
8445 ret = push_insn(t, t + insns[t].imm + 1, BRANCH, env, false);
8458 ret = push_insn(t, t + insns[t].off + 1, FALLTHROUGH, env, true);
8468 init_explored_state(env, t + insns[t].off + 1);
8473 init_explored_state(env, t + 1);
8477 init_explored_state(env, t);
8478 ret = push_insn(t, t + 1, FALLTHROUGH, env, true);
8485 ret = push_insn(t, t + insns[t].off + 1, BRANCH, env, true);
8496 ret = push_insn(t, t + 1, FALLTHROUGH, env, false);
8506 if (env->cfg.cur_stack-- <= 0) {
8507 verbose(env, "pop stack internal bug\n");
8517 verbose(env, "unreachable insn %d\n", i);
8527 env->cfg.insn_state = env->cfg.insn_stack = NULL;
8531 static int check_abnormal_return(struct bpf_verifier_env *env)
8535 for (i = 1; i < env->subprog_cnt; i++) {
8536 if (env->subprog_info[i].has_ld_abs) {
8537 verbose(env, "LD_ABS is not allowed in subprogs without BTF\n");
8540 if (env->subprog_info[i].has_tail_call) {
8541 verbose(env, "tail_call is not allowed in subprogs without BTF\n");
8552 static int check_btf_func(struct bpf_verifier_env *env, const union bpf_attr *attr, union bpf_attr __user *uattr)
8568 if (check_abnormal_return(env)) {
8574 if (nfuncs != env->subprog_cnt) {
8575 verbose(env, "number of funcs in func_info doesn't match number of subprogs\n");
8581 verbose(env, "invalid func info rec size %u\n", urec_size);
8585 prog = env->prog;
8604 verbose(env, "nonzero tailing record in func info");
8624 verbose(env, "nonzero insn_off %u for the first func info record", krecord[i].insn_off);
8628 verbose(env, "same or smaller insn offset (%u) than previous func info record (%u)", krecord[i].insn_off,
8633 if (env->subprog_info[i].start != krecord[i].insn_off) {
8634 verbose(env, "func_info BTF section doesn't match subprog layout in BPF program\n");
8641 verbose(env, "invalid type id %d in func info", krecord[i].type_id);
8653 if (i && !scalar_return && env->subprog_info[i].has_ld_abs) {
8654 verbose(env, "LD_ABS is only allowed in functions that return 'int'.\n");
8657 if (i && !scalar_return && env->subprog_info[i].has_tail_call) {
8658 verbose(env, "tail_call is only allowed in functions that return 'int'.\n");
8677 static void adjust_btf_func(struct bpf_verifier_env *env)
8679 struct bpf_prog_aux *aux = env->prog->aux;
8686 for (i = 0; i < env->subprog_cnt; i++) {
8687 aux->func_info[i].insn_off = env->subprog_info[i].start;
8695 static int check_btf_line(struct bpf_verifier_env *env, const union bpf_attr *attr, union bpf_attr __user *uattr)
8726 prog = env->prog;
8730 sub = env->subprog_info;
8738 verbose(env, "nonzero tailing record in line_info");
8763 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", i, linfo[i].insn_off,
8770 verbose(env, "Invalid insn code at line_info[%u].insn_off\n", i);
8776 verbose(env, "Invalid line_info[%u].line_off or .file_name_off\n", i);
8781 if (s != env->subprog_cnt) {
8786 verbose(env, "missing bpf_line_info for func#%u\n", s);
8796 if (s != env->subprog_cnt) {
8797 verbose(env, "missing bpf_line_info for %u funcs starting from func#%u\n", env->subprog_cnt - s, s);
8812 static int check_btf_info(struct bpf_verifier_env *env, const union bpf_attr *attr, union bpf_attr __user *uattr)
8818 if (check_abnormal_return(env)) {
8828 env->prog->aux->btf = btf;
8830 err = check_btf_func(env, attr, uattr);
8835 err = check_btf_line(env, attr, uattr);
8882 static void clean_func_state(struct bpf_verifier_env *env, struct bpf_func_state *st)
8895 verifier_mark_reg_not_init(env, &st->regs[i]);
8904 verifier_mark_reg_not_init(env, &st->stack[i].spilled_ptr);
8912 static void clean_verifier_state(struct bpf_verifier_env *env, struct bpf_verifier_state *st)
8922 clean_func_state(env, st->frame[i]);
8957 static void clean_live_states(struct bpf_verifier_env *env, int insn, struct bpf_verifier_state *cur)
8962 sl = *explored_state(env, insn);
8975 clean_verifier_state(env, &sl->state);
8982 static bool regsafe(struct bpf_verifier_env *env, struct bpf_reg_state *rold, struct bpf_reg_state *rcur,
9014 if (env->explore_alu_limits) {
9109 static bool stacksafe(struct bpf_verifier_env *env, struct bpf_func_state *old, struct bpf_func_state *cur,
9160 if (!regsafe(env, &old->stack[spi].spilled_ptr, &cur->stack[spi].spilled_ptr, idmap)) {
9211 static bool func_states_equal(struct bpf_verifier_env *env, struct bpf_func_state *old, struct bpf_func_state *cur)
9215 memset(env->idmap_scratch, 0, sizeof(env->idmap_scratch));
9217 if (!regsafe(env, &old->regs[i], &cur->regs[i], env->idmap_scratch)) {
9222 if (!stacksafe(env, old, cur, env->idmap_scratch)) {
9233 static bool states_equal(struct bpf_verifier_env *env, struct bpf_verifier_state *old, struct bpf_verifier_state *cur)
9259 if (!func_states_equal(env, old->frame[i], cur->frame[i])) {
9269 static int propagate_liveness_reg(struct bpf_verifier_env *env, struct bpf_reg_state *reg,
9288 err = mark_reg_read(env, reg, parent_reg, flag);
9303 static int propagate_liveness(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,
9323 err = propagate_liveness_reg(env, &state_reg[i], &parent_reg[i]);
9328 mark_insn_zext(env, &parent_reg[i]);
9336 err = propagate_liveness_reg(env, state_reg, parent_reg);
9348 static int propagate_precision(struct bpf_verifier_env *env, const struct bpf_verifier_state *old)
9360 if (env->log.level & BPF_LOG_LEVEL2) {
9361 verbose(env, "propagating r%d\n", i);
9363 err = mark_chain_precision(env, i);
9377 if (env->log.level & BPF_LOG_LEVEL2) {
9378 verbose(env, "propagating fp%d\n", (-i - 1) * BPF_REG_SIZE);
9380 err = mark_chain_precision_stack(env, i);
9407 static int is_state_visited(struct bpf_verifier_env *env, int insn_idx)
9411 struct bpf_verifier_state *cur = env->cur_state, *new;
9413 bool add_new_state = env->test_state_freq ? true : false;
9415 cur->last_insn_idx = env->prev_insn_idx;
9416 if (!env->insn_aux_data[insn_idx].prune_point) {
9431 if (env->jmps_processed - env->prev_jmps_processed >= 2 && env->insn_processed - env->prev_insn_processed >= 8) {
9435 pprev = explored_state(env, insn_idx);
9438 clean_live_states(env, insn_idx, cur);
9446 if (states_maybe_looping(&sl->state, cur) && states_equal(env, &sl->state, cur)) {
9447 verbose_linfo(env, insn_idx, "; ");
9448 verbose(env, "infinite loop detected at insn %d\n", insn_idx);
9463 if (env->jmps_processed - env->prev_jmps_processed < 20 &&
9464 env->insn_processed - env->prev_insn_processed < 100) {
9469 if (states_equal(env, &sl->state, cur)) {
9474 * If we have any write marks in env->cur_state, they
9481 err = propagate_liveness(env, &sl->state, cur);
9488 err = err ?: push_jmp_history(env, cur);
9489 err = err ?: propagate_precision(env, &sl->state);
9521 env->peak_states--;
9527 sl->next = env->free_list;
9528 env->free_list = sl;
9538 if (env->max_states_per_insn < states_cnt) {
9539 env->max_states_per_insn = states_cnt;
9542 if (!env->bpf_capable && states_cnt > BPF_COMPLEXITY_LIMIT_STATES) {
9543 return push_jmp_history(env, cur);
9547 return push_jmp_history(env, cur);
9563 env->total_states++;
9564 env->peak_states++;
9565 env->prev_jmps_processed = env->jmps_processed;
9566 env->prev_insn_processed = env->insn_processed;
9582 new_sl->next = *explored_state(env, insn_idx);
9583 *explored_state(env, insn_idx) = new_sl;
9652 static int do_check(struct bpf_verifier_env *env)
9654 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
9655 struct bpf_verifier_state *state = env->cur_state;
9656 struct bpf_insn *insns = env->prog->insnsi;
9658 int insn_cnt = env->prog->len;
9667 env->prev_insn_idx = prev_insn_idx;
9668 if (env->insn_idx >= insn_cnt) {
9669 verbose(env, "invalid insn idx %d insn_cnt %d\n", env->insn_idx, insn_cnt);
9673 insn = &insns[env->insn_idx];
9676 if (++env->insn_processed > BPF_COMPLEXITY_LIMIT_INSNS) {
9677 verbose(env, "BPF program is too large. Processed %d insn\n", env->insn_processed);
9681 err = is_state_visited(env, env->insn_idx);
9687 if (env->log.level & BPF_LOG_LEVEL) {
9689 verbose(env, "\nfrom %d to %d%s: safe\n", env->prev_insn_idx, env->insn_idx,
9690 env->cur_state->speculative ? " (speculative execution)" : "");
9692 verbose(env, "%d: safe\n", env->insn_idx);
9706 if ((env->log.level & BPF_LOG_LEVEL2) || ((env->log.level & BPF_LOG_LEVEL) && do_print_state)) {
9707 if (env->log.level & BPF_LOG_LEVEL2) {
9708 verbose(env, "%d:", env->insn_idx);
9710 verbose(env, "\nfrom %d to %d%s:", env->prev_insn_idx, env->insn_idx,
9711 env->cur_state->speculative ? " (speculative execution)" : "");
9713 print_verifier_state(env, state->frame[state->curframe]);
9717 if (env->log.level & BPF_LOG_LEVEL) {
9720 .private_data = env,
9723 verbose_linfo(env, env->insn_idx, "; ");
9724 verbose(env, "%d: ", env->insn_idx);
9725 print_bpf_insn(&cbs, insn, env->allow_ptr_leaks);
9728 if (bpf_prog_is_dev_bound(env->prog->aux)) {
9729 err = bpf_prog_offload_verify_insn(env, env->insn_idx, env->prev_insn_idx);
9735 regs = cur_regs(env);
9736 sanitize_mark_insn_seen(env);
9737 prev_insn_idx = env->insn_idx;
9740 err = check_alu_op(env, insn);
9750 err = check_reg_arg(env, insn->src_reg, SRC_OP);
9755 err = check_reg_arg(env, insn->dst_reg, DST_OP_NO_MARK);
9765 err = check_mem_access(env, env->insn_idx, insn->src_reg, insn->off, BPF_SIZE(insn->code), BPF_READ,
9771 prev_src_type = &env->insn_aux_data[env->insn_idx].ptr_type;
9787 verbose(env, "same insn cannot be used with different pointers\n");
9793 err = check_xadd(env, env->insn_idx, insn);
9797 env->insn_idx++;
9802 err = check_reg_arg(env, insn->src_reg, SRC_OP);
9807 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
9815 err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRITE,
9821 prev_dst_type = &env->insn_aux_data[env->insn_idx].ptr_type;
9826 verbose(env, "same insn cannot be used with different pointers\n");
9831 verbose(env, "BPF_ST uses reserved fields\n");
9835 err = check_reg_arg(env, insn->dst_reg, SRC_OP);
9839 if (is_ctx_reg(env, insn->dst_reg)) {
9840 verbose(env, "BPF_ST stores into R%d %s is not allowed\n", insn->dst_reg,
9841 reg_type_str(env, reg_state(env, insn->dst_reg)->type));
9846 err = check_mem_access(env, env->insn_idx, insn->dst_reg, insn->off, BPF_SIZE(insn->code), BPF_WRITE, -1,
9853 env->jmps_processed++;
9858 verbose(env, "BPF_CALL uses reserved fields\n");
9862 if (env->cur_state->active_spin_lock &&
9864 verbose(env, "function calls are not allowed while holding a lock\n");
9868 err = check_func_call(env, insn, &env->insn_idx);
9870 err = check_helper_call(env, insn->imm, env->insn_idx);
9878 verbose(env, "BPF_JA uses reserved fields\n");
9881 env->insn_idx += insn->off + 1;
9886 verbose(env, "BPF_EXIT uses reserved fields\n");
9889 if (env->cur_state->active_spin_lock) {
9890 verbose(env, "bpf_spin_unlock is missing\n");
9895 err = prepare_func_exit(env, &env->insn_idx);
9903 err = check_reference_leak(env);
9908 err = check_return_code(env);
9913 update_branch_counts(env, env->cur_state);
9914 err = pop_stack(env, &prev_insn_idx, &env->insn_idx, pop_log);
9925 err = check_cond_jmp_op(env, insn, &env->insn_idx);
9933 err = check_ld_abs(env, insn);
9938 err = check_ld_imm(env, insn);
9942 env->insn_idx++;
9943 sanitize_mark_insn_seen(env);
9945 verbose(env, "invalid BPF_LD mode\n");
9949 verbose(env, "unknown insn class %d\n", class);
9952 env->insn_idx++;
9959 static int check_pseudo_btf_id(struct bpf_verifier_env *env, struct bpf_insn *insn, struct bpf_insn_aux_data *aux)
9972 verbose(env, "kernel is missing BTF, make sure CONFIG_DEBUG_INFO_BTF=y is specified in Kconfig.\n");
9977 verbose(env, "reserved field (insn[1].imm) is used in pseudo_btf_id ldimm64 insn.\n");
9983 verbose(env, "ldimm64 insn specifies invalid btf_id %d.\n", id);
9988 verbose(env, "pseudo btf_id %d in ldimm64 isn't KIND_VAR.\n", id);
9995 verbose(env, "ldimm64 failed to find the address for kernel symbol '%s'.\n", sym_name);
10028 verbose(env, "ldimm64 unable to resolve the size of type '%s': %ld\n", tname, PTR_ERR(ret));
10071 static int check_map_prog_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, struct bpf_prog *prog)
10094 verbose(env, "perf_event programs can only use preallocated hash map\n");
10098 verbose(env, "trace type programs can only use preallocated hash map\n");
10103 env,
10108 verbose(env, "tracing progs cannot use bpf_spin_lock yet\n");
10113 verbose(env, "offload device mismatch between prog and map\n");
10118 verbose(env, "bpf_struct_ops map cannot be used in prog\n");
10128 verbose(env, "Sleepable programs can only use preallocated hash maps\n");
10133 verbose(env, "Sleepable programs can only use array and hash maps\n");
10153 static int resolve_pseudo_ldimm64(struct bpf_verifier_env *env)
10155 struct bpf_insn *insn = env->prog->insnsi;
10156 int insn_cnt = env->prog->len;
10159 err = bpf_prog_calc_tag(env->prog);
10166 verbose(env, "BPF_LDX uses reserved fields\n");
10172 verbose(env, "BPF_STX uses reserved fields\n");
10184 verbose(env, "invalid bpf_ld_imm64 insn\n");
10194 aux = &env->insn_aux_data[i];
10195 err = check_pseudo_btf_id(env, insn, aux);
10207 verbose(env, "unrecognized bpf_ld_imm64 insn\n");
10214 verbose(env, "fd %d is not pointing to valid bpf_map\n", insn[0].imm);
10218 err = check_map_prog_compatibility(env, map, env->prog);
10224 aux = &env->insn_aux_data[i];
10231 verbose(env, "direct value offset of %u is not allowed\n", off);
10237 verbose(env, "no direct value access support for this map type\n");
10244 verbose(env, "invalid access to map value pointer, value_size=%u off=%u\n", map->value_size, off);
10257 for (j = 0; j < env->used_map_cnt; j++) {
10258 if (env->used_maps[j] == map) {
10265 if (env->used_map_cnt >= MAX_USED_MAPS) {
10277 aux->map_index = env->used_map_cnt;
10278 env->used_maps[env->used_map_cnt++] = map;
10280 if (bpf_map_is_cgroup_storage(map) && bpf_cgroup_storage_assign(env->prog->aux, map)) {
10281 verbose(env, "only one cgroup storage of each type is allowed\n");
10295 verbose(env, "unknown opcode %02x\n", insn->code);
10308 static void release_maps(struct bpf_verifier_env *env)
10310 __bpf_free_used_maps(env->prog->aux, env->used_maps, env->used_map_cnt);
10314 static void convert_pseudo_ld_imm64(struct bpf_verifier_env *env)
10316 struct bpf_insn *insn = env->prog->insnsi;
10317 int insn_cnt = env->prog->len;
10327 /* single env->prog->insni[off] instruction was replaced with the range
10331 static void adjust_insn_aux_data(struct bpf_verifier_env *env, struct bpf_insn_aux_data *new_data,
10334 struct bpf_insn_aux_data *old_data = env->insn_aux_data;
10344 old_data[off].zext_dst = insn_has_def32(env, insn + off + cnt - 1);
10356 new_data[i].zext_dst = insn_has_def32(env, insn + i);
10358 env->insn_aux_data = new_data;
10362 static void adjust_subprog_starts(struct bpf_verifier_env *env, u32 off, u32 len)
10370 for (i = 0; i <= env->subprog_cnt; i++) {
10371 if (env->subprog_info[i].start <= off) {
10374 env->subprog_info[i].start += len - 1;
10393 static struct bpf_prog *bpf_patch_insn_data(struct bpf_verifier_env *env, u32 off, const struct bpf_insn *patch,
10400 new_data = vzalloc(array_size(env->prog->len + len - 1, sizeof(struct bpf_insn_aux_data)));
10406 new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
10409 verbose(env, "insn %d cannot be patched due to 16-bit range\n", env->insn_aux_data[off].orig_idx);
10414 adjust_insn_aux_data(env, new_data, new_prog, off, len);
10415 adjust_subprog_starts(env, off, len);
10420 static int adjust_subprog_starts_after_remove(struct bpf_verifier_env *env, u32 off, u32 cnt)
10425 for (i = 0; i < env->subprog_cnt; i++) {
10426 if (env->subprog_info[i].start >= off) {
10431 for (j = i; j < env->subprog_cnt; j++) {
10432 if (env->subprog_info[j].start >= off + cnt) {
10439 if (env->subprog_info[j].start != off + cnt) {
10444 struct bpf_prog_aux *aux = env->prog->aux;
10448 move = env->subprog_cnt + 1 - j;
10450 memmove(env->subprog_info + i, env->subprog_info + j, sizeof(*env->subprog_info) * move);
10451 env->subprog_cnt -= j - i;
10465 if (env->subprog_info[i].start == off) {
10471 for (; i <= env->subprog_cnt; i++) {
10472 env->subprog_info[i].start -= cnt;
10478 static int bpf_adj_linfo_after_remove(struct bpf_verifier_env *env, u32 off, u32 cnt)
10480 struct bpf_prog *prog = env->prog;
10531 for (i = 0; i <= env->subprog_cnt; i++) {
10532 if (env->subprog_info[i].linfo_idx > l_off) {
10536 if (env->subprog_info[i].linfo_idx >= l_off + l_cnt) {
10537 env->subprog_info[i].linfo_idx -= l_cnt;
10539 env->subprog_info[i].linfo_idx = l_off;
10547 static int verifier_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
10549 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
10550 unsigned int orig_prog_len = env->prog->len;
10553 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10554 bpf_prog_offload_remove_insns(env, off, cnt);
10557 err = bpf_remove_insns(env->prog, off, cnt);
10562 err = adjust_subprog_starts_after_remove(env, off, cnt);
10567 err = bpf_adj_linfo_after_remove(env, off, cnt);
10588 static void sanitize_dead_code(struct bpf_verifier_env *env)
10590 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
10592 struct bpf_insn *insn = env->prog->insnsi;
10593 const int insn_cnt = env->prog->len;
10621 static void opt_hard_wire_dead_code_branches(struct bpf_verifier_env *env)
10623 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
10625 struct bpf_insn *insn = env->prog->insnsi;
10626 const int insn_cnt = env->prog->len;
10642 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10643 bpf_prog_offload_replace_insn(env, i, &ja);
10650 static int opt_remove_dead_code(struct bpf_verifier_env *env)
10652 struct bpf_insn_aux_data *aux_data = env->insn_aux_data;
10653 int insn_cnt = env->prog->len;
10667 err = verifier_remove_insns(env, i, j);
10671 insn_cnt = env->prog->len;
10677 static int opt_remove_nops(struct bpf_verifier_env *env)
10680 struct bpf_insn *insn = env->prog->insnsi;
10681 int insn_cnt = env->prog->len;
10689 err = verifier_remove_insns(env, i, 1);
10700 static int opt_subreg_zext_lo32_rnd_hi32(struct bpf_verifier_env *env, const union bpf_attr *attr)
10703 struct bpf_insn_aux_data *aux = env->insn_aux_data;
10704 int i, patch_len, delta = 0, len = env->prog->len;
10705 struct bpf_insn *insns = env->prog->insnsi;
10737 if (is_reg64(env, &insn, insn.dst_reg, NULL, DST_OP)) {
10768 new_prog = bpf_patch_insn_data(env, adj_idx, patch, patch_len);
10772 env->prog = new_prog;
10774 aux = env->insn_aux_data;
10786 static int convert_ctx_accesses(struct bpf_verifier_env *env)
10788 const struct bpf_verifier_ops *ops = env->ops;
10790 const int insn_cnt = env->prog->len;
10797 if (ops->gen_prologue || env->seen_direct_write) {
10799 verbose(env, "bpf verifier is misconfigured\n");
10802 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, env->prog);
10804 verbose(env, "bpf verifier is misconfigured\n");
10807 new_prog = bpf_patch_insn_data(env, 0, insn_buf, cnt);
10812 env->prog = new_prog;
10817 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10821 insn = env->prog->insnsi + delta;
10841 if (type == BPF_WRITE && env->insn_aux_data[i + delta].sanitize_stack_spill) {
10848 new_prog = bpf_patch_insn_data(env, i + delta, patch, cnt);
10854 env->prog = new_prog;
10863 switch (env->insn_aux_data[i + delta].ptr_type) {
10883 env->prog->aux->num_exentries++;
10884 } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) {
10885 verbose(env, "Writes through BTF pointers are not allowed\n");
10893 ctx_field_size = env->insn_aux_data[i + delta].ctx_field_size;
10908 verbose(env, "bpf verifier narrow ctx access misconfigured\n");
10924 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, &target_size);
10926 verbose(env, "bpf verifier is misconfigured\n");
10933 verbose(env, "bpf verifier narrow ctx load misconfigured\n");
10949 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
10957 env->prog = new_prog;
10964 static int jit_subprogs(struct bpf_verifier_env *env)
10966 struct bpf_prog *prog = env->prog, **func, *tmp;
10973 if (env->subprog_cnt <= 1) {
10985 subprog = find_subprog(env, i + insn->imm + 1);
10997 env->insn_aux_data[i].call_imm = insn->imm;
11008 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
11013 for (i = 0; i < env->subprog_cnt; i++) {
11015 subprog_end = env->subprog_info[i + 1].start;
11054 func[i]->aux->stack_depth = env->subprog_info[i].stack_depth;
11059 func[i]->aux->linfo_idx = env->subprog_info[i].linfo_idx;
11068 func[i]->aux->tail_call_reachable = env->subprog_info[i].tail_call_reachable;
11081 for (i = 0; i < env->subprog_cnt; i++) {
11103 func[i]->aux->func_cnt = env->subprog_cnt;
11105 for (i = 0; i < env->subprog_cnt; i++) {
11109 verbose(env, "JIT doesn't support bpf-to-bpf calls\n");
11119 for (i = 0; i < env->subprog_cnt; i++) {
11132 insn->off = env->insn_aux_data[i].call_imm;
11133 subprog = find_subprog(env, i + insn->off + 1);
11140 prog->aux->func_cnt = env->subprog_cnt;
11156 for (i = 0; i < env->subprog_cnt; i++) {
11172 insn->imm = env->insn_aux_data[i].call_imm;
11178 static int fixup_call_args(struct bpf_verifier_env *env)
11181 struct bpf_prog *prog = env->prog;
11187 if (env->prog->jit_requested && !bpf_prog_is_dev_bound(env->prog->aux)) {
11188 err = jit_subprogs(env);
11197 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
11201 verbose(env, "tail_calls are not allowed in non-JITed programs with bpf-to-bpf calls\n");
11208 depth = get_callee_stack_depth(env, insn, i);
11224 static int fixup_bpf_calls(struct bpf_verifier_env *env)
11226 struct bpf_prog *prog = env->prog;
11262 new_prog = bpf_patch_insn_data(env, i + delta, patchlet, cnt);
11268 env->prog = prog = new_prog;
11274 cnt = env->ops->gen_ld_abs(insn, insn_buf);
11276 verbose(env, "bpf verifier is misconfigured\n");
11280 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
11286 env->prog = prog = new_prog;
11299 aux = &env->insn_aux_data[i + delta];
11335 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf_in, cnt);
11341 env->prog = prog = new_prog;
11369 if (!allow_tail_call_in_subprogs(env)) {
11382 aux = &env->insn_aux_data[i + delta];
11383 if (env->bpf_capable && !expect_blinding && prog->jit_requested && !bpf_map_key_poisoned(aux) &&
11394 verbose(env, "adding tail call poke descriptor failed\n");
11413 verbose(env, "tail_call abusing map_ptr\n");
11422 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
11428 env->prog = prog = new_prog;
11441 aux = &env->insn_aux_data[i + delta];
11454 verbose(env, "bpf verifier is misconfigured\n");
11458 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
11464 env->prog = prog = new_prog;
11514 new_prog = bpf_patch_insn_data(env, i + delta, insn_buf, cnt);
11520 env->prog = prog = new_prog;
11526 fn = env->ops->get_func_proto(insn->imm, env->prog);
11531 verbose(env, "kernel subsystem misconfigured func %s#%d\n", func_id_name(insn->imm), insn->imm);
11541 verbose(env, "bpf verifier is misconfigured\n");
11547 verbose(env, "tracking tail call prog failed\n");
11555 static void free_states(struct bpf_verifier_env *env)
11560 sl = env->free_list;
11567 env->free_list = NULL;
11569 if (!env->explored_states) {
11573 for (i = 0; i < state_htab_size(env); i++) {
11574 sl = env->explored_states[i];
11582 env->explored_states[i] = NULL;
11586 static int do_check_common(struct bpf_verifier_env *env, int subprog)
11588 bool pop_log = !(env->log.level & BPF_LOG_LEVEL2);
11593 env->prev_linfo = NULL;
11594 env->pass_cnt++;
11608 env->cur_state = state;
11609 init_func_state(env, state->frame[0], BPF_MAIN_FUNC /* callsite */, 0 /* frameno */, subprog);
11612 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
11613 ret = btf_prepare_func_args(env, subprog, regs);
11619 mark_reg_known_zero(env, regs, i);
11621 mark_reg_unknown(env, regs, i);
11627 mark_reg_known_zero(env, regs, BPF_REG_1);
11628 ret = btf_check_func_arg_match(env, subprog, regs);
11643 ret = do_check(env);
11648 if (env->cur_state) {
11649 free_verifier_state(env->cur_state, true);
11650 env->cur_state = NULL;
11652 while (!pop_stack(env, NULL, NULL, false)) {
11656 bpf_vlog_reset(&env->log, 0);
11658 free_states(env);
11679 static int do_check_subprogs(struct bpf_verifier_env *env)
11681 struct bpf_prog_aux *aux = env->prog->aux;
11688 for (i = 1; i < env->subprog_cnt; i++) {
11692 env->insn_idx = env->subprog_info[i].start;
11693 WARN_ON_ONCE(env->insn_idx == 0);
11694 ret = do_check_common(env, i);
11697 } else if (env->log.level & BPF_LOG_LEVEL) {
11698 verbose(env, "Func#%d is safe for any args that match its prototype\n", i);
11704 static int do_check_main(struct bpf_verifier_env *env)
11708 env->insn_idx = 0;
11709 ret = do_check_common(env, 0);
11711 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
11716 static void print_verification_stats(struct bpf_verifier_env *env)
11720 if (env->log.level & BPF_LOG_STATS) {
11721 verbose(env, "verification time %lld usec\n", div_u64(env->verification_time, VERIFIER_ONETHOUSAND));
11722 verbose(env, "stack depth ");
11723 for (i = 0; i < env->subprog_cnt; i++) {
11724 u32 depth = env->subprog_info[i].stack_depth;
11726 verbose(env, "%d", depth);
11727 if (i + 1 < env->subprog_cnt) {
11728 verbose(env, "+");
11731 verbose(env, "\n");
11733 verbose(env,
11736 env->insn_processed, BPF_COMPLEXITY_LIMIT_INSNS, env->max_states_per_insn, env->total_states,
11737 env->peak_states, env->longest_mark_read_walk);
11740 static int check_struct_ops_btf_id(struct bpf_verifier_env *env)
11745 struct bpf_prog *prog = env->prog;
11750 verbose(env, "struct ops programs must have a GPL compatible license\n");
11757 verbose(env, "attach_btf_id %u is not a supported struct\n", btf_id);
11764 verbose(env, "attach to invalid member idx %u of struct %s\n", member_idx, st_ops->name);
11772 verbose(env, "attach to invalid member %s(@idx %u) of struct %s\n", mname, member_idx, st_ops->name);
11779 verbose(env, "attach to unsupported member %s of struct %s\n", mname, st_ops->name);
11786 env->ops = st_ops->verifier_ops;
12062 static int check_attach_btf_id(struct bpf_verifier_env *env)
12064 struct bpf_prog *prog = env->prog;
12073 verbose(env, "Only fentry/fexit/fmod_ret and lsm programs can be sleepable\n");
12078 return check_struct_ops_btf_id(env);
12085 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
12092 * inherit env->ops and expected_attach_type for the rest of the
12095 env->ops = bpf_verifier_ops[tgt_prog->type];
12119 ret = bpf_lsm_verify_prog(&env->log, prog);
12150 struct bpf_verifier_env *env;
12163 env = kzalloc(sizeof(struct bpf_verifier_env), GFP_KERNEL);
12164 if (!env) {
12167 log = &env->log;
12170 env->insn_aux_data = vzalloc(array_size(sizeof(struct bpf_insn_aux_data), len));
12172 if (!env->insn_aux_data) {
12176 env->insn_aux_data[i].orig_idx = i;
12178 env->prog = *prog;
12179 env->ops = bpf_verifier_ops[env->prog->type];
12206 verbose(env, "in-kernel BTF is malformed\n");
12211 env->strict_alignment = !!(attr->prog_flags & BPF_F_STRICT_ALIGNMENT);
12213 env->strict_alignment = true;
12216 env->strict_alignment = false;
12219 env->allow_ptr_leaks = bpf_allow_ptr_leaks();
12220 env->allow_uninit_stack = bpf_allow_uninit_stack();
12221 env->allow_ptr_to_map_access = bpf_allow_ptr_to_map_access();
12222 env->bypass_spec_v1 = bpf_bypass_spec_v1();
12223 env->bypass_spec_v4 = bpf_bypass_spec_v4();
12224 env->bpf_capable = bpf_capable();
12227 env->test_state_freq = attr->prog_flags & BPF_F_TEST_STATE_FREQ;
12230 env->explored_states = kvcalloc(state_htab_size(env), sizeof(struct bpf_verifier_state_list *), GFP_USER);
12232 if (!env->explored_states) {
12236 ret = check_subprogs(env);
12241 ret = check_btf_info(env, attr, uattr);
12246 ret = check_attach_btf_id(env);
12251 ret = resolve_pseudo_ldimm64(env);
12256 if (bpf_prog_is_dev_bound(env->prog->aux)) {
12257 ret = bpf_prog_offload_verifier_prep(env->prog);
12263 ret = check_cfg(env);
12268 ret = do_check_subprogs(env);
12269 ret = ret ?: do_check_main(env);
12271 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) {
12272 ret = bpf_prog_offload_finalize(env);
12276 kvfree(env->explored_states);
12279 ret = check_max_stack_depth(env);
12285 opt_hard_wire_dead_code_branches(env);
12288 ret = opt_remove_dead_code(env);
12291 ret = opt_remove_nops(env);
12295 sanitize_dead_code(env);
12301 ret = convert_ctx_accesses(env);
12305 ret = fixup_bpf_calls(env);
12311 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) {
12312 ret = opt_subreg_zext_lo32_rnd_hi32(env, attr);
12313 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret : false;
12317 ret = fixup_call_args(env);
12320 env->verification_time = ktime_get_ns() - start_time;
12321 print_verification_stats(env);
12331 if (ret == 0 && env->used_map_cnt) {
12333 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, sizeof(env->used_maps[0]), GFP_KERNEL);
12335 if (!env->prog->aux->used_maps) {
12340 memcpy(env->prog->aux->used_maps, env->used_maps, sizeof(env->used_maps[0]) * env->used_map_cnt);
12341 env->prog->aux->used_map_cnt = env->used_map_cnt;
12346 convert_pseudo_ld_imm64(env);
12350 adjust_btf_func(env);
12354 if (!env->prog->aux->used_maps) {
12358 release_maps(env);
12364 if (env->prog->type == BPF_PROG_TYPE_EXT) {
12365 env->prog->expected_attach_type = 0;
12368 *prog = env->prog;
12373 vfree(env->insn_aux_data);
12375 kfree(env);