Lines Matching refs:prog
260 const struct bpf_prog *prog;
263 prog = env->prog;
264 nr_linfo = prog->aux->nr_linfo;
266 if (!nr_linfo || insn_off >= prog->len) {
270 linfo = prog->aux->linfo;
391 verbose(env, "%s\n", ltrim(btf_name_by_offset(env->prog->aux->btf, linfo->line_off)));
1429 int insn_cnt = env->prog->len;
1453 struct bpf_insn *insn = env->prog->insnsi;
1454 int insn_cnt = env->prog->len;
1715 struct bpf_insn *insn = env->prog->insnsi + env->insn_idx;
1801 struct bpf_insn *insn = env->prog->insnsi + idx;
2117 if (i >= env->prog->len) {
2292 u32 dst_reg = env->prog->insnsi[insn_idx].dst_reg;
2893 static enum bpf_prog_type resolve_prog_type(struct bpf_prog *prog)
2895 return prog->aux->dst_prog ? prog->aux->dst_prog->type : prog->type;
2901 enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
2972 env->prog->aux->max_pkt_offset = max_t(u32, env->prog->aux->max_pkt_offset, off + reg->umax_value + size - 1);
2986 if (env->ops->is_valid_access && env->ops->is_valid_access(off, size, t, env->prog, &info)) {
3002 if (env->prog->aux->max_ctx_offset < off + size) {
3003 env->prog->aux->max_ctx_offset = off + size;
3221 struct bpf_insn *insn = env->prog->insnsi;
3317 env->prog->aux->tail_call_reachable = true;
3406 if (off + size > env->prog->aux->max_tp_access) {
3407 env->prog->aux->max_tp_access = off + size;
3873 max_access = &env->prog->aux->max_rdonly_access;
3876 max_access = &env->prog->aux->max_rdwr_access;
4099 max_access = &env->prog->aux->max_rdonly_access;
4102 max_access = &env->prog->aux->max_rdwr_access;
4621 enum bpf_attach_type eatype = env->prog->expected_attach_type;
4622 enum bpf_prog_type type = resolve_prog_type(env->prog);
4655 return env->prog->jit_requested && IS_ENABLED(CONFIG_X86_64);
5088 func_info_aux = env->prog->aux->func_info_aux;
5128 subprog /* subprog number within this prog */);
5323 fn = env->ops->get_func_proto(func_id, env->prog);
5331 if (!env->prog->gpl_compatible && fn->gpl_only) {
5336 if (fn->allowed && !fn->allowed(env->prog)) {
5526 if ((func_id == BPF_FUNC_get_stack || func_id == BPF_FUNC_get_task_stack) && !env->prog->has_callchain_buf) {
5541 env->prog->has_callchain_buf = true;
5545 env->prog->call_get_stack = true;
8089 if (!may_access_skb(resolve_prog_type(env->prog))) {
8163 const struct bpf_prog *prog = env->prog;
8166 enum bpf_prog_type prog_type = resolve_prog_type(env->prog);
8172 !prog->aux->attach_func_proto->type) {
8204 if (env->prog->expected_attach_type == BPF_CGROUP_UDP4_RECVMSG ||
8205 env->prog->expected_attach_type == BPF_CGROUP_UDP6_RECVMSG ||
8206 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETPEERNAME ||
8207 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETPEERNAME ||
8208 env->prog->expected_attach_type == BPF_CGROUP_INET4_GETSOCKNAME ||
8209 env->prog->expected_attach_type == BPF_CGROUP_INET6_GETSOCKNAME) {
8214 if (env->prog->expected_attach_type == BPF_CGROUP_INET_EGRESS) {
8226 if (!env->prog->aux->attach_btf_id) {
8232 switch (env->prog->expected_attach_type) {
8278 env->prog->enforce_expected_attach_type = 1;
8325 return env->prog->len;
8359 if (w < 0 || w >= env->prog->len) {
8374 if (env->cfg.cur_stack >= env->prog->len) {
8402 struct bpf_insn *insns = env->prog->insnsi;
8403 int insn_cnt = env->prog->len;
8559 struct bpf_prog *prog;
8585 prog = env->prog;
8586 btf = prog->aux->btf;
8666 prog->aux->func_info = krecord;
8667 prog->aux->func_info_cnt = nfuncs;
8668 prog->aux->func_info_aux = info_aux;
8679 struct bpf_prog_aux *aux = env->prog->aux;
8700 struct bpf_prog *prog;
8726 prog = env->prog;
8727 btf = prog->aux->btf;
8754 * 2) bounded by prog->len
8762 if ((i && linfo[i].insn_off <= prev_offset) || linfo[i].insn_off >= prog->len) {
8763 verbose(env, "Invalid line_info[%u].insn_off:%u (prev_offset:%u prog->len:%u)\n", i, linfo[i].insn_off,
8764 prev_offset, prog->len);
8769 if (!prog->insnsi[linfo[i].insn_off].code) {
8802 prog->aux->linfo = linfo;
8803 prog->aux->nr_linfo = nr_linfo;
8828 env->prog->aux->btf = btf;
9056 * the rest of the prog is valid for one map element then
9656 struct bpf_insn *insns = env->prog->insnsi;
9658 int insn_cnt = env->prog->len;
9728 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10071 static int check_map_prog_compatibility(struct bpf_verifier_env *env, struct bpf_map *map, struct bpf_prog *prog)
10074 enum bpf_prog_type prog_type = resolve_prog_type(prog);
10112 if ((bpf_prog_is_dev_bound(prog->aux) || bpf_map_is_dev_bound(map)) && !bpf_offload_prog_map_match(prog, map)) {
10113 verbose(env, "offload device mismatch between prog and map\n");
10118 verbose(env, "bpf_struct_ops map cannot be used in prog\n");
10122 if (prog->aux->sleepable) {
10155 struct bpf_insn *insn = env->prog->insnsi;
10156 int insn_cnt = env->prog->len;
10159 err = bpf_prog_calc_tag(env->prog);
10218 err = check_map_prog_compatibility(env, map, env->prog);
10280 if (bpf_map_is_cgroup_storage(map) && bpf_cgroup_storage_assign(env->prog->aux, map)) {
10310 __bpf_free_used_maps(env->prog->aux, env->used_maps, env->used_map_cnt);
10316 struct bpf_insn *insn = env->prog->insnsi;
10317 int insn_cnt = env->prog->len;
10327 /* single env->prog->insni[off] instruction was replaced with the range
10342 * original insn at old prog.
10378 static void adjust_poke_descs(struct bpf_prog *prog, u32 off, u32 len)
10380 struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;
10381 int i, sz = prog->aux->size_poke_tab;
10400 new_data = vzalloc(array_size(env->prog->len + len - 1, sizeof(struct bpf_insn_aux_data)));
10406 new_prog = bpf_patch_insn_single(env->prog, off, patch, len);
10424 /* find first prog starting at or after off (first to remove) */
10430 /* find first prog starting at or after off + cnt (first to stay) */
10437 * the front of previous prog
10444 struct bpf_prog_aux *aux = env->prog->aux;
10464 /* convert i from "first prog to remove" to "first to adjust" */
10480 struct bpf_prog *prog = env->prog;
10484 nr_linfo = prog->aux->nr_linfo;
10489 linfo = prog->aux->linfo;
10509 * last removed linfo. prog is already modified, so prog->len == off
10512 if (prog->len != off && l_cnt && (i == nr_linfo || linfo[i].insn_off != off + cnt)) {
10521 prog->aux->nr_linfo -= l_cnt;
10522 nr_linfo = prog->aux->nr_linfo;
10550 unsigned int orig_prog_len = env->prog->len;
10553 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10557 err = bpf_remove_insns(env->prog, off, cnt);
10592 struct bpf_insn *insn = env->prog->insnsi;
10593 const int insn_cnt = env->prog->len;
10625 struct bpf_insn *insn = env->prog->insnsi;
10626 const int insn_cnt = env->prog->len;
10642 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10653 int insn_cnt = env->prog->len;
10671 insn_cnt = env->prog->len;
10680 struct bpf_insn *insn = env->prog->insnsi;
10681 int insn_cnt = env->prog->len;
10704 int i, patch_len, delta = 0, len = env->prog->len;
10705 struct bpf_insn *insns = env->prog->insnsi;
10772 env->prog = new_prog;
10790 const int insn_cnt = env->prog->len;
10802 cnt = ops->gen_prologue(insn_buf, env->seen_direct_write, env->prog);
10812 env->prog = new_prog;
10817 if (bpf_prog_is_dev_bound(env->prog->aux)) {
10821 insn = env->prog->insnsi + delta;
10854 env->prog = new_prog;
10883 env->prog->aux->num_exentries++;
10884 } else if (resolve_prog_type(env->prog) != BPF_PROG_TYPE_STRUCT_OPS) {
10924 cnt = convert_ctx_access(type, insn, insn_buf, env->prog, &target_size);
10957 env->prog = new_prog;
10966 struct bpf_prog *prog = env->prog, **func, *tmp;
10977 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11002 err = bpf_prog_alloc_jited_linfo(prog);
11008 func = kcalloc(env->subprog_cnt, sizeof(prog), GFP_KERNEL);
11019 * hence main prog stats include the runtime of subprogs.
11027 memcpy(func[i]->insnsi, &prog->insnsi[subprog_start], len * sizeof(struct bpf_insn));
11028 func[i]->type = prog->type;
11035 /* Below members will be freed only at prog->aux */
11036 func[i]->aux->btf = prog->aux->btf;
11037 func[i]->aux->func_info = prog->aux->func_info;
11038 func[i]->aux->poke_tab = prog->aux->poke_tab;
11039 func[i]->aux->size_poke_tab = prog->aux->size_poke_tab;
11041 for (j = 0; j < prog->aux->size_poke_tab; j++) {
11044 poke = &prog->aux->poke_tab[j];
11056 func[i]->aux->linfo = prog->aux->linfo;
11057 func[i]->aux->nr_linfo = prog->aux->nr_linfo;
11058 func[i]->aux->jited_linfo = prog->aux->jited_linfo;
11116 /* finally lock prog and jit images for all functions and
11125 * prog consistent for later dump requests, so they can
11128 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11137 prog->jited = 1;
11138 prog->bpf_func = func[0]->bpf_func;
11139 prog->aux->func = func;
11140 prog->aux->func_cnt = env->subprog_cnt;
11141 bpf_prog_free_unused_jited_linfo(prog);
11148 for (i = 0; i < prog->aux->size_poke_tab; i++) {
11149 map_ptr = prog->aux->poke_tab[i].tail_call.map;
11150 map_ptr->ops->map_poke_untrack(map_ptr, prog->aux);
11154 * released with the main prog.
11165 /* cleanup main prog to be interpreted */
11166 prog->jit_requested = 0;
11167 for (i = 0, insn = prog->insnsi; i < prog->len; i++, insn++) {
11174 bpf_prog_free_jited_linfo(prog);
11181 struct bpf_prog *prog = env->prog;
11182 struct bpf_insn *insn = prog->insnsi;
11187 if (env->prog->jit_requested && !bpf_prog_is_dev_bound(env->prog->aux)) {
11197 if (env->subprog_cnt > 1 && env->prog->aux->tail_call_reachable) {
11204 for (i = 0; i < prog->len; i++, insn++) {
11226 struct bpf_prog *prog = env->prog;
11227 bool expect_blinding = bpf_jit_blinding_enabled(prog);
11228 struct bpf_insn *insn = prog->insnsi;
11230 const int insn_cnt = prog->len;
11268 env->prog = prog = new_prog;
11286 env->prog = prog = new_prog;
11341 env->prog = prog = new_prog;
11354 prog->dst_needed = 1;
11360 prog->kprobe_override = 1;
11368 prog->cb_access = 1;
11370 prog->aux->stack_depth = MAX_BPF_STACK;
11372 prog->aux->max_pkt_offset = MAX_PACKET_OFF;
11383 if (env->bpf_capable && !expect_blinding && prog->jit_requested && !bpf_map_key_poisoned(aux) &&
11392 ret = bpf_jit_add_poke_descriptor(prog, &desc);
11428 env->prog = prog = new_prog;
11437 if (prog->jit_requested && BITS_PER_LONG == VERIFIER_SIXTYFOUR &&
11464 env->prog = prog = new_prog;
11504 if (prog->jit_requested && BITS_PER_LONG == VERIFIER_SIXTYFOUR && insn->imm == BPF_FUNC_jiffies64) {
11520 env->prog = prog = new_prog;
11526 fn = env->ops->get_func_proto(insn->imm, env->prog);
11538 for (i = 0; i < prog->aux->size_poke_tab; i++) {
11539 map_ptr = prog->aux->poke_tab[i].tail_call.map;
11545 ret = map_ptr->ops->map_poke_track(map_ptr, prog->aux);
11547 verbose(env, "tracking tail call prog failed\n");
11612 if (subprog || env->prog->type == BPF_PROG_TYPE_EXT) {
11681 struct bpf_prog_aux *aux = env->prog->aux;
11711 env->prog->aux->stack_depth = env->subprog_info[0].stack_depth;
11745 struct bpf_prog *prog = env->prog;
11749 if (!prog->gpl_compatible) {
11754 btf_id = prog->aux->attach_btf_id;
11762 member_idx = prog->expected_attach_type;
11784 prog->aux->attach_func_proto = func_proto;
11785 prog->aux->attach_func_name = mname;
11831 int bpf_check_attach_target(struct bpf_verifier_log *log, const struct bpf_prog *prog, const struct bpf_prog *tgt_prog,
11834 bool prog_extension = prog->type == BPF_PROG_TYPE_EXT;
11881 if (!prog->jit_requested) {
11890 if (tgt_prog->type == prog->type) {
11925 switch (prog->expected_attach_type) {
11928 bpf_log(log, "Only FENTRY/FEXIT progs are attachable to another BPF prog\n");
11979 if (prog_extension && btf_check_type_match(log, prog, btf, t)) {
11987 if ((prog->aux->saved_dst_prog_type || prog->aux->saved_dst_attach_type) &&
11988 (!tgt_prog || prog->aux->saved_dst_prog_type != tgt_prog->type ||
11989 prog->aux->saved_dst_attach_type != tgt_prog->expected_attach_type)) {
12016 if (prog->aux->sleepable) {
12018 switch (prog->type) {
12042 } else if (prog->expected_attach_type == BPF_MODIFY_RETURN) {
12064 struct bpf_prog *prog = env->prog;
12065 struct bpf_prog *tgt_prog = prog->aux->dst_prog;
12067 u32 btf_id = prog->aux->attach_btf_id;
12072 if (prog->aux->sleepable && prog->type != BPF_PROG_TYPE_TRACING && prog->type != BPF_PROG_TYPE_LSM) {
12077 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS) {
12081 if (prog->type != BPF_PROG_TYPE_TRACING && prog->type != BPF_PROG_TYPE_LSM && prog->type != BPF_PROG_TYPE_EXT) {
12085 ret = bpf_check_attach_target(&env->log, prog, tgt_prog, btf_id, &tgt_info);
12090 if (tgt_prog && prog->type == BPF_PROG_TYPE_EXT) {
12096 prog->expected_attach_type = tgt_prog->expected_attach_type;
12100 prog->aux->attach_func_proto = tgt_info.tgt_type;
12101 prog->aux->attach_func_name = tgt_info.tgt_name;
12104 prog->aux->saved_dst_prog_type = tgt_prog->type;
12105 prog->aux->saved_dst_attach_type = tgt_prog->expected_attach_type;
12108 if (prog->expected_attach_type == BPF_TRACE_RAW_TP) {
12109 prog->aux->attach_btf_trace = true;
12111 } else if (prog->expected_attach_type == BPF_TRACE_ITER) {
12112 if (!bpf_iter_prog_supported(prog)) {
12118 if (prog->type == BPF_PROG_TYPE_LSM) {
12119 ret = bpf_lsm_verify_prog(&env->log, prog);
12131 prog->aux->dst_trampoline = tr;
12147 int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, union bpf_attr __user *uattr)
12169 len = (*prog)->len;
12178 env->prog = *prog;
12179 env->ops = bpf_verifier_ops[env->prog->type];
12256 if (bpf_prog_is_dev_bound(env->prog->aux)) {
12257 ret = bpf_prog_offload_verifier_prep(env->prog);
12271 if (ret == 0 && bpf_prog_is_dev_bound(env->prog->aux)) {
12311 if (ret == 0 && !bpf_prog_is_dev_bound(env->prog->aux)) {
12313 env->prog->aux->verifier_zext = bpf_jit_needs_zext() ? !ret : false;
12333 env->prog->aux->used_maps = kmalloc_array(env->used_map_cnt, sizeof(env->used_maps[0]), GFP_KERNEL);
12335 if (!env->prog->aux->used_maps) {
12340 memcpy(env->prog->aux->used_maps, env->used_maps, sizeof(env->used_maps[0]) * env->used_map_cnt);
12341 env->prog->aux->used_map_cnt = env->used_map_cnt;
12354 if (!env->prog->aux->used_maps) {
12364 if (env->prog->type == BPF_PROG_TYPE_EXT) {
12365 env->prog->expected_attach_type = 0;
12368 *prog = env->prog;