Lines Matching defs:prog
76 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
742 void bpf_program__unload(struct bpf_program *prog)
744 if (!prog)
747 zclose(prog->fd);
749 zfree(&prog->func_info);
750 zfree(&prog->line_info);
753 static void bpf_program__exit(struct bpf_program *prog)
755 if (!prog)
758 bpf_program__unload(prog);
759 zfree(&prog->name);
760 zfree(&prog->sec_name);
761 zfree(&prog->insns);
762 zfree(&prog->reloc_desc);
764 prog->nr_reloc = 0;
765 prog->insns_cnt = 0;
766 prog->sec_idx = -1;
790 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
800 memset(prog, 0, sizeof(*prog));
801 prog->obj = obj;
803 prog->sec_idx = sec_idx;
804 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
805 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
807 prog->insns_cnt = prog->sec_insn_cnt;
809 prog->type = BPF_PROG_TYPE_UNSPEC;
810 prog->fd = -1;
811 prog->exception_cb_idx = -1;
818 prog->autoload = false;
822 prog->autoload = true;
825 prog->autoattach = true;
828 prog->log_level = obj->log_level;
830 prog->sec_name = strdup(sec_name);
831 if (!prog->sec_name)
834 prog->name = strdup(name);
835 if (!prog->name)
838 prog->insns = malloc(insn_data_sz);
839 if (!prog->insns)
841 memcpy(prog->insns, insn_data, insn_data_sz);
845 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
846 bpf_program__exit(prog);
855 struct bpf_program *prog, *progs;
911 prog = &progs[nr_progs];
913 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
919 prog->sym_global = true;
926 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN
928 prog->mark_btf_static = true;
1115 struct bpf_program *prog;
1117 prog = st_ops->progs[i];
1118 if (!prog)
1135 prog->attach_btf_id = kern_type_id;
1136 prog->expected_attach_type = kern_member_idx;
1140 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1141 map->name, mname, prog->name, moff,
2414 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2429 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2459 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
3152 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
3154 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3155 prog->type == BPF_PROG_TYPE_LSM)
3161 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3169 struct bpf_program *prog;
3187 bpf_object__for_each_program(prog, obj) {
3188 if (!prog->autoload)
3190 if (prog_needs_vmlinux_btf(prog))
3245 struct bpf_program *prog = &obj->programs[i];
3250 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3260 if (strcmp(name, prog->name) != 0)
3271 struct bpf_program *prog = &obj->programs[i];
3274 if (prog_is_subprog(obj, prog))
3292 pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n",
3293 prog->name);
3296 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)))
3298 /* Multiple callbacks are specified for the same prog,
3302 if (prog->exception_cb_idx >= 0) {
3303 prog->exception_cb_idx = -1;
3309 pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n",
3310 prog->name);
3326 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",
3327 prog->name, subprog->name);
3331 if (prog->exception_cb_idx >= 0) {
3332 pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n",
3333 prog->name, subprog->name);
3336 prog->exception_cb_idx = k;
3340 if (prog->exception_cb_idx >= 0)
3342 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);
4375 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
4377 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
4384 struct bpf_program *prog;
4386 bpf_object__for_each_program(prog, obj) {
4387 if (prog_is_subprog(obj, prog))
4389 if (!strcmp(prog->name, name))
4390 return prog;
4432 static int bpf_program__record_reloc(struct bpf_program *prog,
4437 struct bpf_insn *insn = &prog->insns[insn_idx];
4438 size_t map_idx, nr_maps = prog->obj->nr_maps;
4439 struct bpf_object *obj = prog->obj;
4446 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
4447 prog->name, sym_name, insn_idx, insn->code);
4462 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
4463 prog->name, sym_name, sym_idx);
4466 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
4467 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4480 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4490 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
4491 prog->name, sym_name, sym_sec_name);
4495 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4496 prog->name, sym_name, (size_t)sym->st_value);
4506 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4507 prog->name, sym_name, shdr_idx);
4517 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4518 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4537 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4538 prog->name, sym_name, sym_sec_name);
4547 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4548 prog->name, map_idx, map->name, map->sec_idx,
4553 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4554 prog->name, sym_sec_name, (size_t)sym->st_value);
4566 pr_warn("prog '%s': bad data relo against section '%s'\n",
4567 prog->name, sym_sec_name);
4574 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4575 prog->name, map_idx, map->name, map->sec_idx,
4580 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4581 prog->name, sym_sec_name);
4592 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4594 return insn_idx >= prog->sec_insn_off &&
4595 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4602 struct bpf_program *prog;
4609 prog = &obj->programs[m];
4611 if (prog->sec_idx < sec_idx ||
4612 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4620 prog = &obj->programs[l];
4621 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4622 return prog;
4631 struct bpf_program *prog;
4715 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4716 if (!prog) {
4722 relos = libbpf_reallocarray(prog->reloc_desc,
4723 prog->nr_reloc + 1, sizeof(*relos));
4726 prog->reloc_desc = relos;
4729 insn_idx -= prog->sec_insn_off;
4730 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4735 prog->nr_reloc++;
5162 int ret, map, prog, insn_cnt = ARRAY_SIZE(insns);
5173 prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
5174 if (prog < 0) {
5179 ret = bpf_prog_bind_map(prog, map, NULL);
5182 close(prog);
5664 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",
5668 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
5738 * runtime due to bpf_program__set_autoload(prog, false),
6108 static int record_relo_core(struct bpf_program *prog,
6113 relos = libbpf_reallocarray(prog->reloc_desc,
6114 prog->nr_reloc + 1, sizeof(*relos));
6117 relo = &relos[prog->nr_reloc];
6121 prog->reloc_desc = relos;
6122 prog->nr_reloc++;
6126 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)
6131 for (i = 0; i < prog->nr_reloc; i++) {
6132 relo = &prog->reloc_desc[i];
6142 static int bpf_core_resolve_relo(struct bpf_program *prog,
6151 const char *prog_name = prog->name;
6167 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
6169 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
6194 struct bpf_program *prog;
6237 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
6238 if (!prog) {
6254 if (!prog->autoload)
6261 insn_idx = insn_idx - prog->sec_insn_off;
6262 if (insn_idx >= prog->insns_cnt)
6264 insn = &prog->insns[insn_idx];
6266 err = record_relo_core(prog, rec, insn_idx);
6268 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",
6269 prog->name, i, err);
6273 if (prog->obj->gen_loader)
6276 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
6278 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
6279 prog->name, i, err);
6283 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
6285 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
6286 prog->name, i, insn_idx, err);
6310 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
6316 pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
6317 prog->name, relo_idx, insn_idx, map_idx, map->name);
6340 static void poison_kfunc_call(struct bpf_program *prog, int relo_idx,
6344 pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n",
6345 prog->name, relo_idx, insn_idx, ext->name);
6366 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
6370 for (i = 0; i < prog->nr_reloc; i++) {
6371 struct reloc_desc *relo = &prog->reloc_desc[i];
6372 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6386 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6400 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6433 poison_kfunc_call(prog, i, relo->insn_idx, insn,
6439 pr_warn("prog '%s': relo #%d: bad insn\n",
6440 prog->name, i);
6452 pr_warn("prog '%s': relo #%d: bad relo type %d\n",
6453 prog->name, i, relo->type);
6462 const struct bpf_program *prog,
6477 if (prog->sec_idx != sec_idx)
6483 if (insn_off < prog->sec_insn_off)
6485 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6514 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6532 const struct bpf_program *prog)
6545 if (main_prog != prog && !main_prog->func_info)
6548 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6554 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6555 prog->name, err);
6563 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6567 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6568 prog->name);
6573 if (main_prog != prog && !main_prog->line_info)
6576 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6582 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6583 prog->name, err);
6591 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6595 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6596 prog->name);
6611 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6613 if (!prog->nr_reloc)
6615 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6616 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6661 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6670 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6682 struct bpf_program *prog)
6690 err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6694 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6695 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6699 relo = find_prog_insn_relo(prog, insn_idx);
6706 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6707 prog->name, insn_idx, relo->type);
6730 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6731 prog->name, insn_idx);
6740 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6746 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6747 prog->name);
6773 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6775 * prog; each main prog can have a different set of
6780 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6782 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6783 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6793 * main prog) is processed separately. For each subprog (non-entry functions,
6796 * hasn't been yet appended and relocated within current main prog. Once its
6797 * relocated, sub_insn_off will point at the position within current main prog
6807 * the *main* prog's instructions (subprog's instructions are always left
6809 * and subprog instructions are always sent only as part of a main prog) and
6811 * prog or any of its subprogs are appended (and relocated), all their
6817 * main prog and any of its called/used subprogs. Those will keep their
6819 * main prog and won't be relocated within the context of current main prog.
6871 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6887 err = bpf_object__reloc_code(obj, prog, prog);
6897 struct bpf_program *prog;
6902 prog = &obj->programs[i];
6903 zfree(&prog->reloc_desc);
6904 prog->nr_reloc = 0;
6940 struct bpf_program *prog;
6962 prog = &obj->programs[i];
6963 for (j = 0; j < prog->nr_reloc; j++) {
6964 struct reloc_desc *relo = &prog->reloc_desc[j];
6965 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6981 prog = &obj->programs[i];
6985 if (prog_is_subprog(obj, prog))
6987 if (!prog->autoload)
6990 err = bpf_object__relocate_calls(obj, prog);
6992 pr_warn("prog '%s': failed to relocate calls: %d\n",
6993 prog->name, err);
6998 if (prog->exception_cb_idx >= 0) {
6999 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];
7007 err = bpf_object__append_subprog_code(obj, prog, subprog);
7010 err = bpf_object__reloc_code(obj, prog, subprog);
7018 prog = &obj->programs[i];
7019 if (prog_is_subprog(obj, prog))
7021 if (!prog->autoload)
7023 err = bpf_object__relocate_data(obj, prog);
7025 pr_warn("prog '%s': failed to relocate data references: %d\n",
7026 prog->name, err);
7098 type = is_map_in_map ? "map" : "prog";
7228 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
7230 struct bpf_insn *insn = prog->insns;
7237 for (i = 0; i < prog->insns_cnt; i++, insn++) {
7263 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
7266 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7267 static int libbpf_prepare_prog_load(struct bpf_program *prog,
7273 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
7279 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
7283 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))
7284 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
7286 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
7290 attach_name = strchr(prog->sec_name, '/');
7301 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
7302 prog->name);
7307 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
7311 /* cache resolved BTF FD and BTF type ID in the prog */
7312 prog->attach_btf_obj_fd = btf_obj_fd;
7313 prog->attach_btf_id = btf_type_id;
7316 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
7326 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
7328 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
7339 __u32 log_level = prog->log_level;
7341 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
7346 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
7347 prog->name, prog->sec_name);
7355 prog_name = prog->name;
7356 load_attr.attach_prog_fd = prog->attach_prog_fd;
7357 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
7358 load_attr.attach_btf_id = prog->attach_btf_id;
7360 load_attr.prog_ifindex = prog->prog_ifindex;
7366 load_attr.func_info = prog->func_info;
7367 load_attr.func_info_rec_size = prog->func_info_rec_size;
7368 load_attr.func_info_cnt = prog->func_info_cnt;
7369 load_attr.line_info = prog->line_info;
7370 load_attr.line_info_rec_size = prog->line_info_rec_size;
7371 load_attr.line_info_cnt = prog->line_info_cnt;
7374 load_attr.prog_flags = prog->prog_flags;
7378 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
7379 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
7381 pr_warn("prog '%s': failed to prepare load attributes: %d\n",
7382 prog->name, err);
7385 insns = prog->insns;
7386 insns_cnt = prog->insns_cnt;
7390 load_attr.expected_attach_type = prog->expected_attach_type;
7393 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
7395 prog - obj->programs);
7407 if (prog->log_buf) {
7408 log_buf = prog->log_buf;
7409 log_buf_size = prog->log_size;
7432 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
7435 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7436 prog->name, log_buf);
7444 map = &prog->obj->maps[i];
7450 pr_warn("prog '%s': failed to bind map '%s': %s\n",
7451 prog->name, map->real_name, cp);
7479 fixup_verifier_log(prog, log_buf, log_buf_size);
7482 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
7486 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7487 prog->name, log_buf);
7547 static void fixup_log_failed_core_relo(struct bpf_program *prog,
7569 relo = find_relo_core(prog, insn_idx);
7573 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
7586 static void fixup_log_missing_map_load(struct bpf_program *prog,
7598 struct bpf_object *obj = prog->obj;
7619 static void fixup_log_missing_kfunc_call(struct bpf_program *prog,
7631 struct bpf_object *obj = prog->obj;
7652 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)
7677 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
7686 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
7695 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz,
7702 static int bpf_program_record_relos(struct bpf_program *prog)
7704 struct bpf_object *obj = prog->obj;
7707 for (i = 0; i < prog->nr_reloc; i++) {
7708 struct reloc_desc *relo = &prog->reloc_desc[i];
7748 struct bpf_program *prog;
7753 prog = &obj->programs[i];
7754 err = bpf_object__sanitize_prog(obj, prog);
7760 prog = &obj->programs[i];
7761 if (prog_is_subprog(obj, prog))
7763 if (!prog->autoload) {
7764 pr_debug("prog '%s': skipped loading\n", prog->name);
7767 prog->log_level |= log_level;
7770 bpf_program_record_relos(prog);
7772 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7773 obj->license, obj->kern_version, &prog->fd);
7775 pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7788 struct bpf_program *prog;
7791 bpf_object__for_each_program(prog, obj) {
7792 prog->sec_def = find_sec_def(prog->sec_name);
7793 if (!prog->sec_def) {
7795 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7796 prog->name, prog->sec_name);
7800 prog->type = prog->sec_def->prog_type;
7801 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7806 if (prog->sec_def->prog_setup_fn) {
7807 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7809 pr_warn("prog '%s': failed to initialize: %d\n",
7810 prog->name, err);
8340 struct bpf_program *prog = st_ops->progs[i];
8344 if (!prog)
8347 prog_fd = bpf_program__fd(prog);
8490 int bpf_program__pin(struct bpf_program *prog, const char *path)
8495 if (prog->fd < 0) {
8496 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
8508 if (bpf_obj_pin(prog->fd, path)) {
8511 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
8515 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
8519 int bpf_program__unpin(struct bpf_program *prog, const char *path)
8523 if (prog->fd < 0) {
8524 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
8536 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
8755 struct bpf_program *prog;
8767 bpf_object__for_each_program(prog, obj) {
8768 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8772 err = bpf_program__pin(prog, buf);
8780 while ((prog = bpf_object__prev_program(obj, prog))) {
8781 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8784 bpf_program__unpin(prog, buf);
8792 struct bpf_program *prog;
8798 bpf_object__for_each_program(prog, obj) {
8801 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8805 err = bpf_program__unpin(prog, buf);
8994 struct bpf_program *prog = prev;
8997 prog = __bpf_program__iter(prog, obj, true);
8998 } while (prog && prog_is_subprog(obj, prog));
9000 return prog;
9006 struct bpf_program *prog = next;
9009 prog = __bpf_program__iter(prog, obj, false);
9010 } while (prog && prog_is_subprog(obj, prog));
9012 return prog;
9015 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
9017 prog->prog_ifindex = ifindex;
9020 const char *bpf_program__name(const struct bpf_program *prog)
9022 return prog->name;
9025 const char *bpf_program__section_name(const struct bpf_program *prog)
9027 return prog->sec_name;
9030 bool bpf_program__autoload(const struct bpf_program *prog)
9032 return prog->autoload;
9035 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
9037 if (prog->obj->loaded)
9040 prog->autoload = autoload;
9044 bool bpf_program__autoattach(const struct bpf_program *prog)
9046 return prog->autoattach;
9049 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
9051 prog->autoattach = autoattach;
9054 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
9056 return prog->insns;
9059 size_t bpf_program__insn_cnt(const struct bpf_program *prog)
9061 return prog->insns_cnt;
9064 int bpf_program__set_insns(struct bpf_program *prog,
9069 if (prog->obj->loaded)
9072 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
9075 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
9080 prog->insns = insns;
9081 prog->insns_cnt = new_insn_cnt;
9085 int bpf_program__fd(const struct bpf_program *prog)
9087 if (!prog)
9090 if (prog->fd < 0)
9093 return prog->fd;
9097 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
9099 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
9101 return prog->type;
9110 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
9112 if (prog->obj->loaded)
9116 if (prog->type == type)
9119 prog->type = type;
9127 if (prog->sec_def != &custom_fallback_def)
9128 prog->sec_def = NULL;
9133 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
9135 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
9137 return prog->expected_attach_type;
9140 int bpf_program__set_expected_attach_type(struct bpf_program *prog,
9143 if (prog->obj->loaded)
9146 prog->expected_attach_type = type;
9150 __u32 bpf_program__flags(const struct bpf_program *prog)
9152 return prog->prog_flags;
9155 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
9157 if (prog->obj->loaded)
9160 prog->prog_flags = flags;
9164 __u32 bpf_program__log_level(const struct bpf_program *prog)
9166 return prog->log_level;
9169 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
9171 if (prog->obj->loaded)
9174 prog->log_level = log_level;
9178 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)
9180 *log_size = prog->log_size;
9181 return prog->log_buf;
9184 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
9188 if (prog->log_size > UINT_MAX)
9190 if (prog->obj->loaded)
9193 prog->log_buf = log_buf;
9194 prog->log_size = log_size;
9207 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9208 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9209 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9211 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9213 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9214 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9215 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9216 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9217 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9218 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9219 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9586 struct bpf_program *prog;
9658 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
9659 if (!prog) {
9660 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
9665 /* prevent the use of BPF prog with invalid type */
9666 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
9667 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
9668 map->name, prog->name);
9675 if (!prog->attach_btf_id) {
9676 prog->attach_btf_id = st_ops->type_id;
9677 prog->expected_attach_type = member_idx;
9680 /* struct_ops BPF prog can be re-used between multiple
9685 if (prog->attach_btf_id != st_ops->type_id ||
9686 prog->expected_attach_type != member_idx) {
9687 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
9688 map->name, prog->name, prog->sec_name, prog->type,
9689 prog->attach_btf_id, prog->expected_attach_type, name);
9693 st_ops->progs[member_idx] = prog;
9804 pr_warn("%s is not found in prog's BTF\n", name);
9848 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
9851 enum bpf_attach_type attach_type = prog->expected_attach_type;
9852 __u32 attach_prog_fd = prog->attach_prog_fd;
9856 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9858 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9863 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
9864 prog->name, attach_prog_fd, attach_name, err);
9873 if (prog->obj->gen_loader) {
9874 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9878 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9881 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",
9882 prog->name, attach_name, err);
10384 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
10388 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
10564 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
10576 pr_warn("prog '%s': invalid perf event FD %d\n",
10577 prog->name, pfd);
10580 prog_fd = bpf_program__fd(prog);
10582 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
10583 prog->name);
10595 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
10602 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
10603 prog->name, pfd,
10610 pr_warn("prog '%s': user context value is not supported\n", prog->name);
10617 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
10618 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10620 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
10621 prog->name, pfd);
10628 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
10629 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10641 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
10643 return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
10965 bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
10998 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11026 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
11027 prog->name, retprobe ? "kretprobe" : "kprobe",
11032 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11036 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
11037 prog->name, retprobe ? "kretprobe" : "kprobe",
11060 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
11068 return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
11071 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
11081 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
11096 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
11286 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
11346 prog_fd = bpf_program__fd(prog);
11350 pr_warn("prog '%s': failed to attach: %s\n",
11351 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11364 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11375 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
11378 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
11380 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
11382 func_name = prog->sec_name + sizeof("kprobe/") - 1;
11396 *link = bpf_program__attach_kprobe_opts(prog, func, &opts);
11401 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11409 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
11412 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
11414 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
11416 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
11418 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
11422 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11432 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
11433 strcmp(prog->sec_name, "kretprobe.multi") == 0)
11436 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
11438 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
11440 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
11448 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
11453 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11461 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
11470 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
11474 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
11475 prog->sec_name);
11719 bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
11776 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
11777 prog->name, path, err);
11814 prog_fd = bpf_program__fd(prog);
11818 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
11819 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11833 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
11870 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
11871 prog->name, binary_path, err);
11904 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11934 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
11935 prog->name, retprobe ? "uretprobe" : "uprobe",
11941 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11945 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
11946 prog->name, retprobe ? "uretprobe" : "uprobe",
11978 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11987 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",
11995 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
11996 prog->name, prog->sec_name);
12013 pr_warn("prog '%s': uretprobes do not support offset specification\n",
12014 prog->name);
12018 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
12022 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
12023 prog->sec_name);
12033 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
12040 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
12044 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
12050 struct bpf_object *obj = prog->obj;
12058 if (bpf_program__fd(prog) < 0) {
12059 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
12060 prog->name);
12070 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
12071 prog->name, binary_path, err);
12089 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
12099 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12105 sec_name = bpf_program__section_name(prog);
12118 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
12180 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
12197 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
12198 prog->name, tp_category, tp_name,
12202 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
12206 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
12207 prog->name, tp_category, tp_name,
12214 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
12218 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
12221 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12228 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
12231 sec_name = strdup(prog->sec_name);
12236 if (str_has_pfx(prog->sec_name, "tp/"))
12248 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
12253 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
12260 prog_fd = bpf_program__fd(prog);
12262 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12275 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
12276 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12283 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12299 if (!str_has_pfx(prog->sec_name, prefixes[i]))
12304 if (prog->sec_name[pfx_len] == '\0')
12307 if (prog->sec_name[pfx_len] != '/')
12310 tp_name = prog->sec_name + pfx_len + 1;
12315 pr_warn("prog '%s': invalid section name '%s'\n",
12316 prog->name, prog->sec_name);
12320 *link = bpf_program__attach_raw_tracepoint(prog, tp_name);
12325 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,
12336 prog_fd = bpf_program__fd(prog);
12338 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12349 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
12353 pr_warn("prog '%s': failed to attach: %s\n",
12354 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
12361 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
12363 return bpf_program__attach_btf_id(prog, NULL);
12366 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
12369 return bpf_program__attach_btf_id(prog, opts);
12372 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
12374 return bpf_program__attach_btf_id(prog, NULL);
12377 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12379 *link = bpf_program__attach_trace(prog);
12383 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12385 *link = bpf_program__attach_lsm(prog);
12390 bpf_program_attach_fd(const struct bpf_program *prog,
12399 prog_fd = bpf_program__fd(prog);
12401 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12410 attach_type = bpf_program__expected_attach_type(prog);
12415 pr_warn("prog '%s': failed to attach to %s: %s\n",
12416 prog->name, target_name,
12425 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
12427 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL);
12431 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
12433 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL);
12436 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
12439 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL);
12443 bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,
12458 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
12459 prog->name);
12463 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
12464 prog->name);
12474 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts);
12478 bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex,
12493 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
12494 prog->name);
12498 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
12499 prog->name);
12508 return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts);
12511 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
12518 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
12519 prog->name);
12523 if (prog->type != BPF_PROG_TYPE_EXT) {
12524 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
12525 prog->name);
12538 return bpf_program_attach_fd(prog, target_fd, "freplace",
12544 return bpf_program__attach_trace(prog);
12549 bpf_program__attach_iter(const struct bpf_program *prog,
12564 prog_fd = bpf_program__fd(prog);
12566 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12580 pr_warn("prog '%s': failed to attach to iterator: %s\n",
12581 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12588 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12590 *link = bpf_program__attach_iter(prog, NULL);
12594 struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog,
12604 prog_fd = bpf_program__fd(prog);
12606 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12627 pr_warn("prog '%s': failed to attach to netfilter: %s\n",
12628 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12636 struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
12641 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12644 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
13317 int bpf_program__set_attach_target(struct bpf_program *prog,
13323 if (!prog || attach_prog_fd < 0)
13326 if (prog->obj->loaded)
13333 prog->attach_prog_fd = attach_prog_fd;
13347 err = bpf_object__load_vmlinux_btf(prog->obj, true);
13350 err = find_kernel_btf_id(prog->obj, attach_func_name,
13351 prog->expected_attach_type,
13357 prog->attach_btf_id = btf_id;
13358 prog->attach_btf_obj_fd = btf_obj_fd;
13359 prog->attach_prog_fd = attach_prog_fd;
13497 struct bpf_program **prog = progs[i].prog;
13500 *prog = bpf_object__find_program_by_name(obj, name);
13501 if (!*prog) {
13681 struct bpf_program *prog = *s->progs[i].prog;
13684 if (!prog->autoload || !prog->autoattach)
13688 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
13695 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
13697 pr_warn("prog '%s': failed to auto-attach: %d\n",
13698 bpf_program__name(prog), err);