Lines Matching refs:prog
72 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
700 void bpf_program__unload(struct bpf_program *prog)
702 if (!prog)
705 zclose(prog->fd);
707 zfree(&prog->func_info);
708 zfree(&prog->line_info);
711 static void bpf_program__exit(struct bpf_program *prog)
713 if (!prog)
716 bpf_program__unload(prog);
717 zfree(&prog->name);
718 zfree(&prog->sec_name);
719 zfree(&prog->insns);
720 zfree(&prog->reloc_desc);
722 prog->nr_reloc = 0;
723 prog->insns_cnt = 0;
724 prog->sec_idx = -1;
748 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
758 memset(prog, 0, sizeof(*prog));
759 prog->obj = obj;
761 prog->sec_idx = sec_idx;
762 prog->sec_insn_off = sec_off / BPF_INSN_SZ;
763 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;
765 prog->insns_cnt = prog->sec_insn_cnt;
767 prog->type = BPF_PROG_TYPE_UNSPEC;
768 prog->fd = -1;
775 prog->autoload = false;
779 prog->autoload = true;
782 prog->autoattach = true;
785 prog->log_level = obj->log_level;
787 prog->sec_name = strdup(sec_name);
788 if (!prog->sec_name)
791 prog->name = strdup(name);
792 if (!prog->name)
795 prog->insns = malloc(insn_data_sz);
796 if (!prog->insns)
798 memcpy(prog->insns, insn_data, insn_data_sz);
802 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name);
803 bpf_program__exit(prog);
812 struct bpf_program *prog, *progs;
868 prog = &progs[nr_progs];
870 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
883 prog->mark_btf_static = true;
1070 struct bpf_program *prog;
1072 prog = st_ops->progs[i];
1073 if (!prog)
1090 prog->attach_btf_id = kern_type_id;
1091 prog->expected_attach_type = kern_member_idx;
1095 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n",
1096 map->name, mname, prog->name, moff,
2317 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";
2332 pr_warn("map '%s': should be map-in-map or prog-array.\n",
2362 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",
3030 static bool prog_needs_vmlinux_btf(struct bpf_program *prog)
3032 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||
3033 prog->type == BPF_PROG_TYPE_LSM)
3039 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)
3047 struct bpf_program *prog;
3065 bpf_object__for_each_program(prog, obj) {
3066 if (!prog->autoload)
3068 if (prog_needs_vmlinux_btf(prog))
3123 struct bpf_program *prog = &obj->programs[i];
3128 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3138 if (strcmp(name, prog->name) != 0)
3991 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
3993 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
4000 struct bpf_program *prog;
4002 bpf_object__for_each_program(prog, obj) {
4003 if (prog_is_subprog(obj, prog))
4005 if (!strcmp(prog->name, name))
4006 return prog;
4048 static int bpf_program__record_reloc(struct bpf_program *prog,
4053 struct bpf_insn *insn = &prog->insns[insn_idx];
4054 size_t map_idx, nr_maps = prog->obj->nr_maps;
4055 struct bpf_object *obj = prog->obj;
4062 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n",
4063 prog->name, sym_name, insn_idx, insn->code);
4078 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n",
4079 prog->name, sym_name, sym_idx);
4082 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n",
4083 prog->name, i, ext->name, ext->sym_idx, insn_idx);
4096 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);
4102 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n",
4103 prog->name, sym_name, sym_sec_name);
4107 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n",
4108 prog->name, sym_name, (size_t)sym->st_value);
4118 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n",
4119 prog->name, sym_name, shdr_idx);
4129 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n",
4130 prog->name, sym_name, (size_t)sym->st_value, insn->imm);
4146 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n",
4147 prog->name, sym_name, sym_sec_name);
4156 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n",
4157 prog->name, map_idx, map->name, map->sec_idx,
4162 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n",
4163 prog->name, sym_sec_name, (size_t)sym->st_value);
4175 pr_warn("prog '%s': bad data relo against section '%s'\n",
4176 prog->name, sym_sec_name);
4183 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n",
4184 prog->name, map_idx, map->name, map->sec_idx,
4189 pr_warn("prog '%s': data relo failed to find map for section '%s'\n",
4190 prog->name, sym_sec_name);
4201 static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx)
4203 return insn_idx >= prog->sec_insn_off &&
4204 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;
4211 struct bpf_program *prog;
4218 prog = &obj->programs[m];
4220 if (prog->sec_idx < sec_idx ||
4221 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))
4229 prog = &obj->programs[l];
4230 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))
4231 return prog;
4240 struct bpf_program *prog;
4310 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4311 if (!prog) {
4317 relos = libbpf_reallocarray(prog->reloc_desc,
4318 prog->nr_reloc + 1, sizeof(*relos));
4321 prog->reloc_desc = relos;
4324 insn_idx -= prog->sec_insn_off;
4325 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],
4330 prog->nr_reloc++;
4757 int ret, map, prog, insn_cnt = ARRAY_SIZE(insns);
4768 prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL);
4769 if (prog < 0) {
4774 ret = bpf_prog_bind_map(prog, map, NULL);
4777 close(prog);
5261 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n",
5265 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n",
5335 * runtime due to bpf_program__set_autoload(prog, false),
5705 static int record_relo_core(struct bpf_program *prog,
5710 relos = libbpf_reallocarray(prog->reloc_desc,
5711 prog->nr_reloc + 1, sizeof(*relos));
5714 relo = &relos[prog->nr_reloc];
5718 prog->reloc_desc = relos;
5719 prog->nr_reloc++;
5723 static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx)
5728 for (i = 0; i < prog->nr_reloc; i++) {
5729 relo = &prog->reloc_desc[i];
5739 static int bpf_core_resolve_relo(struct bpf_program *prog,
5748 const char *prog_name = prog->name;
5764 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5766 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n",
5791 struct bpf_program *prog;
5832 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
5833 if (!prog) {
5849 if (!prog->autoload)
5856 insn_idx = insn_idx - prog->sec_insn_off;
5857 if (insn_idx >= prog->insns_cnt)
5859 insn = &prog->insns[insn_idx];
5861 err = record_relo_core(prog, rec, insn_idx);
5863 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n",
5864 prog->name, i, err);
5868 if (prog->obj->gen_loader)
5871 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
5873 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n",
5874 prog->name, i, err);
5878 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);
5880 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n",
5881 prog->name, i, insn_idx, err);
5905 static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx,
5911 pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n",
5912 prog->name, relo_idx, insn_idx, map_idx, map->name);
5935 static void poison_kfunc_call(struct bpf_program *prog, int relo_idx,
5939 pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n",
5940 prog->name, relo_idx, insn_idx, ext->name);
5961 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
5965 for (i = 0; i < prog->nr_reloc; i++) {
5966 struct reloc_desc *relo = &prog->reloc_desc[i];
5967 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
5981 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
5995 poison_map_ldimm64(prog, i, relo->insn_idx, insn,
6028 poison_kfunc_call(prog, i, relo->insn_idx, insn,
6034 pr_warn("prog '%s': relo #%d: bad insn\n",
6035 prog->name, i);
6047 pr_warn("prog '%s': relo #%d: bad relo type %d\n",
6048 prog->name, i, relo->type);
6057 const struct bpf_program *prog,
6072 if (prog->sec_idx != sec_idx)
6078 if (insn_off < prog->sec_insn_off)
6080 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)
6109 off_adj = prog->sub_insn_off - prog->sec_insn_off;
6127 const struct bpf_program *prog)
6140 if (main_prog != prog && !main_prog->func_info)
6143 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6149 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",
6150 prog->name, err);
6158 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);
6162 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n",
6163 prog->name);
6168 if (main_prog != prog && !main_prog->line_info)
6171 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6177 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",
6178 prog->name, err);
6186 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);
6190 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n",
6191 prog->name);
6206 static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx)
6208 if (!prog->nr_reloc)
6210 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,
6211 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);
6245 struct bpf_program *prog)
6253 err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6257 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {
6258 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6262 relo = find_prog_insn_relo(prog, insn_idx);
6269 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n",
6270 prog->name, insn_idx, relo->type);
6293 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n",
6294 prog->name, insn_idx);
6303 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;
6309 pr_warn("prog '%s': no .text section found yet sub-program call exists\n",
6310 prog->name);
6330 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);
6339 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",
6354 insn = &main_prog->insns[prog->sub_insn_off + insn_idx];
6356 * prog; each main prog can have a different set of
6361 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;
6363 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n",
6364 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);
6374 * main prog) is processed separately. For each subprog (non-entry functions,
6377 * hasn't been yet appended and relocated within current main prog. Once its
6378 * relocated, sub_insn_off will point at the position within current main prog
6388 * the *main* prog's instructions (subprog's instructions are always left
6390 * and subprog instructions are always sent only as part of a main prog) and
6392 * prog or any of its subprogs are appended (and relocated), all their
6398 * main prog and any of its called/used subprogs. Those will keep their
6400 * main prog and won't be relocated within the context of current main prog.
6452 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6468 err = bpf_object__reloc_code(obj, prog, prog);
6478 struct bpf_program *prog;
6483 prog = &obj->programs[i];
6484 zfree(&prog->reloc_desc);
6485 prog->nr_reloc = 0;
6521 struct bpf_program *prog;
6543 prog = &obj->programs[i];
6544 for (j = 0; j < prog->nr_reloc; j++) {
6545 struct reloc_desc *relo = &prog->reloc_desc[j];
6546 struct bpf_insn *insn = &prog->insns[relo->insn_idx];
6562 prog = &obj->programs[i];
6566 if (prog_is_subprog(obj, prog))
6568 if (!prog->autoload)
6571 err = bpf_object__relocate_calls(obj, prog);
6573 pr_warn("prog '%s': failed to relocate calls: %d\n",
6574 prog->name, err);
6580 prog = &obj->programs[i];
6581 if (prog_is_subprog(obj, prog))
6583 if (!prog->autoload)
6585 err = bpf_object__relocate_data(obj, prog);
6587 pr_warn("prog '%s': failed to relocate data references: %d\n",
6588 prog->name, err);
6660 type = is_map_in_map ? "map" : "prog";
6785 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
6787 struct bpf_insn *insn = prog->insns;
6794 for (i = 0; i < prog->insns_cnt; i++, insn++) {
6820 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
6823 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
6824 static int libbpf_prepare_prog_load(struct bpf_program *prog,
6830 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
6836 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))
6840 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))
6841 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;
6843 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {
6847 attach_name = strchr(prog->sec_name, '/');
6858 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n",
6859 prog->name);
6864 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id);
6868 /* cache resolved BTF FD and BTF type ID in the prog */
6869 prog->attach_btf_obj_fd = btf_obj_fd;
6870 prog->attach_btf_id = btf_type_id;
6873 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because
6883 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz);
6885 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
6896 __u32 log_level = prog->log_level;
6898 if (prog->type == BPF_PROG_TYPE_UNSPEC) {
6903 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n",
6904 prog->name, prog->sec_name);
6912 prog_name = prog->name;
6913 load_attr.attach_prog_fd = prog->attach_prog_fd;
6914 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;
6915 load_attr.attach_btf_id = prog->attach_btf_id;
6917 load_attr.prog_ifindex = prog->prog_ifindex;
6923 load_attr.func_info = prog->func_info;
6924 load_attr.func_info_rec_size = prog->func_info_rec_size;
6925 load_attr.func_info_cnt = prog->func_info_cnt;
6926 load_attr.line_info = prog->line_info;
6927 load_attr.line_info_rec_size = prog->line_info_rec_size;
6928 load_attr.line_info_cnt = prog->line_info_cnt;
6931 load_attr.prog_flags = prog->prog_flags;
6935 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {
6936 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);
6938 pr_warn("prog '%s': failed to prepare load attributes: %d\n",
6939 prog->name, err);
6942 insns = prog->insns;
6943 insns_cnt = prog->insns_cnt;
6947 load_attr.expected_attach_type = prog->expected_attach_type;
6950 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
6952 prog - obj->programs);
6964 if (prog->log_buf) {
6965 log_buf = prog->log_buf;
6966 log_buf_size = prog->log_size;
6989 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);
6992 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
6993 prog->name, log_buf);
7001 map = &prog->obj->maps[i];
7007 pr_warn("prog '%s': failed to bind map '%s': %s\n",
7008 prog->name, map->real_name, cp);
7036 fixup_verifier_log(prog, log_buf, log_buf_size);
7039 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);
7043 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",
7044 prog->name, log_buf);
7104 static void fixup_log_failed_core_relo(struct bpf_program *prog,
7126 relo = find_relo_core(prog, insn_idx);
7130 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
7143 static void fixup_log_missing_map_load(struct bpf_program *prog,
7155 struct bpf_object *obj = prog->obj;
7176 static void fixup_log_missing_kfunc_call(struct bpf_program *prog,
7188 struct bpf_object *obj = prog->obj;
7209 static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz)
7234 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz,
7243 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz,
7252 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz,
7259 static int bpf_program_record_relos(struct bpf_program *prog)
7261 struct bpf_object *obj = prog->obj;
7264 for (i = 0; i < prog->nr_reloc; i++) {
7265 struct reloc_desc *relo = &prog->reloc_desc[i];
7305 struct bpf_program *prog;
7310 prog = &obj->programs[i];
7311 err = bpf_object__sanitize_prog(obj, prog);
7317 prog = &obj->programs[i];
7318 if (prog_is_subprog(obj, prog))
7320 if (!prog->autoload) {
7321 pr_debug("prog '%s': skipped loading\n", prog->name);
7324 prog->log_level |= log_level;
7327 bpf_program_record_relos(prog);
7329 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7330 obj->license, obj->kern_version, &prog->fd);
7332 pr_warn("prog '%s': failed to load: %d\n", prog->name, err);
7345 struct bpf_program *prog;
7348 bpf_object__for_each_program(prog, obj) {
7349 prog->sec_def = find_sec_def(prog->sec_name);
7350 if (!prog->sec_def) {
7352 pr_debug("prog '%s': unrecognized ELF section name '%s'\n",
7353 prog->name, prog->sec_name);
7357 prog->type = prog->sec_def->prog_type;
7358 prog->expected_attach_type = prog->sec_def->expected_attach_type;
7363 if (prog->sec_def->prog_setup_fn) {
7364 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);
7366 pr_warn("prog '%s': failed to initialize: %d\n",
7367 prog->name, err);
7895 struct bpf_program *prog = st_ops->progs[i];
7899 if (!prog)
7902 prog_fd = bpf_program__fd(prog);
8045 int bpf_program__pin(struct bpf_program *prog, const char *path)
8050 if (prog->fd < 0) {
8051 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);
8063 if (bpf_obj_pin(prog->fd, path)) {
8066 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);
8070 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);
8074 int bpf_program__unpin(struct bpf_program *prog, const char *path)
8078 if (prog->fd < 0) {
8079 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);
8091 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);
8310 struct bpf_program *prog;
8322 bpf_object__for_each_program(prog, obj) {
8323 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8327 err = bpf_program__pin(prog, buf);
8335 while ((prog = bpf_object__prev_program(obj, prog))) {
8336 if (pathname_concat(buf, sizeof(buf), path, prog->name))
8339 bpf_program__unpin(prog, buf);
8347 struct bpf_program *prog;
8353 bpf_object__for_each_program(prog, obj) {
8356 err = pathname_concat(buf, sizeof(buf), path, prog->name);
8360 err = bpf_program__unpin(prog, buf);
8549 struct bpf_program *prog = prev;
8552 prog = __bpf_program__iter(prog, obj, true);
8553 } while (prog && prog_is_subprog(obj, prog));
8555 return prog;
8561 struct bpf_program *prog = next;
8564 prog = __bpf_program__iter(prog, obj, false);
8565 } while (prog && prog_is_subprog(obj, prog));
8567 return prog;
8570 void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex)
8572 prog->prog_ifindex = ifindex;
8575 const char *bpf_program__name(const struct bpf_program *prog)
8577 return prog->name;
8580 const char *bpf_program__section_name(const struct bpf_program *prog)
8582 return prog->sec_name;
8585 bool bpf_program__autoload(const struct bpf_program *prog)
8587 return prog->autoload;
8590 int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
8592 if (prog->obj->loaded)
8595 prog->autoload = autoload;
8599 bool bpf_program__autoattach(const struct bpf_program *prog)
8601 return prog->autoattach;
8604 void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach)
8606 prog->autoattach = autoattach;
8609 const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog)
8611 return prog->insns;
8614 size_t bpf_program__insn_cnt(const struct bpf_program *prog)
8616 return prog->insns_cnt;
8619 int bpf_program__set_insns(struct bpf_program *prog,
8624 if (prog->obj->loaded)
8627 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));
8630 pr_warn("prog '%s': failed to realloc prog code\n", prog->name);
8635 prog->insns = insns;
8636 prog->insns_cnt = new_insn_cnt;
8640 int bpf_program__fd(const struct bpf_program *prog)
8642 if (!prog)
8645 if (prog->fd < 0)
8648 return prog->fd;
8652 enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog);
8654 enum bpf_prog_type bpf_program__type(const struct bpf_program *prog)
8656 return prog->type;
8665 int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type)
8667 if (prog->obj->loaded)
8671 if (prog->type == type)
8674 prog->type = type;
8682 if (prog->sec_def != &custom_fallback_def)
8683 prog->sec_def = NULL;
8688 enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog);
8690 enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog)
8692 return prog->expected_attach_type;
8695 int bpf_program__set_expected_attach_type(struct bpf_program *prog,
8698 if (prog->obj->loaded)
8701 prog->expected_attach_type = type;
8705 __u32 bpf_program__flags(const struct bpf_program *prog)
8707 return prog->prog_flags;
8710 int bpf_program__set_flags(struct bpf_program *prog, __u32 flags)
8712 if (prog->obj->loaded)
8715 prog->prog_flags = flags;
8719 __u32 bpf_program__log_level(const struct bpf_program *prog)
8721 return prog->log_level;
8724 int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level)
8726 if (prog->obj->loaded)
8729 prog->log_level = log_level;
8733 const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size)
8735 *log_size = prog->log_size;
8736 return prog->log_buf;
8739 int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size)
8743 if (prog->log_size > UINT_MAX)
8745 if (prog->obj->loaded)
8748 prog->log_buf = log_buf;
8749 prog->log_size = log_size;
8762 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8763 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8764 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8765 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8766 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8767 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8768 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8769 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8770 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8771 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link);
8772 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link);
9130 struct bpf_program *prog;
9202 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
9203 if (!prog) {
9204 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n",
9209 /* prevent the use of BPF prog with invalid type */
9210 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {
9211 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n",
9212 map->name, prog->name);
9219 if (!prog->attach_btf_id) {
9220 prog->attach_btf_id = st_ops->type_id;
9221 prog->expected_attach_type = member_idx;
9224 /* struct_ops BPF prog can be re-used between multiple
9229 if (prog->attach_btf_id != st_ops->type_id ||
9230 prog->expected_attach_type != member_idx) {
9231 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n",
9232 map->name, prog->name, prog->sec_name, prog->type,
9233 prog->attach_btf_id, prog->expected_attach_type, name);
9237 st_ops->progs[member_idx] = prog;
9348 pr_warn("%s is not found in prog's BTF\n", name);
9392 static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name,
9395 enum bpf_attach_type attach_type = prog->expected_attach_type;
9396 __u32 attach_prog_fd = prog->attach_prog_fd;
9400 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {
9402 pr_warn("prog '%s': attach program FD is not set\n", prog->name);
9407 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n",
9408 prog->name, attach_prog_fd, attach_name, err);
9417 if (prog->obj->gen_loader) {
9418 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9422 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9425 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n",
9426 prog->name, attach_name, err);
9928 int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog)
9932 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL);
10108 struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd,
10120 pr_warn("prog '%s': invalid perf event FD %d\n",
10121 prog->name, pfd);
10124 prog_fd = bpf_program__fd(prog);
10126 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
10127 prog->name);
10139 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
10146 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n",
10147 prog->name, pfd,
10154 pr_warn("prog '%s': user context value is not supported\n", prog->name);
10161 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n",
10162 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10164 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n",
10165 prog->name, pfd);
10172 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n",
10173 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10185 struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd)
10187 return bpf_program__attach_perf_event_opts(prog, pfd, NULL);
10509 bpf_program__attach_kprobe_opts(const struct bpf_program *prog,
10542 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
10570 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n",
10571 prog->name, retprobe ? "kretprobe" : "kprobe",
10576 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
10580 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n",
10581 prog->name, retprobe ? "kretprobe" : "kprobe",
10604 struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog,
10612 return bpf_program__attach_kprobe_opts(prog, func_name, &opts);
10615 struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog,
10625 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
10640 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts);
10830 bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog,
10890 prog_fd = bpf_program__fd(prog);
10894 pr_warn("prog '%s': failed to attach: %s\n",
10895 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
10908 static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10919 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)
10922 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");
10924 func_name = prog->sec_name + sizeof("kretprobe/") - 1;
10926 func_name = prog->sec_name + sizeof("kprobe/") - 1;
10940 *link = bpf_program__attach_kprobe_opts(prog, func, &opts);
10945 static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10953 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)
10956 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");
10958 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;
10960 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;
10962 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts);
10966 static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
10976 if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||
10977 strcmp(prog->sec_name, "kretprobe.multi") == 0)
10980 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");
10982 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;
10984 spec = prog->sec_name + sizeof("kprobe.multi/") - 1;
10992 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts);
10997 static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11005 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%ms",
11014 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);
11018 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
11019 prog->sec_name);
11253 bpf_program__attach_uprobe_multi(const struct bpf_program *prog,
11310 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
11311 prog->name, path, err);
11348 prog_fd = bpf_program__fd(prog);
11352 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",
11353 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));
11367 bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid,
11404 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
11405 prog->name, binary_path, err);
11438 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11468 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n",
11469 prog->name, retprobe ? "uretprobe" : "uprobe",
11475 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11479 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n",
11480 prog->name, retprobe ? "uretprobe" : "uprobe",
11512 static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11521 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[a-zA-Z0-9_.]+%li",
11529 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n",
11530 prog->name, prog->sec_name);
11537 pr_warn("prog '%s': uretprobes do not support offset specification\n",
11538 prog->name);
11542 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);
11546 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,
11547 prog->sec_name);
11557 struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog,
11564 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts);
11567 struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog,
11573 struct bpf_object *obj = prog->obj;
11581 if (bpf_program__fd(prog) < 0) {
11582 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n",
11583 prog->name);
11593 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n",
11594 prog->name, binary_path, err);
11612 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
11620 static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11626 sec_name = bpf_program__section_name(prog);
11639 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path,
11700 struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog,
11717 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n",
11718 prog->name, tp_category, tp_name,
11722 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts);
11726 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n",
11727 prog->name, tp_category, tp_name,
11734 struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog,
11738 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL);
11741 static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11748 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)
11751 sec_name = strdup(prog->sec_name);
11756 if (str_has_pfx(prog->sec_name, "tp/"))
11768 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name);
11773 struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog,
11780 prog_fd = bpf_program__fd(prog);
11782 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11795 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n",
11796 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11803 static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11819 if (!str_has_pfx(prog->sec_name, prefixes[i]))
11824 if (prog->sec_name[pfx_len] == '\0')
11827 if (prog->sec_name[pfx_len] != '/')
11830 tp_name = prog->sec_name + pfx_len + 1;
11835 pr_warn("prog '%s': invalid section name '%s'\n",
11836 prog->name, prog->sec_name);
11840 *link = bpf_program__attach_raw_tracepoint(prog, tp_name);
11845 static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog,
11856 prog_fd = bpf_program__fd(prog);
11858 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11869 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts);
11873 pr_warn("prog '%s': failed to attach: %s\n",
11874 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));
11881 struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog)
11883 return bpf_program__attach_btf_id(prog, NULL);
11886 struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog,
11889 return bpf_program__attach_btf_id(prog, opts);
11892 struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog)
11894 return bpf_program__attach_btf_id(prog, NULL);
11897 static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11899 *link = bpf_program__attach_trace(prog);
11903 static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link)
11905 *link = bpf_program__attach_lsm(prog);
11910 bpf_program_attach_fd(const struct bpf_program *prog,
11919 prog_fd = bpf_program__fd(prog);
11921 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
11930 attach_type = bpf_program__expected_attach_type(prog);
11935 pr_warn("prog '%s': failed to attach to %s: %s\n",
11936 prog->name, target_name,
11945 bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd)
11947 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL);
11951 bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd)
11953 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL);
11956 struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex)
11959 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL);
11963 bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex,
11978 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n",
11979 prog->name);
11983 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n",
11984 prog->name);
11994 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts);
11997 struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog,
12004 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n",
12005 prog->name);
12009 if (prog->type != BPF_PROG_TYPE_EXT) {
12010 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace",
12011 prog->name);
12024 return bpf_program_attach_fd(prog, target_fd, "freplace",
12030 return bpf_program__attach_trace(prog);
12035 bpf_program__attach_iter(const struct bpf_program *prog,
12050 prog_fd = bpf_program__fd(prog);
12052 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12066 pr_warn("prog '%s': failed to attach to iterator: %s\n",
12067 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12074 static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link)
12076 *link = bpf_program__attach_iter(prog, NULL);
12080 struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog,
12090 prog_fd = bpf_program__fd(prog);
12092 pr_warn("prog '%s': can't attach before loaded\n", prog->name);
12113 pr_warn("prog '%s': failed to attach to netfilter: %s\n",
12114 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));
12122 struct bpf_link *bpf_program__attach(const struct bpf_program *prog)
12127 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
12130 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);
12803 int bpf_program__set_attach_target(struct bpf_program *prog,
12809 if (!prog || attach_prog_fd < 0)
12812 if (prog->obj->loaded)
12819 prog->attach_prog_fd = attach_prog_fd;
12833 err = bpf_object__load_vmlinux_btf(prog->obj, true);
12836 err = find_kernel_btf_id(prog->obj, attach_func_name,
12837 prog->expected_attach_type,
12843 prog->attach_btf_id = btf_id;
12844 prog->attach_btf_obj_fd = btf_obj_fd;
12845 prog->attach_prog_fd = attach_prog_fd;
12983 struct bpf_program **prog = progs[i].prog;
12986 *prog = bpf_object__find_program_by_name(obj, name);
12987 if (!*prog) {
13167 struct bpf_program *prog = *s->progs[i].prog;
13170 if (!prog->autoload || !prog->autoattach)
13174 if (!prog->sec_def || !prog->sec_def->prog_attach_fn)
13181 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);
13183 pr_warn("prog '%s': failed to auto-attach: %d\n",
13184 bpf_program__name(prog), err);