Lines Matching defs:obj
71 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj);
72 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog);
435 struct bpf_object *obj;
505 struct bpf_object *obj;
579 * 0 for vmlinux BTF, index in obj->fd_array for module
690 static const char *elf_sym_str(const struct bpf_object *obj, size_t off);
691 static const char *elf_sec_str(const struct bpf_object *obj, size_t off);
692 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx);
693 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name);
694 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn);
695 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn);
696 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn);
697 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx);
748 bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog,
759 prog->obj = obj;
785 prog->log_level = obj->log_level;
808 bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data,
811 Elf_Data *symbols = obj->efile.symbols;
819 progs = obj->programs;
820 nr_progs = obj->nr_programs;
824 sym = elf_sym_by_idx(obj, i);
834 name = elf_sym_str(obj, sym->st_name);
847 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {
858 * In this case the original obj->programs
866 obj->programs = progs;
870 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name,
886 obj->nr_programs = nr_progs;
1120 static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj)
1126 for (i = 0; i < obj->nr_maps; i++) {
1127 map = &obj->maps[i];
1132 err = bpf_map__init_kern_struct_ops(map, obj->btf,
1133 obj->btf_vmlinux);
1141 static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name,
1156 btf = obj->btf;
1168 type = btf__type_by_id(obj->btf, vsi->type);
1169 var_name = btf__name_by_offset(obj->btf, type->name_off);
1171 type_id = btf__resolve_type(obj->btf, vsi->type);
1178 type = btf__type_by_id(obj->btf, type_id);
1179 tname = btf__name_by_offset(obj->btf, type->name_off);
1189 map = bpf_object__add_map(obj);
1236 static int bpf_object_init_struct_ops(struct bpf_object *obj)
1240 err = init_struct_ops_maps(obj, STRUCT_OPS_SEC, obj->efile.st_ops_shndx,
1241 obj->efile.st_ops_data, 0);
1242 err = err ?: init_struct_ops_maps(obj, STRUCT_OPS_LINK_SEC,
1243 obj->efile.st_ops_link_shndx,
1244 obj->efile.st_ops_link_data,
1254 struct bpf_object *obj;
1257 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1);
1258 if (!obj) {
1263 strcpy(obj->path, path);
1265 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));
1268 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));
1269 end = strchr(obj->name, '.');
1274 obj->efile.fd = -1;
1281 obj->efile.obj_buf = obj_buf;
1282 obj->efile.obj_buf_sz = obj_buf_sz;
1283 obj->efile.btf_maps_shndx = -1;
1284 obj->efile.st_ops_shndx = -1;
1285 obj->efile.st_ops_link_shndx = -1;
1286 obj->kconfig_map_idx = -1;
1288 obj->kern_version = get_kernel_version();
1289 obj->loaded = false;
1291 return obj;
1294 static void bpf_object__elf_finish(struct bpf_object *obj)
1296 if (!obj->efile.elf)
1299 elf_end(obj->efile.elf);
1300 obj->efile.elf = NULL;
1301 obj->efile.symbols = NULL;
1302 obj->efile.st_ops_data = NULL;
1303 obj->efile.st_ops_link_data = NULL;
1305 zfree(&obj->efile.secs);
1306 obj->efile.sec_cnt = 0;
1307 zclose(obj->efile.fd);
1308 obj->efile.obj_buf = NULL;
1309 obj->efile.obj_buf_sz = 0;
1312 static int bpf_object__elf_init(struct bpf_object *obj)
1318 if (obj->efile.elf) {
1323 if (obj->efile.obj_buf_sz > 0) {
1325 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);
1327 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);
1328 if (obj->efile.fd < 0) {
1333 pr_warn("elf: failed to open %s: %s\n", obj->path, cp);
1337 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);
1341 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));
1346 obj->efile.elf = elf;
1350 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);
1356 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);
1360 obj->efile.ehdr = ehdr = elf64_getehdr(elf);
1361 if (!obj->efile.ehdr) {
1362 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));
1367 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {
1369 obj->path, elf_errmsg(-1));
1375 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {
1377 obj->path, elf_errmsg(-1));
1384 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);
1391 bpf_object__elf_finish(obj);
1395 static int bpf_object__check_endianness(struct bpf_object *obj)
1398 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)
1401 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)
1406 pr_warn("elf: endianness mismatch in %s.\n", obj->path);
1411 bpf_object__init_license(struct bpf_object *obj, void *data, size_t size)
1414 pr_warn("invalid license section in %s\n", obj->path);
1420 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));
1421 pr_debug("license of %s is %s\n", obj->path, obj->license);
1426 bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size)
1431 pr_warn("invalid kver section in %s\n", obj->path);
1435 obj->kern_version = kver;
1436 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);
1448 static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size)
1456 scn = elf_sec_by_name(obj, name);
1457 data = elf_sec_data(obj, scn);
1466 static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name)
1468 Elf_Data *symbols = obj->efile.symbols;
1473 Elf64_Sym *sym = elf_sym_by_idx(obj, si);
1482 sname = elf_sym_str(obj, sym->st_name);
1494 static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
1499 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,
1500 sizeof(*obj->maps), obj->nr_maps + 1);
1504 map = &obj->maps[obj->nr_maps++];
1505 map->obj = obj;
1543 static char *internal_map_name(struct bpf_object *obj, const char *real_name)
1588 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));
1590 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
1602 map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map);
1609 static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map)
1618 t = btf__type_by_id(obj->btf, map->btf_value_type_id);
1624 vt = btf__type_by_id(obj->btf, vsi->type);
1636 bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
1644 map = bpf_object__add_map(obj);
1652 map->name = internal_map_name(obj, real_name);
1668 (void) map_fill_btf_type_info(obj, map);
1670 if (map_is_mmapable(obj, map))
1692 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
1696 static int bpf_object__init_global_data_maps(struct bpf_object *obj)
1703 * Populate obj->maps with libbpf internal maps.
1705 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {
1706 sec_desc = &obj->efile.secs[sec_idx];
1714 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1715 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
1721 obj->has_rodata = true;
1722 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1723 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
1729 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx));
1730 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
1746 static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
1751 for (i = 0; i < obj->nr_extern; i++) {
1752 if (strcmp(obj->externs[i].name, name) == 0)
1753 return &obj->externs[i];
1908 static int bpf_object__process_kconfig_line(struct bpf_object *obj,
1938 ext = find_extern_by_name(obj, buf);
1972 static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data)
1997 err = bpf_object__process_kconfig_line(obj, buf, data);
2010 static int bpf_object__read_kconfig_mem(struct bpf_object *obj,
2025 err = bpf_object__process_kconfig_line(obj, buf, data);
2037 static int bpf_object__init_kconfig_map(struct bpf_object *obj)
2043 for (i = 0; i < obj->nr_extern; i++) {
2044 ext = &obj->externs[i];
2053 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG,
2054 ".kconfig", obj->efile.symbols_shndx,
2059 obj->kconfig_map_idx = obj->nr_maps - 1;
2515 static int bpf_object__init_user_btf_map(struct bpf_object *obj,
2530 var = btf__type_by_id(obj->btf, vi->type);
2532 map_name = btf__name_by_offset(obj->btf, var->name_off);
2553 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
2564 map = bpf_object__add_map(obj);
2580 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);
2608 err = map_fill_btf_type_info(obj, map);
2615 static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict,
2625 if (obj->efile.btf_maps_shndx < 0)
2628 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);
2629 data = elf_sec_data(obj, scn);
2632 MAPS_ELF_SEC, obj->path);
2636 nr_types = btf__type_cnt(obj->btf);
2638 t = btf__type_by_id(obj->btf, i);
2641 name = btf__name_by_offset(obj->btf, t->name_off);
2644 obj->efile.btf_maps_sec_btf_id = i;
2656 err = bpf_object__init_user_btf_map(obj, sec, i,
2657 obj->efile.btf_maps_shndx,
2667 static int bpf_object__init_maps(struct bpf_object *obj,
2677 err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path);
2678 err = err ?: bpf_object__init_global_data_maps(obj);
2679 err = err ?: bpf_object__init_kconfig_map(obj);
2680 err = err ?: bpf_object_init_struct_ops(obj);
2685 static bool section_have_execinstr(struct bpf_object *obj, int idx)
2689 sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx));
2696 static bool btf_needs_sanitization(struct bpf_object *obj)
2698 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2699 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2700 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2701 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2702 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2703 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2704 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2710 static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf)
2712 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC);
2713 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC);
2714 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT);
2715 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC);
2716 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG);
2717 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG);
2718 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64);
2810 static bool libbpf_needs_btf(const struct bpf_object *obj)
2812 return obj->efile.btf_maps_shndx >= 0 ||
2813 obj->efile.st_ops_shndx >= 0 ||
2814 obj->efile.st_ops_link_shndx >= 0 ||
2815 obj->nr_extern > 0;
2818 static bool kernel_needs_btf(const struct bpf_object *obj)
2820 return obj->efile.st_ops_shndx >= 0 || obj->efile.st_ops_link_shndx >= 0;
2823 static int bpf_object__init_btf(struct bpf_object *obj,
2830 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);
2831 err = libbpf_get_error(obj->btf);
2833 obj->btf = NULL;
2838 btf__set_pointer_size(obj->btf, 8);
2844 if (!obj->btf) {
2849 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);
2850 err = libbpf_get_error(obj->btf_ext);
2854 obj->btf_ext = NULL;
2859 ext_segs[0] = &obj->btf_ext->func_info;
2860 ext_segs[1] = &obj->btf_ext->line_info;
2861 ext_segs[2] = &obj->btf_ext->core_relo_info;
2884 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
2887 scn = elf_sec_by_name(obj, sec_name);
2896 if (err && libbpf_needs_btf(obj)) {
2911 static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
2943 err = find_elf_sec_sz(obj, sec_name, &size);
2977 sym = find_elf_var_sym(obj, var_name);
3004 static int bpf_object_fixup_btf(struct bpf_object *obj)
3008 if (!obj->btf)
3011 n = btf__type_cnt(obj->btf);
3013 struct btf_type *t = btf_type_by_id(obj->btf, i);
3021 err = btf_fixup_datasec(obj, obj->btf, t);
3045 static bool obj_needs_vmlinux_btf(const struct bpf_object *obj)
3053 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)
3057 for (i = 0; i < obj->nr_extern; i++) {
3060 ext = &obj->externs[i];
3065 bpf_object__for_each_program(prog, obj) {
3075 static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force)
3080 if (obj->btf_vmlinux || obj->gen_loader)
3083 if (!force && !obj_needs_vmlinux_btf(obj))
3086 obj->btf_vmlinux = btf__load_vmlinux_btf();
3087 err = libbpf_get_error(obj->btf_vmlinux);
3090 obj->btf_vmlinux = NULL;
3096 static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj)
3098 struct btf *kern_btf = obj->btf;
3102 if (!obj->btf)
3105 if (!kernel_supports(obj, FEAT_BTF)) {
3106 if (kernel_needs_btf(obj)) {
3122 for (i = 0; i < obj->nr_programs; i++) {
3123 struct bpf_program *prog = &obj->programs[i];
3128 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))
3131 n = btf__type_cnt(obj->btf);
3133 t = btf_type_by_id(obj->btf, j);
3137 name = btf__str_by_offset(obj->btf, t->name_off);
3146 sanitize = btf_needs_sanitization(obj);
3152 raw_data = btf__raw_data(obj->btf, &sz);
3159 btf__set_pointer_size(obj->btf, 8);
3160 err = bpf_object__sanitize_btf(obj, kern_btf);
3165 if (obj->gen_loader) {
3171 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);
3178 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,
3179 obj->log_level ? 1 : 0);
3184 btf__set_fd(obj->btf, btf__fd(kern_btf));
3191 btf_mandatory = kernel_needs_btf(obj);
3201 static const char *elf_sym_str(const struct bpf_object *obj, size_t off)
3205 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);
3208 off, obj->path, elf_errmsg(-1));
3215 static const char *elf_sec_str(const struct bpf_object *obj, size_t off)
3219 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);
3222 off, obj->path, elf_errmsg(-1));
3229 static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx)
3233 scn = elf_getscn(obj->efile.elf, idx);
3236 idx, obj->path, elf_errmsg(-1));
3242 static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name)
3245 Elf *elf = obj->efile.elf;
3249 sec_name = elf_sec_name(obj, scn);
3261 static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn)
3271 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3278 static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn)
3286 sh = elf_sec_hdr(obj, scn);
3290 name = elf_sec_str(obj, sh->sh_name);
3293 elf_ndxscn(scn), obj->path, elf_errmsg(-1));
3300 static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn)
3310 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>",
3311 obj->path, elf_errmsg(-1));
3318 static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx)
3320 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))
3323 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;
3386 static int bpf_object__elf_collect(struct bpf_object *obj)
3389 Elf *elf = obj->efile.elf;
3403 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {
3405 obj->path, elf_errmsg(-1));
3408 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));
3409 if (!obj->efile.secs)
3417 sh = elf_sec_hdr(obj, scn);
3422 if (obj->efile.symbols) {
3423 pr_warn("elf: multiple symbol tables in %s\n", obj->path);
3427 data = elf_sec_data(obj, scn);
3433 obj->efile.symbols = data;
3434 obj->efile.symbols_shndx = idx;
3435 obj->efile.strtabidx = sh->sh_link;
3439 if (!obj->efile.symbols) {
3441 obj->path);
3448 sec_desc = &obj->efile.secs[idx];
3450 sh = elf_sec_hdr(obj, scn);
3454 name = elf_sec_str(obj, sh->sh_name);
3461 data = elf_sec_data(obj, scn);
3471 err = bpf_object__init_license(obj, data->d_buf, data->d_size);
3475 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);
3482 obj->efile.btf_maps_shndx = idx;
3496 obj->efile.text_shndx = idx;
3497 err = bpf_object__add_programs(obj, data, name, idx);
3511 obj->efile.st_ops_data = data;
3512 obj->efile.st_ops_shndx = idx;
3514 obj->efile.st_ops_link_data = data;
3515 obj->efile.st_ops_link_shndx = idx;
3524 targ_sec_idx >= obj->efile.sec_cnt)
3528 if (!section_have_execinstr(obj, targ_sec_idx) &&
3534 elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>");
3552 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {
3553 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);
3560 if (obj->nr_programs)
3561 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);
3563 return bpf_object__init_btf(obj, btf_data, btf_ext_data);
3772 static int bpf_object__collect_externs(struct bpf_object *obj)
3783 if (!obj->efile.symbols)
3786 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);
3787 sh = elf_sec_hdr(obj, scn);
3791 dummy_var_btf_id = add_dummy_ksym_var(obj->btf);
3799 Elf64_Sym *sym = elf_sym_by_idx(obj, i);
3805 ext_name = elf_sym_str(obj, sym->st_name);
3809 ext = obj->externs;
3810 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));
3813 obj->externs = ext;
3814 ext = &ext[obj->nr_extern];
3816 obj->nr_extern++;
3818 ext->btf_id = find_extern_btf_id(obj->btf, ext_name);
3824 t = btf__type_by_id(obj->btf, ext->btf_id);
3825 ext->name = btf__name_by_offset(obj->btf, t->name_off);
3837 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);
3843 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);
3844 sec_name = btf__name_by_offset(obj->btf, sec->name_off);
3854 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);
3860 ext->kcfg.align = btf__align_of(obj->btf, t->type);
3866 ext->kcfg.type = find_kcfg_type(obj->btf, t->type,
3875 skip_mods_and_typedefs(obj->btf, t->type,
3882 pr_debug("collected %d externs total\n", obj->nr_extern);
3884 if (!obj->nr_extern)
3888 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);
3898 int int_btf_id = find_int_btf_id(obj->btf);
3906 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);
3907 for (i = 0; i < obj->nr_extern; i++) {
3908 ext = &obj->externs[i];
3921 vt = (void *)btf__type_by_id(obj->btf, vs->type);
3922 ext_name = btf__name_by_offset(obj->btf, vt->name_off);
3923 ext = find_extern_by_name(obj, ext_name);
3934 func_proto = btf__type_by_id(obj->btf,
3961 for (i = 0; i < obj->nr_extern; i++) {
3962 ext = &obj->externs[i];
3976 t = btf__type_by_id(obj->btf, vs->type);
3977 ext_name = btf__name_by_offset(obj->btf, t->name_off);
3978 ext = find_extern_by_name(obj, ext_name);
3991 static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog)
3993 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;
3997 bpf_object__find_program_by_name(const struct bpf_object *obj,
4002 bpf_object__for_each_program(prog, obj) {
4003 if (prog_is_subprog(obj, prog))
4011 static bool bpf_object__shndx_is_data(const struct bpf_object *obj,
4014 switch (obj->efile.secs[shndx].sec_type) {
4024 static bool bpf_object__shndx_is_maps(const struct bpf_object *obj,
4027 return shndx == obj->efile.btf_maps_shndx;
4031 bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx)
4033 if (shndx == obj->efile.symbols_shndx)
4036 switch (obj->efile.secs[shndx].sec_type) {
4054 size_t map_idx, nr_maps = prog->obj->nr_maps;
4055 struct bpf_object *obj = prog->obj;
4069 int i, n = obj->nr_extern;
4073 ext = &obj->externs[i];
4100 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {
4101 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4124 if (sym_is_subprog(sym, obj->efile.text_shndx)) {
4140 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx);
4141 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx));
4145 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) {
4151 map = &obj->maps[map_idx];
4174 if (!bpf_object__shndx_is_data(obj, shdr_idx)) {
4180 map = &obj->maps[map_idx];
4207 static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj,
4210 int l = 0, r = obj->nr_programs - 1, m;
4213 if (!obj->nr_programs)
4218 prog = &obj->programs[m];
4229 prog = &obj->programs[l];
4236 bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data)
4250 if (sec_idx >= obj->efile.sec_cnt)
4253 scn = elf_sec_by_idx(obj, sec_idx);
4254 scn_data = elf_sec_data(obj, scn);
4258 relo_sec_name = elf_sec_str(obj, shdr->sh_name);
4259 sec_name = elf_sec_name(obj, scn);
4275 sym = elf_sym_by_idx(obj, sym_idx);
4282 if (sym->st_shndx >= obj->efile.sec_cnt) {
4302 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));
4304 sym_name = elf_sym_str(obj, sym->st_name);
4310 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
4335 static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map)
4339 if (!obj->btf)
4346 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))
4356 id = btf__find_by_name(obj->btf, map->real_name);
4408 if (map->obj->loaded)
4492 if (map->obj->loaded)
4505 bpf_object__probe_loading(struct bpf_object *obj)
4514 if (obj->gen_loader)
4971 bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id)
4976 if (obj && obj->gen_loader)
5062 bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map)
5068 if (obj->gen_loader) {
5069 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,
5072 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);
5100 static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner)
5107 if (kernel_supports(obj, FEAT_PROG_NAME))
5117 if (obj->btf && btf__fd(obj->btf) >= 0) {
5118 create_attr.btf_fd = btf__fd(obj->btf);
5128 err = bpf_object__create_map(obj, map->inner_map, true);
5163 if (obj->gen_loader) {
5164 bpf_gen__map_create(obj->gen_loader, def->type, map_name,
5166 &create_attr, is_inner ? -1 : map - obj->maps);
5197 if (obj->gen_loader)
5206 static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map)
5219 if (obj->gen_loader) {
5220 bpf_gen__populate_outer_map(obj->gen_loader,
5221 map - obj->maps, i,
5222 targ_map - obj->maps);
5242 static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map)
5248 if (obj->gen_loader)
5275 static int bpf_object_init_prog_arrays(struct bpf_object *obj)
5280 for (i = 0; i < obj->nr_maps; i++) {
5281 map = &obj->maps[i];
5286 err = init_prog_array_slots(obj, map);
5314 bpf_object__create_maps(struct bpf_object *obj)
5322 for (i = 0; i < obj->nr_maps; i++) {
5323 map = &obj->maps[i];
5339 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA))
5372 err = bpf_object__create_map(obj, map, false);
5380 err = bpf_object__populate_internal_map(obj, map);
5388 err = init_map_in_map_slots(obj, map);
5418 zclose(obj->maps[j].fd);
5507 static int load_module_btfs(struct bpf_object *obj)
5516 if (obj->btf_modules_loaded)
5519 if (obj->gen_loader)
5523 obj->btf_modules_loaded = true;
5526 if (!kernel_supports(obj, FEAT_MODULE_BTF))
5570 btf = btf_get_from_fd(fd, obj->btf_vmlinux);
5578 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,
5579 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);
5583 mod_btf = &obj->btf_modules[obj->btf_module_cnt++];
5604 bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id)
5630 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;
5640 if (obj->btf_vmlinux_override)
5644 err = load_module_btfs(obj);
5648 for (i = 0; i < obj->btf_module_cnt; i++) {
5650 obj->btf_modules[i].btf,
5651 obj->btf_modules[i].name,
5652 btf__type_cnt(obj->btf_vmlinux),
5764 cands = bpf_core_find_cands(prog->obj, local_btf, local_id);
5783 bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path)
5796 if (obj->btf_ext->core_relo_info.len == 0)
5800 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);
5801 err = libbpf_get_error(obj->btf_vmlinux_override);
5814 seg = &obj->btf_ext->core_relo_info;
5820 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);
5832 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx);
5868 if (prog->obj->gen_loader)
5871 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);
5888 /* obj->btf_vmlinux and module BTFs are freed after object load */
5889 btf__free(obj->btf_vmlinux_override);
5890 obj->btf_vmlinux_override = NULL;
5923 * where lower 123 is map index into obj->maps[] array
5950 * where lower 123 is extern index into obj->externs[] array
5961 bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog)
5973 map = &obj->maps[relo->map_idx];
5974 if (obj->gen_loader) {
5986 map = &obj->maps[relo->map_idx];
5988 if (obj->gen_loader) {
6000 ext = &obj->externs[relo->ext_idx];
6002 if (obj->gen_loader) {
6004 insn[0].imm = obj->kconfig_map_idx;
6007 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;
6022 ext = &obj->externs[relo->ext_idx];
6056 static int adjust_prog_btf_ext_info(const struct bpf_object *obj,
6125 reloc_prog_func_and_line_info(const struct bpf_object *obj,
6134 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))
6143 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,
6171 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,
6244 bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog,
6253 err = reloc_prog_func_and_line_info(obj, main_prog, prog);
6307 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);
6346 err = bpf_object__reloc_code(obj, main_prog, subprog);
6452 bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog)
6460 for (i = 0; i < obj->nr_programs; i++) {
6461 subprog = &obj->programs[i];
6462 if (!prog_is_subprog(obj, subprog))
6468 err = bpf_object__reloc_code(obj, prog, prog);
6476 bpf_object__free_relocs(struct bpf_object *obj)
6482 for (i = 0; i < obj->nr_programs; i++) {
6483 prog = &obj->programs[i];
6504 static void bpf_object__sort_relos(struct bpf_object *obj)
6508 for (i = 0; i < obj->nr_programs; i++) {
6509 struct bpf_program *p = &obj->programs[i];
6519 bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path)
6525 if (obj->btf_ext) {
6526 err = bpf_object__relocate_core(obj, targ_btf_path);
6532 bpf_object__sort_relos(obj);
6542 for (i = 0; i < obj->nr_programs; i++) {
6543 prog = &obj->programs[i];
6561 for (i = 0; i < obj->nr_programs; i++) {
6562 prog = &obj->programs[i];
6566 if (prog_is_subprog(obj, prog))
6571 err = bpf_object__relocate_calls(obj, prog);
6579 for (i = 0; i < obj->nr_programs; i++) {
6580 prog = &obj->programs[i];
6581 if (prog_is_subprog(obj, prog))
6585 err = bpf_object__relocate_data(obj, prog);
6596 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
6599 static int bpf_object__collect_map_relos(struct bpf_object *obj,
6616 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)
6618 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);
6630 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
6636 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
6642 for (j = 0; j < obj->nr_maps; j++) {
6643 map = &obj->maps[j];
6644 if (map->sec_idx != obj->efile.btf_maps_shndx)
6652 if (j == obj->nr_maps) {
6662 if (sym->st_shndx != obj->efile.btf_maps_shndx) {
6673 targ_map = bpf_object__find_map_by_name(obj, name);
6680 targ_prog = bpf_object__find_program_by_name(obj, name);
6688 prog_is_subprog(obj, targ_prog)) {
6697 var = btf__type_by_id(obj->btf, vi->type);
6698 def = skip_mods_and_typedefs(obj->btf, var->type, NULL);
6702 mname = btf__name_by_offset(obj->btf, member->name_off);
6736 static int bpf_object__collect_relos(struct bpf_object *obj)
6740 for (i = 0; i < obj->efile.sec_cnt; i++) {
6741 struct elf_sec_desc *sec_desc = &obj->efile.secs[i];
6758 if (idx == obj->efile.st_ops_shndx || idx == obj->efile.st_ops_link_shndx)
6759 err = bpf_object__collect_st_ops_relos(obj, shdr, data);
6760 else if (idx == obj->efile.btf_maps_shndx)
6761 err = bpf_object__collect_map_relos(obj, shdr, data);
6763 err = bpf_object__collect_prog_relos(obj, shdr, data);
6768 bpf_object__sort_relos(obj);
6785 static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog)
6791 if (obj->gen_loader)
6805 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6810 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN))
6830 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))
6840 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))
6885 static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog,
6911 if (kernel_supports(obj, FEAT_PROG_NAME))
6920 btf_fd = bpf_object__btf_fd(obj);
6921 if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {
6932 load_attr.fd_array = obj->fd_array;
6949 if (obj->gen_loader) {
6950 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,
6952 prog - obj->programs);
6968 } else if (obj->log_buf) {
6969 log_buf = obj->log_buf;
6970 log_buf_size = obj->log_size;
6996 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {
7000 for (i = 0; i < obj->nr_maps; i++) {
7001 map = &prog->obj->maps[i];
7130 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);
7153 * "345" in "2001000345" is a map index in obj->maps to fetch map name.
7155 struct bpf_object *obj = prog->obj;
7164 if (map_idx < 0 || map_idx >= obj->nr_maps)
7166 map = &obj->maps[map_idx];
7186 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.
7188 struct bpf_object *obj = prog->obj;
7197 if (ext_idx < 0 || ext_idx >= obj->nr_extern)
7199 ext = &obj->externs[ext_idx];
7261 struct bpf_object *obj = prog->obj;
7266 struct extern_desc *ext = &obj->externs[relo->ext_idx];
7273 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?
7275 bpf_gen__record_extern(obj->gen_loader, ext->name,
7280 bpf_gen__record_extern(obj->gen_loader, ext->name,
7292 bpf_gen__record_relo_core(obj->gen_loader, &cr);
7303 bpf_object__load_progs(struct bpf_object *obj, int log_level)
7309 for (i = 0; i < obj->nr_programs; i++) {
7310 prog = &obj->programs[i];
7311 err = bpf_object__sanitize_prog(obj, prog);
7316 for (i = 0; i < obj->nr_programs; i++) {
7317 prog = &obj->programs[i];
7318 if (prog_is_subprog(obj, prog))
7326 if (obj->gen_loader)
7329 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
7330 obj->license, obj->kern_version, &prog->fd);
7337 bpf_object__free_relocs(obj);
7343 static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts)
7348 bpf_object__for_each_program(prog, obj) {
7380 struct bpf_object *obj;
7416 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name);
7417 if (IS_ERR(obj))
7418 return obj;
7420 obj->log_buf = log_buf;
7421 obj->log_size = log_size;
7422 obj->log_level = log_level;
7430 obj->btf_custom_path = strdup(btf_tmp_path);
7431 if (!obj->btf_custom_path) {
7439 obj->kconfig = strdup(kconfig);
7440 if (!obj->kconfig) {
7446 err = bpf_object__elf_init(obj);
7447 err = err ? : bpf_object__check_endianness(obj);
7448 err = err ? : bpf_object__elf_collect(obj);
7449 err = err ? : bpf_object__collect_externs(obj);
7450 err = err ? : bpf_object_fixup_btf(obj);
7451 err = err ? : bpf_object__init_maps(obj, opts);
7452 err = err ? : bpf_object_init_progs(obj, opts);
7453 err = err ? : bpf_object__collect_relos(obj);
7457 bpf_object__elf_finish(obj);
7459 return obj;
7461 bpf_object__close(obj);
7491 static int bpf_object_unload(struct bpf_object *obj)
7495 if (!obj)
7498 for (i = 0; i < obj->nr_maps; i++) {
7499 zclose(obj->maps[i].fd);
7500 if (obj->maps[i].st_ops)
7501 zfree(&obj->maps[i].st_ops->kern_vdata);
7504 for (i = 0; i < obj->nr_programs; i++)
7505 bpf_program__unload(&obj->programs[i]);
7510 static int bpf_object__sanitize_maps(struct bpf_object *obj)
7514 bpf_object__for_each_map(m, obj) {
7517 if (!kernel_supports(obj, FEAT_ARRAY_MMAP))
7561 struct bpf_object *obj = ctx;
7565 ext = find_extern_by_name(obj, sym_name);
7569 t = btf__type_by_id(obj->btf, ext->btf_id);
7586 static int bpf_object__read_kallsyms_file(struct bpf_object *obj)
7588 return libbpf_kallsyms_parse(kallsyms_cb, obj);
7591 static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name,
7599 btf = obj->btf_vmlinux;
7604 err = load_module_btfs(obj);
7608 for (i = 0; i < obj->btf_module_cnt; i++) {
7610 mod_btf = &obj->btf_modules[i];
7625 static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj,
7635 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);
7652 err = bpf_core_types_are_compat(obj->btf, local_type_id,
7658 local_type = btf__type_by_id(obj->btf, local_type_id);
7659 local_name = btf__name_by_offset(obj->btf, local_type->name_off);
7678 static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj,
7689 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,
7702 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,
7717 if (obj->fd_array_cnt == INT16_MAX) {
7723 if (!obj->fd_array_cnt)
7724 obj->fd_array_cnt = 1;
7726 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),
7727 obj->fd_array_cnt + 1);
7730 mod_btf->fd_array_idx = obj->fd_array_cnt;
7732 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;
7750 static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj)
7756 for (i = 0; i < obj->nr_extern; i++) {
7757 ext = &obj->externs[i];
7761 if (obj->gen_loader) {
7767 t = btf__type_by_id(obj->btf, ext->btf_id);
7769 err = bpf_object__resolve_ksym_var_btf_id(obj, ext);
7771 err = bpf_object__resolve_ksym_func_btf_id(obj, ext);
7778 static int bpf_object__resolve_externs(struct bpf_object *obj,
7787 if (obj->nr_extern == 0)
7790 if (obj->kconfig_map_idx >= 0)
7791 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;
7793 for (i = 0; i < obj->nr_extern; i++) {
7794 ext = &obj->externs[i];
7820 value = kernel_supports(obj, FEAT_BPF_COOKIE);
7822 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER);
7846 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data);
7850 for (i = 0; i < obj->nr_extern; i++) {
7851 ext = &obj->externs[i];
7859 err = bpf_object__read_kconfig_file(obj, kcfg_data);
7864 err = bpf_object__read_kallsyms_file(obj);
7869 err = bpf_object__resolve_ksyms_btf_id(obj);
7873 for (i = 0; i < obj->nr_extern; i++) {
7874 ext = &obj->externs[i];
7908 static int bpf_object_prepare_struct_ops(struct bpf_object *obj)
7912 for (i = 0; i < obj->nr_maps; i++)
7913 if (bpf_map__is_struct_ops(&obj->maps[i]))
7914 bpf_map_prepare_vdata(&obj->maps[i]);
7919 static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path)
7923 if (!obj)
7926 if (obj->loaded) {
7927 pr_warn("object '%s': load can't be attempted twice\n", obj->name);
7931 if (obj->gen_loader)
7932 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);
7934 err = bpf_object__probe_loading(obj);
7935 err = err ? : bpf_object__load_vmlinux_btf(obj, false);
7936 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);
7937 err = err ? : bpf_object__sanitize_and_load_btf(obj);
7938 err = err ? : bpf_object__sanitize_maps(obj);
7939 err = err ? : bpf_object__init_kern_struct_ops_maps(obj);
7940 err = err ? : bpf_object__create_maps(obj);
7941 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);
7942 err = err ? : bpf_object__load_progs(obj, extra_log_level);
7943 err = err ? : bpf_object_init_prog_arrays(obj);
7944 err = err ? : bpf_object_prepare_struct_ops(obj);
7946 if (obj->gen_loader) {
7948 if (obj->btf)
7949 btf__set_fd(obj->btf, -1);
7950 for (i = 0; i < obj->nr_maps; i++)
7951 obj->maps[i].fd = -1;
7953 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);
7957 zfree(&obj->fd_array);
7960 for (i = 0; i < obj->btf_module_cnt; i++) {
7961 close(obj->btf_modules[i].fd);
7962 btf__free(obj->btf_modules[i].btf);
7963 free(obj->btf_modules[i].name);
7965 free(obj->btf_modules);
7968 btf__free(obj->btf_vmlinux);
7969 obj->btf_vmlinux = NULL;
7971 obj->loaded = true; /* doesn't matter if successfully or not */
7979 for (i = 0; i < obj->nr_maps; i++)
7980 if (obj->maps[i].pinned && !obj->maps[i].reused)
7981 bpf_map__unpin(&obj->maps[i], NULL);
7983 bpf_object_unload(obj);
7984 pr_warn("failed to load object '%s'\n", obj->path);
7988 int bpf_object__load(struct bpf_object *obj)
7990 return bpf_object_load(obj, 0, NULL);
8230 int bpf_object__pin_maps(struct bpf_object *obj, const char *path)
8235 if (!obj)
8238 if (!obj->loaded) {
8243 bpf_object__for_each_map(map, obj) {
8268 while ((map = bpf_object__prev_map(obj, map))) {
8278 int bpf_object__unpin_maps(struct bpf_object *obj, const char *path)
8283 if (!obj)
8286 bpf_object__for_each_map(map, obj) {
8308 int bpf_object__pin_programs(struct bpf_object *obj, const char *path)
8314 if (!obj)
8317 if (!obj->loaded) {
8322 bpf_object__for_each_program(prog, obj) {
8335 while ((prog = bpf_object__prev_program(obj, prog))) {
8345 int bpf_object__unpin_programs(struct bpf_object *obj, const char *path)
8350 if (!obj)
8353 bpf_object__for_each_program(prog, obj) {
8368 int bpf_object__pin(struct bpf_object *obj, const char *path)
8372 err = bpf_object__pin_maps(obj, path);
8376 err = bpf_object__pin_programs(obj, path);
8378 bpf_object__unpin_maps(obj, path);
8385 int bpf_object__unpin(struct bpf_object *obj, const char *path)
8389 err = bpf_object__unpin_programs(obj, path);
8393 err = bpf_object__unpin_maps(obj, path);
8433 void bpf_object__close(struct bpf_object *obj)
8437 if (IS_ERR_OR_NULL(obj))
8440 usdt_manager_free(obj->usdt_man);
8441 obj->usdt_man = NULL;
8443 bpf_gen__free(obj->gen_loader);
8444 bpf_object__elf_finish(obj);
8445 bpf_object_unload(obj);
8446 btf__free(obj->btf);
8447 btf__free(obj->btf_vmlinux);
8448 btf_ext__free(obj->btf_ext);
8450 for (i = 0; i < obj->nr_maps; i++)
8451 bpf_map__destroy(&obj->maps[i]);
8453 zfree(&obj->btf_custom_path);
8454 zfree(&obj->kconfig);
8456 for (i = 0; i < obj->nr_extern; i++)
8457 zfree(&obj->externs[i].essent_name);
8459 zfree(&obj->externs);
8460 obj->nr_extern = 0;
8462 zfree(&obj->maps);
8463 obj->nr_maps = 0;
8465 if (obj->programs && obj->nr_programs) {
8466 for (i = 0; i < obj->nr_programs; i++)
8467 bpf_program__exit(&obj->programs[i]);
8469 zfree(&obj->programs);
8471 free(obj);
8474 const char *bpf_object__name(const struct bpf_object *obj)
8476 return obj ? obj->name : libbpf_err_ptr(-EINVAL);
8479 unsigned int bpf_object__kversion(const struct bpf_object *obj)
8481 return obj ? obj->kern_version : 0;
8484 struct btf *bpf_object__btf(const struct bpf_object *obj)
8486 return obj ? obj->btf : NULL;
8489 int bpf_object__btf_fd(const struct bpf_object *obj)
8491 return obj->btf ? btf__fd(obj->btf) : -1;
8494 int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version)
8496 if (obj->loaded)
8499 obj->kern_version = kern_version;
8504 int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts)
8516 obj->gen_loader = gen;
8521 __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj,
8524 size_t nr_programs = obj->nr_programs;
8532 return forward ? &obj->programs[0] :
8533 &obj->programs[nr_programs - 1];
8535 if (p->obj != obj) {
8540 idx = (p - obj->programs) + (forward ? 1 : -1);
8541 if (idx >= obj->nr_programs || idx < 0)
8543 return &obj->programs[idx];
8547 bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev)
8552 prog = __bpf_program__iter(prog, obj, true);
8553 } while (prog && prog_is_subprog(obj, prog));
8559 bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next)
8564 prog = __bpf_program__iter(prog, obj, false);
8565 } while (prog && prog_is_subprog(obj, prog));
8592 if (prog->obj->loaded)
8624 if (prog->obj->loaded)
8667 if (prog->obj->loaded)
8698 if (prog->obj->loaded)
8712 if (prog->obj->loaded)
8726 if (prog->obj->loaded)
8745 if (prog->obj->loaded)
9104 static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj,
9111 for (i = 0; i < obj->nr_maps; i++) {
9112 map = &obj->maps[i];
9125 static int bpf_object__collect_st_ops_relos(struct bpf_object *obj,
9141 btf = obj->btf;
9150 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));
9157 name = elf_sym_str(obj, sym->st_name) ?: "<?>";
9158 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);
9202 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx);
9355 static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name,
9361 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);
9370 ret = load_module_btfs(obj);
9374 for (i = 0; i < obj->btf_module_cnt; i++) {
9375 const struct module_btf *mod = &obj->btf_modules[i];
9417 if (prog->obj->gen_loader) {
9418 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);
9422 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id);
9574 btf = bpf_object__btf(map->obj);
9730 __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i)
9735 if (!obj || !obj->maps)
9738 s = obj->maps;
9739 e = obj->maps + obj->nr_maps;
9747 idx = (m - obj->maps) + i;
9748 if (idx >= obj->nr_maps || idx < 0)
9750 return &obj->maps[idx];
9754 bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev)
9757 return obj->maps;
9759 return __bpf_map__iter(prev, obj, 1);
9763 bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next)
9766 if (!obj->nr_maps)
9768 return obj->maps + obj->nr_maps - 1;
9771 return __bpf_map__iter(next, obj, -1);
9775 bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name)
9779 bpf_object__for_each_map(pos, obj) {
9802 bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name)
9804 return bpf_map__fd(bpf_object__find_map_by_name(obj, name));
10139 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {
10542 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
10625 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {
11438 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))
11573 struct bpf_object *obj = prog->obj;
11603 if (IS_ERR(obj->usdt_man))
11604 return libbpf_ptr(obj->usdt_man);
11605 if (!obj->usdt_man) {
11606 obj->usdt_man = usdt_manager_new(obj);
11607 if (IS_ERR(obj->usdt_man))
11608 return libbpf_ptr(obj->usdt_man);
11612 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,
12812 if (prog->obj->loaded)
12833 err = bpf_object__load_vmlinux_btf(prog->obj, true);
12836 err = find_kernel_btf_id(prog->obj, attach_func_name,
12952 static int populate_skeleton_maps(const struct bpf_object *obj,
12963 *map = bpf_object__find_map_by_name(obj, name);
12976 static int populate_skeleton_progs(const struct bpf_object *obj,
12986 *prog = bpf_object__find_program_by_name(obj, name);
13001 struct bpf_object *obj;
13016 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);
13017 err = libbpf_get_error(obj);
13024 *s->obj = obj;
13025 err = populate_skeleton_maps(obj, s->maps, s->map_cnt);
13031 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt);
13051 if (!s->obj)
13054 btf = bpf_object__btf(s->obj);
13057 bpf_object__name(s->obj));
13061 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt);
13067 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt);
13114 err = bpf_object__load(*s->obj);
13222 if (s->obj)
13223 bpf_object__close(*s->obj);