1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3/* 4 * Common eBPF ELF object loading operations. 5 * 6 * Copyright (C) 2013-2015 Alexei Starovoitov <ast@kernel.org> 7 * Copyright (C) 2015 Wang Nan <wangnan0@huawei.com> 8 * Copyright (C) 2015 Huawei Inc. 9 * Copyright (C) 2017 Nicira, Inc. 10 * Copyright (C) 2019 Isovalent, Inc. 11 */ 12 13#ifndef _GNU_SOURCE 14#define _GNU_SOURCE 15#endif 16#include <stdlib.h> 17#include <stdio.h> 18#include <stdarg.h> 19#include <libgen.h> 20#include <inttypes.h> 21#include <limits.h> 22#include <string.h> 23#include <unistd.h> 24#include <endian.h> 25#include <fcntl.h> 26#include <errno.h> 27#include <ctype.h> 28#include <asm/unistd.h> 29#include <linux/err.h> 30#include <linux/kernel.h> 31#include <linux/bpf.h> 32#include <linux/btf.h> 33#include <linux/filter.h> 34#include <linux/limits.h> 35#include <linux/perf_event.h> 36#include <linux/ring_buffer.h> 37#include <linux/version.h> 38#include <sys/epoll.h> 39#include <sys/ioctl.h> 40#include <sys/mman.h> 41#include <sys/stat.h> 42#include <sys/types.h> 43#include <sys/vfs.h> 44#include <sys/utsname.h> 45#include <sys/resource.h> 46#ifdef HAVE_LIBELF 47#include <libelf.h> 48#include <gelf.h> 49#endif 50 51#include <zlib.h> 52 53#include "libbpf.h" 54#include "bpf.h" 55#include "btf.h" 56#include "str_error.h" 57#include "libbpf_internal.h" 58#include "hashmap.h" 59#include "bpf_gen_internal.h" 60#include "zip.h" 61 62#ifndef BPF_FS_MAGIC 63#define BPF_FS_MAGIC 0xcafe4a11 64#endif 65 66#define BPF_INSN_SZ (sizeof(struct bpf_insn)) 67 68/* vsprintf() in __base_pr() uses nonliteral format string. It may break 69 * compilation if user enables corresponding warning. Disable it explicitly. 70 */ 71#pragma GCC diagnostic ignored "-Wformat-nonliteral" 72 73#define __printf(a, b) __attribute__((format(printf, a, b))) 74 75static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); 76static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); 77 78static const char * const attach_type_name[] = { 79 [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress", 80 [BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress", 81 [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create", 82 [BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release", 83 [BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops", 84 [BPF_CGROUP_DEVICE] = "cgroup_device", 85 [BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind", 86 [BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind", 87 [BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect", 88 [BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect", 89 [BPF_CGROUP_UNIX_CONNECT] = "cgroup_unix_connect", 90 [BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind", 91 [BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind", 92 [BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername", 93 [BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername", 94 [BPF_CGROUP_UNIX_GETPEERNAME] = "cgroup_unix_getpeername", 95 [BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname", 96 [BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname", 97 [BPF_CGROUP_UNIX_GETSOCKNAME] = "cgroup_unix_getsockname", 98 [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg", 99 [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg", 100 [BPF_CGROUP_UNIX_SENDMSG] = "cgroup_unix_sendmsg", 101 [BPF_CGROUP_SYSCTL] = "cgroup_sysctl", 102 [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg", 103 [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg", 104 [BPF_CGROUP_UNIX_RECVMSG] = "cgroup_unix_recvmsg", 105 [BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt", 106 [BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt", 107 [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser", 108 [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict", 109 [BPF_SK_SKB_VERDICT] = "sk_skb_verdict", 110 [BPF_SK_MSG_VERDICT] = "sk_msg_verdict", 111 [BPF_LIRC_MODE2] = "lirc_mode2", 112 [BPF_FLOW_DISSECTOR] = "flow_dissector", 113 [BPF_TRACE_RAW_TP] = "trace_raw_tp", 114 [BPF_TRACE_FENTRY] = "trace_fentry", 115 [BPF_TRACE_FEXIT] = "trace_fexit", 116 [BPF_MODIFY_RETURN] = "modify_return", 117 [BPF_LSM_MAC] = "lsm_mac", 118 [BPF_LSM_CGROUP] = "lsm_cgroup", 119 [BPF_SK_LOOKUP] = "sk_lookup", 120 [BPF_TRACE_ITER] = "trace_iter", 121 [BPF_XDP_DEVMAP] = "xdp_devmap", 122 [BPF_XDP_CPUMAP] = "xdp_cpumap", 123 [BPF_XDP] = "xdp", 124 [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select", 125 [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate", 126 [BPF_PERF_EVENT] = "perf_event", 127 [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi", 128 [BPF_STRUCT_OPS] = "struct_ops", 129 [BPF_NETFILTER] = "netfilter", 130 [BPF_TCX_INGRESS] = "tcx_ingress", 131 [BPF_TCX_EGRESS] = "tcx_egress", 132 [BPF_TRACE_UPROBE_MULTI] = "trace_uprobe_multi", 133 [BPF_NETKIT_PRIMARY] = "netkit_primary", 134 [BPF_NETKIT_PEER] = "netkit_peer", 135}; 136 137static const char * const link_type_name[] = { 138 [BPF_LINK_TYPE_UNSPEC] = "unspec", 139 [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 140 [BPF_LINK_TYPE_TRACING] = "tracing", 141 [BPF_LINK_TYPE_CGROUP] = "cgroup", 142 [BPF_LINK_TYPE_ITER] = "iter", 143 [BPF_LINK_TYPE_NETNS] = "netns", 144 [BPF_LINK_TYPE_XDP] = "xdp", 145 [BPF_LINK_TYPE_PERF_EVENT] = "perf_event", 146 [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", 147 [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops", 148 [BPF_LINK_TYPE_NETFILTER] = "netfilter", 149 [BPF_LINK_TYPE_TCX] = "tcx", 150 [BPF_LINK_TYPE_UPROBE_MULTI] = "uprobe_multi", 151 [BPF_LINK_TYPE_NETKIT] = "netkit", 152}; 153 154static const char * const map_type_name[] = { 155 [BPF_MAP_TYPE_UNSPEC] = "unspec", 156 [BPF_MAP_TYPE_HASH] = "hash", 157 [BPF_MAP_TYPE_ARRAY] = "array", 158 [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array", 159 [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array", 160 [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash", 161 [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array", 162 [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace", 163 [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array", 164 [BPF_MAP_TYPE_LRU_HASH] = "lru_hash", 165 [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash", 166 [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie", 167 [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps", 168 [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", 169 [BPF_MAP_TYPE_DEVMAP] = "devmap", 170 [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash", 171 [BPF_MAP_TYPE_SOCKMAP] = "sockmap", 172 [BPF_MAP_TYPE_CPUMAP] = "cpumap", 173 [BPF_MAP_TYPE_XSKMAP] = "xskmap", 174 [BPF_MAP_TYPE_SOCKHASH] = "sockhash", 175 [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", 176 [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", 177 [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", 178 [BPF_MAP_TYPE_QUEUE] = "queue", 179 [BPF_MAP_TYPE_STACK] = "stack", 180 [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage", 181 [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops", 182 [BPF_MAP_TYPE_RINGBUF] = "ringbuf", 183 [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", 184 [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", 185 [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", 186 [BPF_MAP_TYPE_USER_RINGBUF] = "user_ringbuf", 187 [BPF_MAP_TYPE_CGRP_STORAGE] = "cgrp_storage", 188}; 189 190static const char * const prog_type_name[] = { 191 [BPF_PROG_TYPE_UNSPEC] = "unspec", 192 [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", 193 [BPF_PROG_TYPE_KPROBE] = "kprobe", 194 [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", 195 [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", 196 [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", 197 [BPF_PROG_TYPE_XDP] = "xdp", 198 [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", 199 [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", 200 [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", 201 [BPF_PROG_TYPE_LWT_IN] = "lwt_in", 202 [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", 203 [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", 204 [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", 205 [BPF_PROG_TYPE_SK_SKB] = "sk_skb", 206 [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", 207 [BPF_PROG_TYPE_SK_MSG] = "sk_msg", 208 [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", 209 [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", 210 [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", 211 [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", 212 [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", 213 [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", 214 [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl", 215 [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable", 216 [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt", 217 [BPF_PROG_TYPE_TRACING] = "tracing", 218 [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops", 219 [BPF_PROG_TYPE_EXT] = "ext", 220 [BPF_PROG_TYPE_LSM] = "lsm", 221 [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup", 222 [BPF_PROG_TYPE_SYSCALL] = "syscall", 223 [BPF_PROG_TYPE_NETFILTER] = "netfilter", 224}; 225 226static int __base_pr(enum libbpf_print_level level, const char *format, 227 va_list args) 228{ 229 if (level == LIBBPF_DEBUG) 230 return 0; 231 232 return vfprintf(stderr, format, args); 233} 234 235static libbpf_print_fn_t __libbpf_pr = __base_pr; 236 237libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn) 238{ 239 libbpf_print_fn_t old_print_fn; 240 241 old_print_fn = __atomic_exchange_n(&__libbpf_pr, fn, __ATOMIC_RELAXED); 242 243 return old_print_fn; 244} 245 246__printf(2, 3) 247void libbpf_print(enum libbpf_print_level level, const char *format, ...) 248{ 249 va_list args; 250 int old_errno; 251 libbpf_print_fn_t print_fn; 252 253 print_fn = __atomic_load_n(&__libbpf_pr, __ATOMIC_RELAXED); 254 if (!print_fn) 255 return; 256 257 old_errno = errno; 258 259 va_start(args, format); 260 __libbpf_pr(level, format, args); 261 va_end(args); 262 263 errno = old_errno; 264} 265 266static void pr_perm_msg(int err) 267{ 268 struct rlimit limit; 269 char buf[100]; 270 271 if (err != -EPERM || geteuid() != 0) 272 return; 273 274 err = getrlimit(RLIMIT_MEMLOCK, &limit); 275 if (err) 276 return; 277 278 if (limit.rlim_cur == RLIM_INFINITY) 279 return; 280 281 if (limit.rlim_cur < 1024) 282 snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur); 283 else if (limit.rlim_cur < 1024*1024) 284 snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024); 285 else 286 snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024)); 287 288 pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n", 289 buf); 290} 291 292#define STRERR_BUFSIZE 128 293 294/* Copied from tools/perf/util/util.h */ 295#ifndef zfree 296# define zfree(ptr) ({ free(*ptr); *ptr = NULL; }) 297#endif 298 299#ifndef zclose 300# define zclose(fd) ({ \ 301 int ___err = 0; \ 302 if ((fd) >= 0) \ 303 ___err = close((fd)); \ 304 fd = -1; \ 305 ___err; }) 306#endif 307 308static inline __u64 ptr_to_u64(const void *ptr) 309{ 310 return (__u64) (unsigned long) ptr; 311} 312 313int libbpf_set_strict_mode(enum libbpf_strict_mode mode) 314{ 315 /* as of v1.0 libbpf_set_strict_mode() is a no-op */ 316 return 0; 317} 318 319__u32 libbpf_major_version(void) 320{ 321 return LIBBPF_MAJOR_VERSION; 322} 323 324__u32 libbpf_minor_version(void) 325{ 326 return LIBBPF_MINOR_VERSION; 327} 328 329const char *libbpf_version_string(void) 330{ 331#define __S(X) #X 332#define _S(X) __S(X) 333 return "v" _S(LIBBPF_MAJOR_VERSION) "." _S(LIBBPF_MINOR_VERSION); 334#undef _S 335#undef __S 336} 337 338enum reloc_type { 339 RELO_LD64, 340 RELO_CALL, 341 RELO_DATA, 342 RELO_EXTERN_LD64, 343 RELO_EXTERN_CALL, 344 RELO_SUBPROG_ADDR, 345 RELO_CORE, 346}; 347 348struct reloc_desc { 349 enum reloc_type type; 350 int insn_idx; 351 union { 352 const struct bpf_core_relo *core_relo; /* used when type == RELO_CORE */ 353 struct { 354 int map_idx; 355 int sym_off; 356 int ext_idx; 357 }; 358 }; 359}; 360 361/* stored as sec_def->cookie for all libbpf-supported SEC()s */ 362enum sec_def_flags { 363 SEC_NONE = 0, 364 /* expected_attach_type is optional, if kernel doesn't support that */ 365 SEC_EXP_ATTACH_OPT = 1, 366 /* legacy, only used by libbpf_get_type_names() and 367 * libbpf_attach_type_by_name(), not used by libbpf itself at all. 368 * This used to be associated with cgroup (and few other) BPF programs 369 * that were attachable through BPF_PROG_ATTACH command. Pretty 370 * meaningless nowadays, though. 371 */ 372 SEC_ATTACHABLE = 2, 373 SEC_ATTACHABLE_OPT = SEC_ATTACHABLE | SEC_EXP_ATTACH_OPT, 374 /* attachment target is specified through BTF ID in either kernel or 375 * other BPF program's BTF object 376 */ 377 SEC_ATTACH_BTF = 4, 378 /* BPF program type allows sleeping/blocking in kernel */ 379 SEC_SLEEPABLE = 8, 380 /* BPF program support non-linear XDP buffer */ 381 SEC_XDP_FRAGS = 16, 382 /* Setup proper attach type for usdt probes. */ 383 SEC_USDT = 32, 384}; 385 386struct bpf_sec_def { 387 char *sec; 388 enum bpf_prog_type prog_type; 389 enum bpf_attach_type expected_attach_type; 390 long cookie; 391 int handler_id; 392 393 libbpf_prog_setup_fn_t prog_setup_fn; 394 libbpf_prog_prepare_load_fn_t prog_prepare_load_fn; 395 libbpf_prog_attach_fn_t prog_attach_fn; 396}; 397 398/* 399 * bpf_prog should be a better name but it has been used in 400 * linux/filter.h. 401 */ 402struct bpf_program { 403 char *name; 404 char *sec_name; 405 size_t sec_idx; 406 const struct bpf_sec_def *sec_def; 407 /* this program's instruction offset (in number of instructions) 408 * within its containing ELF section 409 */ 410 size_t sec_insn_off; 411 /* number of original instructions in ELF section belonging to this 412 * program, not taking into account subprogram instructions possible 413 * appended later during relocation 414 */ 415 size_t sec_insn_cnt; 416 /* Offset (in number of instructions) of the start of instruction 417 * belonging to this BPF program within its containing main BPF 418 * program. For the entry-point (main) BPF program, this is always 419 * zero. For a sub-program, this gets reset before each of main BPF 420 * programs are processed and relocated and is used to determined 421 * whether sub-program was already appended to the main program, and 422 * if yes, at which instruction offset. 423 */ 424 size_t sub_insn_off; 425 426 /* instructions that belong to BPF program; insns[0] is located at 427 * sec_insn_off instruction within its ELF section in ELF file, so 428 * when mapping ELF file instruction index to the local instruction, 429 * one needs to subtract sec_insn_off; and vice versa. 430 */ 431 struct bpf_insn *insns; 432 /* actual number of instruction in this BPF program's image; for 433 * entry-point BPF programs this includes the size of main program 434 * itself plus all the used sub-programs, appended at the end 435 */ 436 size_t insns_cnt; 437 438 struct reloc_desc *reloc_desc; 439 int nr_reloc; 440 441 /* BPF verifier log settings */ 442 char *log_buf; 443 size_t log_size; 444 __u32 log_level; 445 446 struct bpf_object *obj; 447 448 int fd; 449 bool autoload; 450 bool autoattach; 451 bool sym_global; 452 bool mark_btf_static; 453 enum bpf_prog_type type; 454 enum bpf_attach_type expected_attach_type; 455 int exception_cb_idx; 456 457 int prog_ifindex; 458 __u32 attach_btf_obj_fd; 459 __u32 attach_btf_id; 460 __u32 attach_prog_fd; 461 462 void *func_info; 463 __u32 func_info_rec_size; 464 __u32 func_info_cnt; 465 466 void *line_info; 467 __u32 line_info_rec_size; 468 __u32 line_info_cnt; 469 __u32 prog_flags; 470}; 471 472struct bpf_struct_ops { 473 const char *tname; 474 const struct btf_type *type; 475 struct bpf_program **progs; 476 __u32 *kern_func_off; 477 /* e.g. struct tcp_congestion_ops in bpf_prog's btf format */ 478 void *data; 479 /* e.g. struct bpf_struct_ops_tcp_congestion_ops in 480 * btf_vmlinux's format. 481 * struct bpf_struct_ops_tcp_congestion_ops { 482 * [... some other kernel fields ...] 483 * struct tcp_congestion_ops data; 484 * } 485 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops) 486 * bpf_map__init_kern_struct_ops() will populate the "kern_vdata" 487 * from "data". 488 */ 489 void *kern_vdata; 490 __u32 type_id; 491}; 492 493#define DATA_SEC ".data" 494#define BSS_SEC ".bss" 495#define RODATA_SEC ".rodata" 496#define KCONFIG_SEC ".kconfig" 497#define KSYMS_SEC ".ksyms" 498#define STRUCT_OPS_SEC ".struct_ops" 499#define STRUCT_OPS_LINK_SEC ".struct_ops.link" 500 501enum libbpf_map_type { 502 LIBBPF_MAP_UNSPEC, 503 LIBBPF_MAP_DATA, 504 LIBBPF_MAP_BSS, 505 LIBBPF_MAP_RODATA, 506 LIBBPF_MAP_KCONFIG, 507}; 508 509struct bpf_map_def { 510 unsigned int type; 511 unsigned int key_size; 512 unsigned int value_size; 513 unsigned int max_entries; 514 unsigned int map_flags; 515}; 516 517struct bpf_map { 518 struct bpf_object *obj; 519 char *name; 520 /* real_name is defined for special internal maps (.rodata*, 521 * .data*, .bss, .kconfig) and preserves their original ELF section 522 * name. This is important to be able to find corresponding BTF 523 * DATASEC information. 524 */ 525 char *real_name; 526 int fd; 527 int sec_idx; 528 size_t sec_offset; 529 int map_ifindex; 530 int inner_map_fd; 531 struct bpf_map_def def; 532 __u32 numa_node; 533 __u32 btf_var_idx; 534 __u32 btf_key_type_id; 535 __u32 btf_value_type_id; 536 __u32 btf_vmlinux_value_type_id; 537 enum libbpf_map_type libbpf_type; 538 void *mmaped; 539 struct bpf_struct_ops *st_ops; 540 struct bpf_map *inner_map; 541 void **init_slots; 542 int init_slots_sz; 543 char *pin_path; 544 bool pinned; 545 bool reused; 546 bool autocreate; 547 __u64 map_extra; 548}; 549 550enum extern_type { 551 EXT_UNKNOWN, 552 EXT_KCFG, 553 EXT_KSYM, 554}; 555 556enum kcfg_type { 557 KCFG_UNKNOWN, 558 KCFG_CHAR, 559 KCFG_BOOL, 560 KCFG_INT, 561 KCFG_TRISTATE, 562 KCFG_CHAR_ARR, 563}; 564 565struct extern_desc { 566 enum extern_type type; 567 int sym_idx; 568 int btf_id; 569 int sec_btf_id; 570 const char *name; 571 char *essent_name; 572 bool is_set; 573 bool is_weak; 574 union { 575 struct { 576 enum kcfg_type type; 577 int sz; 578 int align; 579 int data_off; 580 bool is_signed; 581 } kcfg; 582 struct { 583 unsigned long long addr; 584 585 /* target btf_id of the corresponding kernel var. */ 586 int kernel_btf_obj_fd; 587 int kernel_btf_id; 588 589 /* local btf_id of the ksym extern's type. */ 590 __u32 type_id; 591 /* BTF fd index to be patched in for insn->off, this is 592 * 0 for vmlinux BTF, index in obj->fd_array for module 593 * BTF 594 */ 595 __s16 btf_fd_idx; 596 } ksym; 597 }; 598}; 599 600struct module_btf { 601 struct btf *btf; 602 char *name; 603 __u32 id; 604 int fd; 605 int fd_array_idx; 606}; 607 608enum sec_type { 609 SEC_UNUSED = 0, 610 SEC_RELO, 611 SEC_BSS, 612 SEC_DATA, 613 SEC_RODATA, 614}; 615 616struct elf_sec_desc { 617 enum sec_type sec_type; 618#if defined HAVE_LIBELF 619 Elf64_Shdr *shdr; 620#elif defined HAVE_ELFIO 621 psection_t psection; 622 Elf_Data realdata; 623#endif 624 Elf_Data *data; 625}; 626 627struct elf_state { 628 int fd; 629 const void *obj_buf; 630 size_t obj_buf_sz; 631#if defined HAVE_LIBELF 632 Elf *elf; 633#elif defined HAVE_ELFIO 634 pelfio_t elf; 635 Elf64_Ehdr eheader; 636 pstring_t shstring; 637 pstring_t strstring; 638 Elf_Data realsymbols; 639 Elf_Data realst_ops_data; 640#endif 641 Elf64_Ehdr *ehdr; 642 Elf_Data *symbols; 643 Elf_Data *st_ops_data; 644 Elf_Data *st_ops_link_data; 645 size_t shstrndx; /* section index for section name strings */ 646 size_t strtabidx; 647 struct elf_sec_desc *secs; 648 size_t sec_cnt; 649 int btf_maps_shndx; 650 __u32 btf_maps_sec_btf_id; 651 int text_shndx; 652 int symbols_shndx; 653 int st_ops_shndx; 654 int st_ops_link_shndx; 655}; 656 657struct usdt_manager; 658 659struct bpf_object { 660 char name[BPF_OBJ_NAME_LEN]; 661 char license[64]; 662 __u32 kern_version; 663 664 struct bpf_program *programs; 665 size_t nr_programs; 666 struct bpf_map *maps; 667 size_t nr_maps; 668 size_t maps_cap; 669 670 char *kconfig; 671 struct extern_desc *externs; 672 int nr_extern; 673 int kconfig_map_idx; 674 675 bool loaded; 676 bool has_subcalls; 677 bool has_rodata; 678 679 struct bpf_gen *gen_loader; 680 681 /* Information when doing ELF related work. Only valid if efile.elf is not NULL */ 682 struct elf_state efile; 683 684 struct btf *btf; 685 struct btf_ext *btf_ext; 686 687 /* Parse and load BTF vmlinux if any of the programs in the object need 688 * it at load time. 689 */ 690 struct btf *btf_vmlinux; 691 /* Path to the custom BTF to be used for BPF CO-RE relocations as an 692 * override for vmlinux BTF. 693 */ 694 char *btf_custom_path; 695 /* vmlinux BTF override for CO-RE relocations */ 696 struct btf *btf_vmlinux_override; 697 /* Lazily initialized kernel module BTFs */ 698 struct module_btf *btf_modules; 699 bool btf_modules_loaded; 700 size_t btf_module_cnt; 701 size_t btf_module_cap; 702 703 /* optional log settings passed to BPF_BTF_LOAD and BPF_PROG_LOAD commands */ 704 char *log_buf; 705 size_t log_size; 706 __u32 log_level; 707 708 int *fd_array; 709 size_t fd_array_cap; 710 size_t fd_array_cnt; 711 712 struct usdt_manager *usdt_man; 713 714 char path[]; 715}; 716 717static const char *elf_sym_str(const struct bpf_object *obj, size_t off); 718static const char *elf_sec_str(const struct bpf_object *obj, size_t off); 719#ifdef HAVE_LIBELF 720static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx); 721static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name); 722#endif 723#if defined HAVE_LIBELF 724static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn); 725#elif defined HAVE_ELFIO 726static Elf64_Shdr *elf_sec_hdr_by_idx(const struct bpf_object *obj, size_t idx, Elf64_Shdr *sheader); 727#endif 728#if defined HAVE_LIBELF 729static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn); 730#elif defined HAVE_ELFIO 731static const char *elf_sec_name_by_idx(const struct bpf_object *obj, size_t idx); 732#endif 733#if defined HAVE_LIBELF 734static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn); 735#elif defined HAVE_ELFIO 736static Elf_Data *elf_sec_data_by_name(const struct bpf_object *obj, const char *name, Elf_Data *data); 737static Elf_Data *elf_sec_data_by_idx(const struct bpf_object *obj, size_t idx, Elf_Data *data); 738#endif 739static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx); 740static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx); 741 742void bpf_program__unload(struct bpf_program *prog) 743{ 744 if (!prog) 745 return; 746 747 zclose(prog->fd); 748 749 zfree(&prog->func_info); 750 zfree(&prog->line_info); 751} 752 753static void bpf_program__exit(struct bpf_program *prog) 754{ 755 if (!prog) 756 return; 757 758 bpf_program__unload(prog); 759 zfree(&prog->name); 760 zfree(&prog->sec_name); 761 zfree(&prog->insns); 762 zfree(&prog->reloc_desc); 763 764 prog->nr_reloc = 0; 765 prog->insns_cnt = 0; 766 prog->sec_idx = -1; 767} 768 769static bool insn_is_subprog_call(const struct bpf_insn *insn) 770{ 771 return BPF_CLASS(insn->code) == BPF_JMP && 772 BPF_OP(insn->code) == BPF_CALL && 773 BPF_SRC(insn->code) == BPF_K && 774 insn->src_reg == BPF_PSEUDO_CALL && 775 insn->dst_reg == 0 && 776 insn->off == 0; 777} 778 779static bool is_call_insn(const struct bpf_insn *insn) 780{ 781 return insn->code == (BPF_JMP | BPF_CALL); 782} 783 784static bool insn_is_pseudo_func(struct bpf_insn *insn) 785{ 786 return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC; 787} 788 789static int 790bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, 791 const char *name, size_t sec_idx, const char *sec_name, 792 size_t sec_off, void *insn_data, size_t insn_data_sz) 793{ 794 if (insn_data_sz == 0 || insn_data_sz % BPF_INSN_SZ || sec_off % BPF_INSN_SZ) { 795 pr_warn("sec '%s': corrupted program '%s', offset %zu, size %zu\n", 796 sec_name, name, sec_off, insn_data_sz); 797 return -EINVAL; 798 } 799 800 memset(prog, 0, sizeof(*prog)); 801 prog->obj = obj; 802 803 prog->sec_idx = sec_idx; 804 prog->sec_insn_off = sec_off / BPF_INSN_SZ; 805 prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ; 806 /* insns_cnt can later be increased by appending used subprograms */ 807 prog->insns_cnt = prog->sec_insn_cnt; 808 809 prog->type = BPF_PROG_TYPE_UNSPEC; 810 prog->fd = -1; 811 prog->exception_cb_idx = -1; 812 813 /* libbpf's convention for SEC("?abc...") is that it's just like 814 * SEC("abc...") but the corresponding bpf_program starts out with 815 * autoload set to false. 816 */ 817 if (sec_name[0] == '?') { 818 prog->autoload = false; 819 /* from now on forget there was ? in section name */ 820 sec_name++; 821 } else { 822 prog->autoload = true; 823 } 824 825 prog->autoattach = true; 826 827 /* inherit object's log_level */ 828 prog->log_level = obj->log_level; 829 830 prog->sec_name = strdup(sec_name); 831 if (!prog->sec_name) 832 goto errout; 833 834 prog->name = strdup(name); 835 if (!prog->name) 836 goto errout; 837 838 prog->insns = malloc(insn_data_sz); 839 if (!prog->insns) 840 goto errout; 841 memcpy(prog->insns, insn_data, insn_data_sz); 842 843 return 0; 844errout: 845 pr_warn("sec '%s': failed to allocate memory for prog '%s'\n", sec_name, name); 846 bpf_program__exit(prog); 847 return -ENOMEM; 848} 849 850static int 851bpf_object__add_programs(struct bpf_object *obj, Elf_Data *sec_data, 852 const char *sec_name, int sec_idx) 853{ 854 Elf_Data *symbols = obj->efile.symbols; 855 struct bpf_program *prog, *progs; 856 void *data = sec_data->d_buf; 857 size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms; 858 int nr_progs, err, i; 859 const char *name; 860 Elf64_Sym *sym; 861 862 progs = obj->programs; 863 nr_progs = obj->nr_programs; 864 nr_syms = symbols->d_size / sizeof(Elf64_Sym); 865 866 for (i = 0; i < nr_syms; i++) { 867 sym = elf_sym_by_idx(obj, i); 868 869 if (sym->st_shndx != sec_idx) 870 continue; 871 if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC) 872 continue; 873 874 prog_sz = sym->st_size; 875 sec_off = sym->st_value; 876 877 name = elf_sym_str(obj, sym->st_name); 878 if (!name) { 879 pr_warn("sec '%s': failed to get symbol name for offset %zu\n", 880 sec_name, sec_off); 881 return -LIBBPF_ERRNO__FORMAT; 882 } 883 884 if (sec_off + prog_sz > sec_sz) { 885 pr_warn("sec '%s': program at offset %zu crosses section boundary\n", 886 sec_name, sec_off); 887 return -LIBBPF_ERRNO__FORMAT; 888 } 889 890 if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { 891 pr_warn("sec '%s': program '%s' is static and not supported\n", sec_name, name); 892 return -ENOTSUP; 893 } 894 895 pr_debug("sec '%s': found program '%s' at insn offset %zu (%zu bytes), code size %zu insns (%zu bytes)\n", 896 sec_name, name, sec_off / BPF_INSN_SZ, sec_off, prog_sz / BPF_INSN_SZ, prog_sz); 897 898 progs = libbpf_reallocarray(progs, nr_progs + 1, sizeof(*progs)); 899 if (!progs) { 900 /* 901 * In this case the original obj->programs 902 * is still valid, so don't need special treat for 903 * bpf_close_object(). 904 */ 905 pr_warn("sec '%s': failed to alloc memory for new program '%s'\n", 906 sec_name, name); 907 return -ENOMEM; 908 } 909 obj->programs = progs; 910 911 prog = &progs[nr_progs]; 912 913 err = bpf_object__init_prog(obj, prog, name, sec_idx, sec_name, 914 sec_off, data + sec_off, prog_sz); 915 if (err) 916 return err; 917 918 if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL) 919 prog->sym_global = true; 920 921 /* if function is a global/weak symbol, but has restricted 922 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF FUNC 923 * as static to enable more permissive BPF verification mode 924 * with more outside context available to BPF verifier 925 */ 926 if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 927 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)) 928 prog->mark_btf_static = true; 929 930 nr_progs++; 931 obj->nr_programs = nr_progs; 932 } 933 934 return 0; 935} 936 937static const struct btf_member * 938find_member_by_offset(const struct btf_type *t, __u32 bit_offset) 939{ 940 struct btf_member *m; 941 int i; 942 943 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 944 if (btf_member_bit_offset(t, i) == bit_offset) 945 return m; 946 } 947 948 return NULL; 949} 950 951static const struct btf_member * 952find_member_by_name(const struct btf *btf, const struct btf_type *t, 953 const char *name) 954{ 955 struct btf_member *m; 956 int i; 957 958 for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { 959 if (!strcmp(btf__name_by_offset(btf, m->name_off), name)) 960 return m; 961 } 962 963 return NULL; 964} 965 966#define STRUCT_OPS_VALUE_PREFIX "bpf_struct_ops_" 967static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 968 const char *name, __u32 kind); 969 970static int 971find_struct_ops_kern_types(const struct btf *btf, const char *tname, 972 const struct btf_type **type, __u32 *type_id, 973 const struct btf_type **vtype, __u32 *vtype_id, 974 const struct btf_member **data_member) 975{ 976 const struct btf_type *kern_type, *kern_vtype; 977 const struct btf_member *kern_data_member; 978 __s32 kern_vtype_id, kern_type_id; 979 __u32 i; 980 981 kern_type_id = btf__find_by_name_kind(btf, tname, BTF_KIND_STRUCT); 982 if (kern_type_id < 0) { 983 pr_warn("struct_ops init_kern: struct %s is not found in kernel BTF\n", 984 tname); 985 return kern_type_id; 986 } 987 kern_type = btf__type_by_id(btf, kern_type_id); 988 989 /* Find the corresponding "map_value" type that will be used 990 * in map_update(BPF_MAP_TYPE_STRUCT_OPS). For example, 991 * find "struct bpf_struct_ops_tcp_congestion_ops" from the 992 * btf_vmlinux. 993 */ 994 kern_vtype_id = find_btf_by_prefix_kind(btf, STRUCT_OPS_VALUE_PREFIX, 995 tname, BTF_KIND_STRUCT); 996 if (kern_vtype_id < 0) { 997 pr_warn("struct_ops init_kern: struct %s%s is not found in kernel BTF\n", 998 STRUCT_OPS_VALUE_PREFIX, tname); 999 return kern_vtype_id; 1000 } 1001 kern_vtype = btf__type_by_id(btf, kern_vtype_id); 1002 1003 /* Find "struct tcp_congestion_ops" from 1004 * struct bpf_struct_ops_tcp_congestion_ops { 1005 * [ ... ] 1006 * struct tcp_congestion_ops data; 1007 * } 1008 */ 1009 kern_data_member = btf_members(kern_vtype); 1010 for (i = 0; i < btf_vlen(kern_vtype); i++, kern_data_member++) { 1011 if (kern_data_member->type == kern_type_id) 1012 break; 1013 } 1014 if (i == btf_vlen(kern_vtype)) { 1015 pr_warn("struct_ops init_kern: struct %s data is not found in struct %s%s\n", 1016 tname, STRUCT_OPS_VALUE_PREFIX, tname); 1017 return -EINVAL; 1018 } 1019 1020 *type = kern_type; 1021 *type_id = kern_type_id; 1022 *vtype = kern_vtype; 1023 *vtype_id = kern_vtype_id; 1024 *data_member = kern_data_member; 1025 1026 return 0; 1027} 1028 1029static bool bpf_map__is_struct_ops(const struct bpf_map *map) 1030{ 1031 return map->def.type == BPF_MAP_TYPE_STRUCT_OPS; 1032} 1033 1034/* Init the map's fields that depend on kern_btf */ 1035static int bpf_map__init_kern_struct_ops(struct bpf_map *map, 1036 const struct btf *btf, 1037 const struct btf *kern_btf) 1038{ 1039 const struct btf_member *member, *kern_member, *kern_data_member; 1040 const struct btf_type *type, *kern_type, *kern_vtype; 1041 __u32 i, kern_type_id, kern_vtype_id, kern_data_off; 1042 struct bpf_struct_ops *st_ops; 1043 void *data, *kern_data; 1044 const char *tname; 1045 int err; 1046 1047 st_ops = map->st_ops; 1048 type = st_ops->type; 1049 tname = st_ops->tname; 1050 err = find_struct_ops_kern_types(kern_btf, tname, 1051 &kern_type, &kern_type_id, 1052 &kern_vtype, &kern_vtype_id, 1053 &kern_data_member); 1054 if (err) 1055 return err; 1056 1057 pr_debug("struct_ops init_kern %s: type_id:%u kern_type_id:%u kern_vtype_id:%u\n", 1058 map->name, st_ops->type_id, kern_type_id, kern_vtype_id); 1059 1060 map->def.value_size = kern_vtype->size; 1061 map->btf_vmlinux_value_type_id = kern_vtype_id; 1062 1063 st_ops->kern_vdata = calloc(1, kern_vtype->size); 1064 if (!st_ops->kern_vdata) 1065 return -ENOMEM; 1066 1067 data = st_ops->data; 1068 kern_data_off = kern_data_member->offset / 8; 1069 kern_data = st_ops->kern_vdata + kern_data_off; 1070 1071 member = btf_members(type); 1072 for (i = 0; i < btf_vlen(type); i++, member++) { 1073 const struct btf_type *mtype, *kern_mtype; 1074 __u32 mtype_id, kern_mtype_id; 1075 void *mdata, *kern_mdata; 1076 __s64 msize, kern_msize; 1077 __u32 moff, kern_moff; 1078 __u32 kern_member_idx; 1079 const char *mname; 1080 1081 mname = btf__name_by_offset(btf, member->name_off); 1082 kern_member = find_member_by_name(kern_btf, kern_type, mname); 1083 if (!kern_member) { 1084 pr_warn("struct_ops init_kern %s: Cannot find member %s in kernel BTF\n", 1085 map->name, mname); 1086 return -ENOTSUP; 1087 } 1088 1089 kern_member_idx = kern_member - btf_members(kern_type); 1090 if (btf_member_bitfield_size(type, i) || 1091 btf_member_bitfield_size(kern_type, kern_member_idx)) { 1092 pr_warn("struct_ops init_kern %s: bitfield %s is not supported\n", 1093 map->name, mname); 1094 return -ENOTSUP; 1095 } 1096 1097 moff = member->offset / 8; 1098 kern_moff = kern_member->offset / 8; 1099 1100 mdata = data + moff; 1101 kern_mdata = kern_data + kern_moff; 1102 1103 mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id); 1104 kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type, 1105 &kern_mtype_id); 1106 if (BTF_INFO_KIND(mtype->info) != 1107 BTF_INFO_KIND(kern_mtype->info)) { 1108 pr_warn("struct_ops init_kern %s: Unmatched member type %s %u != %u(kernel)\n", 1109 map->name, mname, BTF_INFO_KIND(mtype->info), 1110 BTF_INFO_KIND(kern_mtype->info)); 1111 return -ENOTSUP; 1112 } 1113 1114 if (btf_is_ptr(mtype)) { 1115 struct bpf_program *prog; 1116 1117 prog = st_ops->progs[i]; 1118 if (!prog) 1119 continue; 1120 1121 kern_mtype = skip_mods_and_typedefs(kern_btf, 1122 kern_mtype->type, 1123 &kern_mtype_id); 1124 1125 /* mtype->type must be a func_proto which was 1126 * guaranteed in bpf_object__collect_st_ops_relos(), 1127 * so only check kern_mtype for func_proto here. 1128 */ 1129 if (!btf_is_func_proto(kern_mtype)) { 1130 pr_warn("struct_ops init_kern %s: kernel member %s is not a func ptr\n", 1131 map->name, mname); 1132 return -ENOTSUP; 1133 } 1134 1135 prog->attach_btf_id = kern_type_id; 1136 prog->expected_attach_type = kern_member_idx; 1137 1138 st_ops->kern_func_off[i] = kern_data_off + kern_moff; 1139 1140 pr_debug("struct_ops init_kern %s: func ptr %s is set to prog %s from data(+%u) to kern_data(+%u)\n", 1141 map->name, mname, prog->name, moff, 1142 kern_moff); 1143 1144 continue; 1145 } 1146 1147 msize = btf__resolve_size(btf, mtype_id); 1148 kern_msize = btf__resolve_size(kern_btf, kern_mtype_id); 1149 if (msize < 0 || kern_msize < 0 || msize != kern_msize) { 1150 pr_warn("struct_ops init_kern %s: Error in size of member %s: %zd != %zd(kernel)\n", 1151 map->name, mname, (ssize_t)msize, 1152 (ssize_t)kern_msize); 1153 return -ENOTSUP; 1154 } 1155 1156 pr_debug("struct_ops init_kern %s: copy %s %u bytes from data(+%u) to kern_data(+%u)\n", 1157 map->name, mname, (unsigned int)msize, 1158 moff, kern_moff); 1159 memcpy(kern_mdata, mdata, msize); 1160 } 1161 1162 return 0; 1163} 1164 1165static int bpf_object__init_kern_struct_ops_maps(struct bpf_object *obj) 1166{ 1167 struct bpf_map *map; 1168 size_t i; 1169 int err; 1170 1171 for (i = 0; i < obj->nr_maps; i++) { 1172 map = &obj->maps[i]; 1173 1174 if (!bpf_map__is_struct_ops(map)) 1175 continue; 1176 1177 err = bpf_map__init_kern_struct_ops(map, obj->btf, 1178 obj->btf_vmlinux); 1179 if (err) 1180 return err; 1181 } 1182 1183 return 0; 1184} 1185 1186static int init_struct_ops_maps(struct bpf_object *obj, const char *sec_name, 1187 int shndx, Elf_Data *data, __u32 map_flags) 1188{ 1189 const struct btf_type *type, *datasec; 1190 const struct btf_var_secinfo *vsi; 1191 struct bpf_struct_ops *st_ops; 1192 const char *tname, *var_name; 1193 __s32 type_id, datasec_id; 1194 const struct btf *btf; 1195 struct bpf_map *map; 1196 __u32 i; 1197 1198 if (shndx == -1) 1199 return 0; 1200 1201 btf = obj->btf; 1202 datasec_id = btf__find_by_name_kind(btf, sec_name, 1203 BTF_KIND_DATASEC); 1204 if (datasec_id < 0) { 1205 pr_warn("struct_ops init: DATASEC %s not found\n", 1206 sec_name); 1207 return -EINVAL; 1208 } 1209 1210 datasec = btf__type_by_id(btf, datasec_id); 1211 vsi = btf_var_secinfos(datasec); 1212 for (i = 0; i < btf_vlen(datasec); i++, vsi++) { 1213 type = btf__type_by_id(obj->btf, vsi->type); 1214 var_name = btf__name_by_offset(obj->btf, type->name_off); 1215 1216 type_id = btf__resolve_type(obj->btf, vsi->type); 1217 if (type_id < 0) { 1218 pr_warn("struct_ops init: Cannot resolve var type_id %u in DATASEC %s\n", 1219 vsi->type, sec_name); 1220 return -EINVAL; 1221 } 1222 1223 type = btf__type_by_id(obj->btf, type_id); 1224 tname = btf__name_by_offset(obj->btf, type->name_off); 1225 if (!tname[0]) { 1226 pr_warn("struct_ops init: anonymous type is not supported\n"); 1227 return -ENOTSUP; 1228 } 1229 if (!btf_is_struct(type)) { 1230 pr_warn("struct_ops init: %s is not a struct\n", tname); 1231 return -EINVAL; 1232 } 1233 1234 map = bpf_object__add_map(obj); 1235 if (IS_ERR(map)) 1236 return PTR_ERR(map); 1237 1238 map->sec_idx = shndx; 1239 map->sec_offset = vsi->offset; 1240 map->name = strdup(var_name); 1241 if (!map->name) 1242 return -ENOMEM; 1243 1244 map->def.type = BPF_MAP_TYPE_STRUCT_OPS; 1245 map->def.key_size = sizeof(int); 1246 map->def.value_size = type->size; 1247 map->def.max_entries = 1; 1248 map->def.map_flags = map_flags; 1249 1250 map->st_ops = calloc(1, sizeof(*map->st_ops)); 1251 if (!map->st_ops) 1252 return -ENOMEM; 1253 st_ops = map->st_ops; 1254 st_ops->data = malloc(type->size); 1255 st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs)); 1256 st_ops->kern_func_off = malloc(btf_vlen(type) * 1257 sizeof(*st_ops->kern_func_off)); 1258 if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off) 1259 return -ENOMEM; 1260 1261 if (vsi->offset + type->size > data->d_size) { 1262 pr_warn("struct_ops init: var %s is beyond the end of DATASEC %s\n", 1263 var_name, sec_name); 1264 return -EINVAL; 1265 } 1266 1267 memcpy(st_ops->data, 1268 data->d_buf + vsi->offset, 1269 type->size); 1270 st_ops->tname = tname; 1271 st_ops->type = type; 1272 st_ops->type_id = type_id; 1273 1274 pr_debug("struct_ops init: struct %s(type_id=%u) %s found at offset %u\n", 1275 tname, type_id, var_name, vsi->offset); 1276 } 1277 1278 return 0; 1279} 1280 1281static int bpf_object_init_struct_ops(struct bpf_object *obj) 1282{ 1283 int err; 1284 1285 err = init_struct_ops_maps(obj, STRUCT_OPS_SEC, obj->efile.st_ops_shndx, 1286 obj->efile.st_ops_data, 0); 1287 err = err ?: init_struct_ops_maps(obj, STRUCT_OPS_LINK_SEC, 1288 obj->efile.st_ops_link_shndx, 1289 obj->efile.st_ops_link_data, 1290 BPF_F_LINK); 1291 return err; 1292} 1293 1294static struct bpf_object *bpf_object__new(const char *path, 1295 const void *obj_buf, 1296 size_t obj_buf_sz, 1297 const char *obj_name) 1298{ 1299 struct bpf_object *obj; 1300 char *end; 1301 1302 obj = calloc(1, sizeof(struct bpf_object) + strlen(path) + 1); 1303 if (!obj) { 1304 pr_warn("alloc memory failed for %s\n", path); 1305 return ERR_PTR(-ENOMEM); 1306 } 1307 1308 strcpy(obj->path, path); 1309 if (obj_name) { 1310 libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name)); 1311 } else { 1312 /* Using basename() GNU version which doesn't modify arg. */ 1313 libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name)); 1314 end = strchr(obj->name, '.'); 1315 if (end) 1316 *end = 0; 1317 } 1318 1319 obj->efile.fd = -1; 1320 /* 1321 * Caller of this function should also call 1322 * bpf_object__elf_finish() after data collection to return 1323 * obj_buf to user. If not, we should duplicate the buffer to 1324 * avoid user freeing them before elf finish. 1325 */ 1326 obj->efile.obj_buf = obj_buf; 1327 obj->efile.obj_buf_sz = obj_buf_sz; 1328 obj->efile.btf_maps_shndx = -1; 1329 obj->efile.st_ops_shndx = -1; 1330 obj->efile.st_ops_link_shndx = -1; 1331 obj->kconfig_map_idx = -1; 1332 1333 obj->kern_version = get_kernel_version(); 1334 obj->loaded = false; 1335 1336 return obj; 1337} 1338 1339static void bpf_object__elf_finish(struct bpf_object *obj) 1340{ 1341 if (!obj->efile.elf) 1342 return; 1343#if defined HAVE_LIBELF 1344 elf_end(obj->efile.elf); 1345#elif defined HAVE_ELFIO 1346 if (obj->efile.shstring) { 1347 elfio_string_section_accessor_delete(obj->efile.shstring); 1348 } 1349 if (obj->efile.strstring) { 1350 elfio_string_section_accessor_delete(obj->efile.strstring); 1351 } 1352 elfio_delete(obj->efile.elf); 1353#endif 1354 obj->efile.elf = NULL; 1355 obj->efile.symbols = NULL; 1356 obj->efile.st_ops_data = NULL; 1357 obj->efile.st_ops_link_data = NULL; 1358 1359 zfree(&obj->efile.secs); 1360 obj->efile.sec_cnt = 0; 1361 zclose(obj->efile.fd); 1362 obj->efile.obj_buf = NULL; 1363 obj->efile.obj_buf_sz = 0; 1364} 1365 1366static int bpf_object__elf_init(struct bpf_object *obj) 1367{ 1368 Elf64_Ehdr *ehdr; 1369 int err = 0; 1370#ifdef HAVE_LIBELF 1371 Elf *elf; 1372#elif defined HAVE_ELFIO 1373 pelfio_t elf; 1374#endif 1375 1376 if (obj->efile.elf) { 1377 pr_warn("elf: init internal error\n"); 1378 return -LIBBPF_ERRNO__LIBELF; 1379 } 1380 1381 if (obj->efile.obj_buf_sz > 0) { 1382 /* obj_buf should have been validated by bpf_object__open_mem(). */ 1383#ifdef HAVE_LIBELF 1384 elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); 1385#elif defined HAVE_ELFIO 1386 char memfd_path[PATH_MAX] = {0}; 1387 elf = elfio_new(); 1388 int fdm = syscall(__NR_memfd_create, "bpfelf", MFD_CLOEXEC); 1389 ftruncate(fdm, obj->efile.obj_buf_sz); 1390 write(fdm, (char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); 1391 snprintf(memfd_path, PATH_MAX, "/proc/self/fd/%d", fdm); 1392 elfio_load(elf, memfd_path); 1393#endif 1394 } else { 1395 obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC); 1396 if (obj->efile.fd < 0) { 1397 char errmsg[STRERR_BUFSIZE], *cp; 1398 1399 err = -errno; 1400 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 1401 pr_warn("elf: failed to open %s: %s\n", obj->path, cp); 1402 return err; 1403 } 1404#ifdef HAVE_LIBELF 1405 elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL); 1406#endif 1407 } 1408 1409 if (!elf) { 1410 pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1)); 1411 err = -LIBBPF_ERRNO__LIBELF; 1412 goto errout; 1413 } 1414 1415 obj->efile.elf = elf; 1416#ifdef HAVE_LIBELF 1417 if (elf_kind(elf) != ELF_K_ELF) { 1418 err = -LIBBPF_ERRNO__FORMAT; 1419 pr_warn("elf: '%s' is not a proper ELF object\n", obj->path); 1420 goto errout; 1421 } 1422 1423 if (gelf_getclass(elf) != ELFCLASS64) { 1424#elif defined HAVE_ELFIO 1425 if (elfio_get_class(elf) != ELFCLASS64 ) { 1426#endif 1427 err = -LIBBPF_ERRNO__FORMAT; 1428 pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path); 1429 goto errout; 1430 } 1431#ifdef HAVE_LIBELF 1432 obj->efile.ehdr = ehdr = elf64_getehdr(elf); 1433#elif defined HAVE_ELFIO 1434 obj->efile.ehdr = ehdr = (Elf64_Ehdr*)obj->efile.obj_buf; 1435#endif 1436 if (!obj->efile.ehdr) { 1437 pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1)); 1438 err = -LIBBPF_ERRNO__FORMAT; 1439 goto errout; 1440 } 1441 1442#ifdef HAVE_LIBELF 1443 if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) { 1444 pr_warn("elf: failed to get section names section index for %s: %s\n", 1445 obj->path, elf_errmsg(-1)); 1446 err = -LIBBPF_ERRNO__FORMAT; 1447 goto errout; 1448 } 1449 1450 /* Elf is corrupted/truncated, avoid calling elf_strptr. */ 1451 if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) { 1452 pr_warn("elf: failed to get section names strings from %s: %s\n", 1453 obj->path, elf_errmsg(-1)); 1454 err = -LIBBPF_ERRNO__FORMAT; 1455 goto errout; 1456 } 1457#elif defined HAVE_ELFIO 1458 obj->efile.shstrndx = elfio_get_section_name_str_index(elf); 1459#endif 1460 /* Old LLVM set e_machine to EM_NONE */ 1461 if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) { 1462 pr_warn("elf: %s is not a valid eBPF object file\n", obj->path); 1463 err = -LIBBPF_ERRNO__FORMAT; 1464 goto errout; 1465 } 1466 1467 return 0; 1468errout: 1469 bpf_object__elf_finish(obj); 1470 return err; 1471} 1472 1473static int bpf_object__check_endianness(struct bpf_object *obj) 1474{ 1475#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 1476 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB) 1477 return 0; 1478#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 1479 if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB) 1480 return 0; 1481#else 1482# error "Unrecognized __BYTE_ORDER__" 1483#endif 1484 pr_warn("elf: endianness mismatch in %s.\n", obj->path); 1485 return -LIBBPF_ERRNO__ENDIAN; 1486} 1487 1488static int 1489bpf_object__init_license(struct bpf_object *obj, void *data, size_t size) 1490{ 1491 if (!data) { 1492 pr_warn("invalid license section in %s\n", obj->path); 1493 return -LIBBPF_ERRNO__FORMAT; 1494 } 1495 /* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't 1496 * go over allowed ELF data section buffer 1497 */ 1498 libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license))); 1499 pr_debug("license of %s is %s\n", obj->path, obj->license); 1500 return 0; 1501} 1502 1503static int 1504bpf_object__init_kversion(struct bpf_object *obj, void *data, size_t size) 1505{ 1506 __u32 kver; 1507 1508 if (!data || size != sizeof(kver)) { 1509 pr_warn("invalid kver section in %s\n", obj->path); 1510 return -LIBBPF_ERRNO__FORMAT; 1511 } 1512 memcpy(&kver, data, sizeof(kver)); 1513 obj->kern_version = kver; 1514 pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version); 1515 return 0; 1516} 1517 1518static bool bpf_map_type__is_map_in_map(enum bpf_map_type type) 1519{ 1520 if (type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 1521 type == BPF_MAP_TYPE_HASH_OF_MAPS) 1522 return true; 1523 return false; 1524} 1525 1526static int find_elf_sec_sz(const struct bpf_object *obj, const char *name, __u32 *size) 1527{ 1528 Elf_Data *data; 1529#ifdef HAVE_LIBELF 1530 Elf_Scn *scn; 1531#endif 1532 1533 if (!name) 1534 return -EINVAL; 1535#if defined HAVE_LIBELF 1536 scn = elf_sec_by_name(obj, name); 1537 data = elf_sec_data(obj, scn); 1538#elif defined HAVE_ELFIO 1539 Elf_Data realdata; 1540 data = &realdata; 1541 data = elf_sec_data_by_name(obj, name, data); 1542#endif 1543 if (data) { 1544 *size = data->d_size; 1545 return 0; /* found it */ 1546 } 1547 1548 return -ENOENT; 1549} 1550 1551static Elf64_Sym *find_elf_var_sym(const struct bpf_object *obj, const char *name) 1552{ 1553 Elf_Data *symbols = obj->efile.symbols; 1554 const char *sname; 1555 size_t si; 1556 1557 for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) { 1558 Elf64_Sym *sym = elf_sym_by_idx(obj, si); 1559 1560 if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT) 1561 continue; 1562 1563 if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && 1564 ELF64_ST_BIND(sym->st_info) != STB_WEAK) 1565 continue; 1566 1567 sname = elf_sym_str(obj, sym->st_name); 1568 if (!sname) { 1569 pr_warn("failed to get sym name string for var %s\n", name); 1570 return ERR_PTR(-EIO); 1571 } 1572 if (strcmp(name, sname) == 0) 1573 return sym; 1574 } 1575 1576 return ERR_PTR(-ENOENT); 1577} 1578 1579static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) 1580{ 1581 struct bpf_map *map; 1582 int err; 1583 1584 err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap, 1585 sizeof(*obj->maps), obj->nr_maps + 1); 1586 if (err) 1587 return ERR_PTR(err); 1588 1589 map = &obj->maps[obj->nr_maps++]; 1590 map->obj = obj; 1591 map->fd = -1; 1592 map->inner_map_fd = -1; 1593 map->autocreate = true; 1594 1595 return map; 1596} 1597 1598static size_t bpf_map_mmap_sz(unsigned int value_sz, unsigned int max_entries) 1599{ 1600 const long page_sz = sysconf(_SC_PAGE_SIZE); 1601 size_t map_sz; 1602 1603 map_sz = (size_t)roundup(value_sz, 8) * max_entries; 1604 map_sz = roundup(map_sz, page_sz); 1605 return map_sz; 1606} 1607 1608static int bpf_map_mmap_resize(struct bpf_map *map, size_t old_sz, size_t new_sz) 1609{ 1610 void *mmaped; 1611 1612 if (!map->mmaped) 1613 return -EINVAL; 1614 1615 if (old_sz == new_sz) 1616 return 0; 1617 1618 mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1619 if (mmaped == MAP_FAILED) 1620 return -errno; 1621 1622 memcpy(mmaped, map->mmaped, min(old_sz, new_sz)); 1623 munmap(map->mmaped, old_sz); 1624 map->mmaped = mmaped; 1625 return 0; 1626} 1627 1628static char *internal_map_name(struct bpf_object *obj, const char *real_name) 1629{ 1630 char map_name[BPF_OBJ_NAME_LEN], *p; 1631 int pfx_len, sfx_len = max((size_t)7, strlen(real_name)); 1632 1633 /* This is one of the more confusing parts of libbpf for various 1634 * reasons, some of which are historical. The original idea for naming 1635 * internal names was to include as much of BPF object name prefix as 1636 * possible, so that it can be distinguished from similar internal 1637 * maps of a different BPF object. 1638 * As an example, let's say we have bpf_object named 'my_object_name' 1639 * and internal map corresponding to '.rodata' ELF section. The final 1640 * map name advertised to user and to the kernel will be 1641 * 'my_objec.rodata', taking first 8 characters of object name and 1642 * entire 7 characters of '.rodata'. 1643 * Somewhat confusingly, if internal map ELF section name is shorter 1644 * than 7 characters, e.g., '.bss', we still reserve 7 characters 1645 * for the suffix, even though we only have 4 actual characters, and 1646 * resulting map will be called 'my_objec.bss', not even using all 15 1647 * characters allowed by the kernel. Oh well, at least the truncated 1648 * object name is somewhat consistent in this case. But if the map 1649 * name is '.kconfig', we'll still have entirety of '.kconfig' added 1650 * (8 chars) and thus will be left with only first 7 characters of the 1651 * object name ('my_obje'). Happy guessing, user, that the final map 1652 * name will be "my_obje.kconfig". 1653 * Now, with libbpf starting to support arbitrarily named .rodata.* 1654 * and .data.* data sections, it's possible that ELF section name is 1655 * longer than allowed 15 chars, so we now need to be careful to take 1656 * only up to 15 first characters of ELF name, taking no BPF object 1657 * name characters at all. So '.rodata.abracadabra' will result in 1658 * '.rodata.abracad' kernel and user-visible name. 1659 * We need to keep this convoluted logic intact for .data, .bss and 1660 * .rodata maps, but for new custom .data.custom and .rodata.custom 1661 * maps we use their ELF names as is, not prepending bpf_object name 1662 * in front. We still need to truncate them to 15 characters for the 1663 * kernel. Full name can be recovered for such maps by using DATASEC 1664 * BTF type associated with such map's value type, though. 1665 */ 1666 if (sfx_len >= BPF_OBJ_NAME_LEN) 1667 sfx_len = BPF_OBJ_NAME_LEN - 1; 1668 1669 /* if there are two or more dots in map name, it's a custom dot map */ 1670 if (strchr(real_name + 1, '.') != NULL) 1671 pfx_len = 0; 1672 else 1673 pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name)); 1674 1675 snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name, 1676 sfx_len, real_name); 1677 1678 /* sanitise map name to characters allowed by kernel */ 1679 for (p = map_name; *p && p < map_name + sizeof(map_name); p++) 1680 if (!isalnum(*p) && *p != '_' && *p != '.') 1681 *p = '_'; 1682 1683 return strdup(map_name); 1684} 1685 1686static int 1687map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map); 1688 1689/* Internal BPF map is mmap()'able only if at least one of corresponding 1690 * DATASEC's VARs are to be exposed through BPF skeleton. I.e., it's a GLOBAL 1691 * variable and it's not marked as __hidden (which turns it into, effectively, 1692 * a STATIC variable). 1693 */ 1694static bool map_is_mmapable(struct bpf_object *obj, struct bpf_map *map) 1695{ 1696 const struct btf_type *t, *vt; 1697 struct btf_var_secinfo *vsi; 1698 int i, n; 1699 1700 if (!map->btf_value_type_id) 1701 return false; 1702 1703 t = btf__type_by_id(obj->btf, map->btf_value_type_id); 1704 if (!btf_is_datasec(t)) 1705 return false; 1706 1707 vsi = btf_var_secinfos(t); 1708 for (i = 0, n = btf_vlen(t); i < n; i++, vsi++) { 1709 vt = btf__type_by_id(obj->btf, vsi->type); 1710 if (!btf_is_var(vt)) 1711 continue; 1712 1713 if (btf_var(vt)->linkage != BTF_VAR_STATIC) 1714 return true; 1715 } 1716 1717 return false; 1718} 1719 1720static int 1721bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type, 1722 const char *real_name, int sec_idx, void *data, size_t data_sz) 1723{ 1724 struct bpf_map_def *def; 1725 struct bpf_map *map; 1726 size_t mmap_sz; 1727 int err; 1728 1729 map = bpf_object__add_map(obj); 1730 if (IS_ERR(map)) 1731 return PTR_ERR(map); 1732 1733 map->libbpf_type = type; 1734 map->sec_idx = sec_idx; 1735 map->sec_offset = 0; 1736 map->real_name = strdup(real_name); 1737 map->name = internal_map_name(obj, real_name); 1738 if (!map->real_name || !map->name) { 1739 zfree(&map->real_name); 1740 zfree(&map->name); 1741 return -ENOMEM; 1742 } 1743 1744 def = &map->def; 1745 def->type = BPF_MAP_TYPE_ARRAY; 1746 def->key_size = sizeof(int); 1747 def->value_size = data_sz; 1748 def->max_entries = 1; 1749 def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG 1750 ? BPF_F_RDONLY_PROG : 0; 1751 1752 /* failures are fine because of maps like .rodata.str1.1 */ 1753 (void) map_fill_btf_type_info(obj, map); 1754 1755 if (map_is_mmapable(obj, map)) 1756 def->map_flags |= BPF_F_MMAPABLE; 1757 1758 pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n", 1759 map->name, map->sec_idx, map->sec_offset, def->map_flags); 1760 1761 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 1762 map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, 1763 MAP_SHARED | MAP_ANONYMOUS, -1, 0); 1764 if (map->mmaped == MAP_FAILED) { 1765 err = -errno; 1766 map->mmaped = NULL; 1767 pr_warn("failed to alloc map '%s' content buffer: %d\n", 1768 map->name, err); 1769 zfree(&map->real_name); 1770 zfree(&map->name); 1771 return err; 1772 } 1773 1774 if (data) 1775 memcpy(map->mmaped, data, data_sz); 1776 1777 pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name); 1778 return 0; 1779} 1780 1781static int bpf_object__init_global_data_maps(struct bpf_object *obj) 1782{ 1783 struct elf_sec_desc *sec_desc; 1784 const char *sec_name; 1785 int err = 0, sec_idx; 1786 1787 /* 1788 * Populate obj->maps with libbpf internal maps. 1789 */ 1790 for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) { 1791 sec_desc = &obj->efile.secs[sec_idx]; 1792 1793 /* Skip recognized sections with size 0. */ 1794 if (!sec_desc->data || sec_desc->data->d_size == 0) 1795 continue; 1796 1797 switch (sec_desc->sec_type) { 1798 case SEC_DATA: 1799#if defined HAVE_LIBELF 1800 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1801#elif defined HAVE_ELFIO 1802 sec_name = elf_sec_name_by_idx(obj, sec_idx); 1803#endif 1804 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA, 1805 sec_name, sec_idx, 1806 sec_desc->data->d_buf, 1807 sec_desc->data->d_size); 1808 break; 1809 case SEC_RODATA: 1810 obj->has_rodata = true; 1811#if defined HAVE_LIBELF 1812 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1813#elif defined HAVE_ELFIO 1814 sec_name = elf_sec_name_by_idx(obj, sec_idx); 1815#endif 1816 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA, 1817 sec_name, sec_idx, 1818 sec_desc->data->d_buf, 1819 sec_desc->data->d_size); 1820 break; 1821 case SEC_BSS: 1822#if defined HAVE_LIBELF 1823 sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, sec_idx)); 1824#elif defined HAVE_ELFIO 1825 sec_name = elf_sec_name_by_idx(obj, sec_idx); 1826#endif 1827 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS, 1828 sec_name, sec_idx, 1829 NULL, 1830 sec_desc->data->d_size); 1831 break; 1832 default: 1833 /* skip */ 1834 break; 1835 } 1836 if (err) 1837 return err; 1838 } 1839 return 0; 1840} 1841 1842 1843static struct extern_desc *find_extern_by_name(const struct bpf_object *obj, 1844 const void *name) 1845{ 1846 int i; 1847 1848 for (i = 0; i < obj->nr_extern; i++) { 1849 if (strcmp(obj->externs[i].name, name) == 0) 1850 return &obj->externs[i]; 1851 } 1852 return NULL; 1853} 1854 1855static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, 1856 char value) 1857{ 1858 switch (ext->kcfg.type) { 1859 case KCFG_BOOL: 1860 if (value == 'm') { 1861 pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n", 1862 ext->name, value); 1863 return -EINVAL; 1864 } 1865 *(bool *)ext_val = value == 'y' ? true : false; 1866 break; 1867 case KCFG_TRISTATE: 1868 if (value == 'y') 1869 *(enum libbpf_tristate *)ext_val = TRI_YES; 1870 else if (value == 'm') 1871 *(enum libbpf_tristate *)ext_val = TRI_MODULE; 1872 else /* value == 'n' */ 1873 *(enum libbpf_tristate *)ext_val = TRI_NO; 1874 break; 1875 case KCFG_CHAR: 1876 *(char *)ext_val = value; 1877 break; 1878 case KCFG_UNKNOWN: 1879 case KCFG_INT: 1880 case KCFG_CHAR_ARR: 1881 default: 1882 pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n", 1883 ext->name, value); 1884 return -EINVAL; 1885 } 1886 ext->is_set = true; 1887 return 0; 1888} 1889 1890static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, 1891 const char *value) 1892{ 1893 size_t len; 1894 1895 if (ext->kcfg.type != KCFG_CHAR_ARR) { 1896 pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n", 1897 ext->name, value); 1898 return -EINVAL; 1899 } 1900 1901 len = strlen(value); 1902 if (value[len - 1] != '"') { 1903 pr_warn("extern (kcfg) '%s': invalid string config '%s'\n", 1904 ext->name, value); 1905 return -EINVAL; 1906 } 1907 1908 /* strip quotes */ 1909 len -= 2; 1910 if (len >= ext->kcfg.sz) { 1911 pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n", 1912 ext->name, value, len, ext->kcfg.sz - 1); 1913 len = ext->kcfg.sz - 1; 1914 } 1915 memcpy(ext_val, value + 1, len); 1916 ext_val[len] = '\0'; 1917 ext->is_set = true; 1918 return 0; 1919} 1920 1921static int parse_u64(const char *value, __u64 *res) 1922{ 1923 char *value_end; 1924 int err; 1925 1926 errno = 0; 1927 *res = strtoull(value, &value_end, 0); 1928 if (errno) { 1929 err = -errno; 1930 pr_warn("failed to parse '%s' as integer: %d\n", value, err); 1931 return err; 1932 } 1933 if (*value_end) { 1934 pr_warn("failed to parse '%s' as integer completely\n", value); 1935 return -EINVAL; 1936 } 1937 return 0; 1938} 1939 1940static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) 1941{ 1942 int bit_sz = ext->kcfg.sz * 8; 1943 1944 if (ext->kcfg.sz == 8) 1945 return true; 1946 1947 /* Validate that value stored in u64 fits in integer of `ext->sz` 1948 * bytes size without any loss of information. If the target integer 1949 * is signed, we rely on the following limits of integer type of 1950 * Y bits and subsequent transformation: 1951 * 1952 * -2^(Y-1) <= X <= 2^(Y-1) - 1 1953 * 0 <= X + 2^(Y-1) <= 2^Y - 1 1954 * 0 <= X + 2^(Y-1) < 2^Y 1955 * 1956 * For unsigned target integer, check that all the (64 - Y) bits are 1957 * zero. 1958 */ 1959 if (ext->kcfg.is_signed) 1960 return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz); 1961 else 1962 return (v >> bit_sz) == 0; 1963} 1964 1965static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, 1966 __u64 value) 1967{ 1968 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR && 1969 ext->kcfg.type != KCFG_BOOL) { 1970 pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n", 1971 ext->name, (unsigned long long)value); 1972 return -EINVAL; 1973 } 1974 if (ext->kcfg.type == KCFG_BOOL && value > 1) { 1975 pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n", 1976 ext->name, (unsigned long long)value); 1977 return -EINVAL; 1978 1979 } 1980 if (!is_kcfg_value_in_range(ext, value)) { 1981 pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n", 1982 ext->name, (unsigned long long)value, ext->kcfg.sz); 1983 return -ERANGE; 1984 } 1985 switch (ext->kcfg.sz) { 1986 case 1: 1987 *(__u8 *)ext_val = value; 1988 break; 1989 case 2: 1990 *(__u16 *)ext_val = value; 1991 break; 1992 case 4: 1993 *(__u32 *)ext_val = value; 1994 break; 1995 case 8: 1996 *(__u64 *)ext_val = value; 1997 break; 1998 default: 1999 return -EINVAL; 2000 } 2001 ext->is_set = true; 2002 return 0; 2003} 2004 2005static int bpf_object__process_kconfig_line(struct bpf_object *obj, 2006 char *buf, void *data) 2007{ 2008 struct extern_desc *ext; 2009 char *sep, *value; 2010 int len, err = 0; 2011 void *ext_val; 2012 __u64 num; 2013 2014 if (!str_has_pfx(buf, "CONFIG_")) 2015 return 0; 2016 2017 sep = strchr(buf, '='); 2018 if (!sep) { 2019 pr_warn("failed to parse '%s': no separator\n", buf); 2020 return -EINVAL; 2021 } 2022 2023 /* Trim ending '\n' */ 2024 len = strlen(buf); 2025 if (buf[len - 1] == '\n') 2026 buf[len - 1] = '\0'; 2027 /* Split on '=' and ensure that a value is present. */ 2028 *sep = '\0'; 2029 if (!sep[1]) { 2030 *sep = '='; 2031 pr_warn("failed to parse '%s': no value\n", buf); 2032 return -EINVAL; 2033 } 2034 2035 ext = find_extern_by_name(obj, buf); 2036 if (!ext || ext->is_set) 2037 return 0; 2038 2039 ext_val = data + ext->kcfg.data_off; 2040 value = sep + 1; 2041 2042 switch (*value) { 2043 case 'y': case 'n': case 'm': 2044 err = set_kcfg_value_tri(ext, ext_val, *value); 2045 break; 2046 case '"': 2047 err = set_kcfg_value_str(ext, ext_val, value); 2048 break; 2049 default: 2050 /* assume integer */ 2051 err = parse_u64(value, &num); 2052 if (err) { 2053 pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value); 2054 return err; 2055 } 2056 if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { 2057 pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value); 2058 return -EINVAL; 2059 } 2060 err = set_kcfg_value_num(ext, ext_val, num); 2061 break; 2062 } 2063 if (err) 2064 return err; 2065 pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value); 2066 return 0; 2067} 2068 2069static int bpf_object__read_kconfig_file(struct bpf_object *obj, void *data) 2070{ 2071 char buf[PATH_MAX]; 2072 struct utsname uts; 2073 int len, err = 0; 2074 gzFile file; 2075 2076 uname(&uts); 2077 len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release); 2078 if (len < 0) 2079 return -EINVAL; 2080 else if (len >= PATH_MAX) 2081 return -ENAMETOOLONG; 2082 2083 /* gzopen also accepts uncompressed files. */ 2084 file = gzopen(buf, "re"); 2085 if (!file) 2086 file = gzopen("/proc/config.gz", "re"); 2087 2088 if (!file) { 2089 pr_warn("failed to open system Kconfig\n"); 2090 return -ENOENT; 2091 } 2092 2093 while (gzgets(file, buf, sizeof(buf))) { 2094 err = bpf_object__process_kconfig_line(obj, buf, data); 2095 if (err) { 2096 pr_warn("error parsing system Kconfig line '%s': %d\n", 2097 buf, err); 2098 goto out; 2099 } 2100 } 2101 2102out: 2103 gzclose(file); 2104 return err; 2105} 2106 2107static int bpf_object__read_kconfig_mem(struct bpf_object *obj, 2108 const char *config, void *data) 2109{ 2110 char buf[PATH_MAX]; 2111 int err = 0; 2112 FILE *file; 2113 2114 file = fmemopen((void *)config, strlen(config), "r"); 2115 if (!file) { 2116 err = -errno; 2117 pr_warn("failed to open in-memory Kconfig: %d\n", err); 2118 return err; 2119 } 2120 2121 while (fgets(buf, sizeof(buf), file)) { 2122 err = bpf_object__process_kconfig_line(obj, buf, data); 2123 if (err) { 2124 pr_warn("error parsing in-memory Kconfig line '%s': %d\n", 2125 buf, err); 2126 break; 2127 } 2128 } 2129 2130 fclose(file); 2131 return err; 2132} 2133 2134static int bpf_object__init_kconfig_map(struct bpf_object *obj) 2135{ 2136 struct extern_desc *last_ext = NULL, *ext; 2137 size_t map_sz; 2138 int i, err; 2139 2140 for (i = 0; i < obj->nr_extern; i++) { 2141 ext = &obj->externs[i]; 2142 if (ext->type == EXT_KCFG) 2143 last_ext = ext; 2144 } 2145 2146 if (!last_ext) 2147 return 0; 2148 2149 map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz; 2150 err = bpf_object__init_internal_map(obj, LIBBPF_MAP_KCONFIG, 2151 ".kconfig", obj->efile.symbols_shndx, 2152 NULL, map_sz); 2153 if (err) 2154 return err; 2155 2156 obj->kconfig_map_idx = obj->nr_maps - 1; 2157 2158 return 0; 2159} 2160 2161const struct btf_type * 2162skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) 2163{ 2164 const struct btf_type *t = btf__type_by_id(btf, id); 2165 2166 if (res_id) 2167 *res_id = id; 2168 2169 while (btf_is_mod(t) || btf_is_typedef(t)) { 2170 if (res_id) 2171 *res_id = t->type; 2172 t = btf__type_by_id(btf, t->type); 2173 } 2174 2175 return t; 2176} 2177 2178static const struct btf_type * 2179resolve_func_ptr(const struct btf *btf, __u32 id, __u32 *res_id) 2180{ 2181 const struct btf_type *t; 2182 2183 t = skip_mods_and_typedefs(btf, id, NULL); 2184 if (!btf_is_ptr(t)) 2185 return NULL; 2186 2187 t = skip_mods_and_typedefs(btf, t->type, res_id); 2188 2189 return btf_is_func_proto(t) ? t : NULL; 2190} 2191 2192static const char *__btf_kind_str(__u16 kind) 2193{ 2194 switch (kind) { 2195 case BTF_KIND_UNKN: return "void"; 2196 case BTF_KIND_INT: return "int"; 2197 case BTF_KIND_PTR: return "ptr"; 2198 case BTF_KIND_ARRAY: return "array"; 2199 case BTF_KIND_STRUCT: return "struct"; 2200 case BTF_KIND_UNION: return "union"; 2201 case BTF_KIND_ENUM: return "enum"; 2202 case BTF_KIND_FWD: return "fwd"; 2203 case BTF_KIND_TYPEDEF: return "typedef"; 2204 case BTF_KIND_VOLATILE: return "volatile"; 2205 case BTF_KIND_CONST: return "const"; 2206 case BTF_KIND_RESTRICT: return "restrict"; 2207 case BTF_KIND_FUNC: return "func"; 2208 case BTF_KIND_FUNC_PROTO: return "func_proto"; 2209 case BTF_KIND_VAR: return "var"; 2210 case BTF_KIND_DATASEC: return "datasec"; 2211 case BTF_KIND_FLOAT: return "float"; 2212 case BTF_KIND_DECL_TAG: return "decl_tag"; 2213 case BTF_KIND_TYPE_TAG: return "type_tag"; 2214 case BTF_KIND_ENUM64: return "enum64"; 2215 default: return "unknown"; 2216 } 2217} 2218 2219const char *btf_kind_str(const struct btf_type *t) 2220{ 2221 return __btf_kind_str(btf_kind(t)); 2222} 2223 2224/* 2225 * Fetch integer attribute of BTF map definition. Such attributes are 2226 * represented using a pointer to an array, in which dimensionality of array 2227 * encodes specified integer value. E.g., int (*type)[BPF_MAP_TYPE_ARRAY]; 2228 * encodes `type => BPF_MAP_TYPE_ARRAY` key/value pair completely using BTF 2229 * type definition, while using only sizeof(void *) space in ELF data section. 2230 */ 2231static bool get_map_field_int(const char *map_name, const struct btf *btf, 2232 const struct btf_member *m, __u32 *res) 2233{ 2234 const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL); 2235 const char *name = btf__name_by_offset(btf, m->name_off); 2236 const struct btf_array *arr_info; 2237 const struct btf_type *arr_t; 2238 2239 if (!btf_is_ptr(t)) { 2240 pr_warn("map '%s': attr '%s': expected PTR, got %s.\n", 2241 map_name, name, btf_kind_str(t)); 2242 return false; 2243 } 2244 2245 arr_t = btf__type_by_id(btf, t->type); 2246 if (!arr_t) { 2247 pr_warn("map '%s': attr '%s': type [%u] not found.\n", 2248 map_name, name, t->type); 2249 return false; 2250 } 2251 if (!btf_is_array(arr_t)) { 2252 pr_warn("map '%s': attr '%s': expected ARRAY, got %s.\n", 2253 map_name, name, btf_kind_str(arr_t)); 2254 return false; 2255 } 2256 arr_info = btf_array(arr_t); 2257 *res = arr_info->nelems; 2258 return true; 2259} 2260 2261static int pathname_concat(char *buf, size_t buf_sz, const char *path, const char *name) 2262{ 2263 int len; 2264 2265 len = snprintf(buf, buf_sz, "%s/%s", path, name); 2266 if (len < 0) 2267 return -EINVAL; 2268 if (len >= buf_sz) 2269 return -ENAMETOOLONG; 2270 2271 return 0; 2272} 2273 2274static int build_map_pin_path(struct bpf_map *map, const char *path) 2275{ 2276 char buf[PATH_MAX]; 2277 int err; 2278 2279 if (!path) 2280 path = "/sys/fs/bpf"; 2281 2282 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 2283 if (err) 2284 return err; 2285 2286 return bpf_map__set_pin_path(map, buf); 2287} 2288 2289/* should match definition in bpf_helpers.h */ 2290enum libbpf_pin_type { 2291 LIBBPF_PIN_NONE, 2292 /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ 2293 LIBBPF_PIN_BY_NAME, 2294}; 2295 2296int parse_btf_map_def(const char *map_name, struct btf *btf, 2297 const struct btf_type *def_t, bool strict, 2298 struct btf_map_def *map_def, struct btf_map_def *inner_def) 2299{ 2300 const struct btf_type *t; 2301 const struct btf_member *m; 2302 bool is_inner = inner_def == NULL; 2303 int vlen, i; 2304 2305 vlen = btf_vlen(def_t); 2306 m = btf_members(def_t); 2307 for (i = 0; i < vlen; i++, m++) { 2308 const char *name = btf__name_by_offset(btf, m->name_off); 2309 2310 if (!name) { 2311 pr_warn("map '%s': invalid field #%d.\n", map_name, i); 2312 return -EINVAL; 2313 } 2314 if (strcmp(name, "type") == 0) { 2315 if (!get_map_field_int(map_name, btf, m, &map_def->map_type)) 2316 return -EINVAL; 2317 map_def->parts |= MAP_DEF_MAP_TYPE; 2318 } else if (strcmp(name, "max_entries") == 0) { 2319 if (!get_map_field_int(map_name, btf, m, &map_def->max_entries)) 2320 return -EINVAL; 2321 map_def->parts |= MAP_DEF_MAX_ENTRIES; 2322 } else if (strcmp(name, "map_flags") == 0) { 2323 if (!get_map_field_int(map_name, btf, m, &map_def->map_flags)) 2324 return -EINVAL; 2325 map_def->parts |= MAP_DEF_MAP_FLAGS; 2326 } else if (strcmp(name, "numa_node") == 0) { 2327 if (!get_map_field_int(map_name, btf, m, &map_def->numa_node)) 2328 return -EINVAL; 2329 map_def->parts |= MAP_DEF_NUMA_NODE; 2330 } else if (strcmp(name, "key_size") == 0) { 2331 __u32 sz; 2332 2333 if (!get_map_field_int(map_name, btf, m, &sz)) 2334 return -EINVAL; 2335 if (map_def->key_size && map_def->key_size != sz) { 2336 pr_warn("map '%s': conflicting key size %u != %u.\n", 2337 map_name, map_def->key_size, sz); 2338 return -EINVAL; 2339 } 2340 map_def->key_size = sz; 2341 map_def->parts |= MAP_DEF_KEY_SIZE; 2342 } else if (strcmp(name, "key") == 0) { 2343 __s64 sz; 2344 2345 t = btf__type_by_id(btf, m->type); 2346 if (!t) { 2347 pr_warn("map '%s': key type [%d] not found.\n", 2348 map_name, m->type); 2349 return -EINVAL; 2350 } 2351 if (!btf_is_ptr(t)) { 2352 pr_warn("map '%s': key spec is not PTR: %s.\n", 2353 map_name, btf_kind_str(t)); 2354 return -EINVAL; 2355 } 2356 sz = btf__resolve_size(btf, t->type); 2357 if (sz < 0) { 2358 pr_warn("map '%s': can't determine key size for type [%u]: %zd.\n", 2359 map_name, t->type, (ssize_t)sz); 2360 return sz; 2361 } 2362 if (map_def->key_size && map_def->key_size != sz) { 2363 pr_warn("map '%s': conflicting key size %u != %zd.\n", 2364 map_name, map_def->key_size, (ssize_t)sz); 2365 return -EINVAL; 2366 } 2367 map_def->key_size = sz; 2368 map_def->key_type_id = t->type; 2369 map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE; 2370 } else if (strcmp(name, "value_size") == 0) { 2371 __u32 sz; 2372 2373 if (!get_map_field_int(map_name, btf, m, &sz)) 2374 return -EINVAL; 2375 if (map_def->value_size && map_def->value_size != sz) { 2376 pr_warn("map '%s': conflicting value size %u != %u.\n", 2377 map_name, map_def->value_size, sz); 2378 return -EINVAL; 2379 } 2380 map_def->value_size = sz; 2381 map_def->parts |= MAP_DEF_VALUE_SIZE; 2382 } else if (strcmp(name, "value") == 0) { 2383 __s64 sz; 2384 2385 t = btf__type_by_id(btf, m->type); 2386 if (!t) { 2387 pr_warn("map '%s': value type [%d] not found.\n", 2388 map_name, m->type); 2389 return -EINVAL; 2390 } 2391 if (!btf_is_ptr(t)) { 2392 pr_warn("map '%s': value spec is not PTR: %s.\n", 2393 map_name, btf_kind_str(t)); 2394 return -EINVAL; 2395 } 2396 sz = btf__resolve_size(btf, t->type); 2397 if (sz < 0) { 2398 pr_warn("map '%s': can't determine value size for type [%u]: %zd.\n", 2399 map_name, t->type, (ssize_t)sz); 2400 return sz; 2401 } 2402 if (map_def->value_size && map_def->value_size != sz) { 2403 pr_warn("map '%s': conflicting value size %u != %zd.\n", 2404 map_name, map_def->value_size, (ssize_t)sz); 2405 return -EINVAL; 2406 } 2407 map_def->value_size = sz; 2408 map_def->value_type_id = t->type; 2409 map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE; 2410 } 2411 else if (strcmp(name, "values") == 0) { 2412 bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type); 2413 bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY; 2414 const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value"; 2415 char inner_map_name[128]; 2416 int err; 2417 2418 if (is_inner) { 2419 pr_warn("map '%s': multi-level inner maps not supported.\n", 2420 map_name); 2421 return -ENOTSUP; 2422 } 2423 if (i != vlen - 1) { 2424 pr_warn("map '%s': '%s' member should be last.\n", 2425 map_name, name); 2426 return -EINVAL; 2427 } 2428 if (!is_map_in_map && !is_prog_array) { 2429 pr_warn("map '%s': should be map-in-map or prog-array.\n", 2430 map_name); 2431 return -ENOTSUP; 2432 } 2433 if (map_def->value_size && map_def->value_size != 4) { 2434 pr_warn("map '%s': conflicting value size %u != 4.\n", 2435 map_name, map_def->value_size); 2436 return -EINVAL; 2437 } 2438 map_def->value_size = 4; 2439 t = btf__type_by_id(btf, m->type); 2440 if (!t) { 2441 pr_warn("map '%s': %s type [%d] not found.\n", 2442 map_name, desc, m->type); 2443 return -EINVAL; 2444 } 2445 if (!btf_is_array(t) || btf_array(t)->nelems) { 2446 pr_warn("map '%s': %s spec is not a zero-sized array.\n", 2447 map_name, desc); 2448 return -EINVAL; 2449 } 2450 t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL); 2451 if (!btf_is_ptr(t)) { 2452 pr_warn("map '%s': %s def is of unexpected kind %s.\n", 2453 map_name, desc, btf_kind_str(t)); 2454 return -EINVAL; 2455 } 2456 t = skip_mods_and_typedefs(btf, t->type, NULL); 2457 if (is_prog_array) { 2458 if (!btf_is_func_proto(t)) { 2459 pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n", 2460 map_name, btf_kind_str(t)); 2461 return -EINVAL; 2462 } 2463 continue; 2464 } 2465 if (!btf_is_struct(t)) { 2466 pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n", 2467 map_name, btf_kind_str(t)); 2468 return -EINVAL; 2469 } 2470 2471 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", map_name); 2472 err = parse_btf_map_def(inner_map_name, btf, t, strict, inner_def, NULL); 2473 if (err) 2474 return err; 2475 2476 map_def->parts |= MAP_DEF_INNER_MAP; 2477 } else if (strcmp(name, "pinning") == 0) { 2478 __u32 val; 2479 2480 if (is_inner) { 2481 pr_warn("map '%s': inner def can't be pinned.\n", map_name); 2482 return -EINVAL; 2483 } 2484 if (!get_map_field_int(map_name, btf, m, &val)) 2485 return -EINVAL; 2486 if (val != LIBBPF_PIN_NONE && val != LIBBPF_PIN_BY_NAME) { 2487 pr_warn("map '%s': invalid pinning value %u.\n", 2488 map_name, val); 2489 return -EINVAL; 2490 } 2491 map_def->pinning = val; 2492 map_def->parts |= MAP_DEF_PINNING; 2493 } else if (strcmp(name, "map_extra") == 0) { 2494 __u32 map_extra; 2495 2496 if (!get_map_field_int(map_name, btf, m, &map_extra)) 2497 return -EINVAL; 2498 map_def->map_extra = map_extra; 2499 map_def->parts |= MAP_DEF_MAP_EXTRA; 2500 } else { 2501 if (strict) { 2502 pr_warn("map '%s': unknown field '%s'.\n", map_name, name); 2503 return -ENOTSUP; 2504 } 2505 pr_debug("map '%s': ignoring unknown field '%s'.\n", map_name, name); 2506 } 2507 } 2508 2509 if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) { 2510 pr_warn("map '%s': map type isn't specified.\n", map_name); 2511 return -EINVAL; 2512 } 2513 2514 return 0; 2515} 2516 2517static size_t adjust_ringbuf_sz(size_t sz) 2518{ 2519 __u32 page_sz = sysconf(_SC_PAGE_SIZE); 2520 __u32 mul; 2521 2522 /* if user forgot to set any size, make sure they see error */ 2523 if (sz == 0) 2524 return 0; 2525 /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be 2526 * a power-of-2 multiple of kernel's page size. If user diligently 2527 * satisified these conditions, pass the size through. 2528 */ 2529 if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) 2530 return sz; 2531 2532 /* Otherwise find closest (page_sz * power_of_2) product bigger than 2533 * user-set size to satisfy both user size request and kernel 2534 * requirements and substitute correct max_entries for map creation. 2535 */ 2536 for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) { 2537 if (mul * page_sz > sz) 2538 return mul * page_sz; 2539 } 2540 2541 /* if it's impossible to satisfy the conditions (i.e., user size is 2542 * very close to UINT_MAX but is not a power-of-2 multiple of 2543 * page_size) then just return original size and let kernel reject it 2544 */ 2545 return sz; 2546} 2547 2548static bool map_is_ringbuf(const struct bpf_map *map) 2549{ 2550 return map->def.type == BPF_MAP_TYPE_RINGBUF || 2551 map->def.type == BPF_MAP_TYPE_USER_RINGBUF; 2552} 2553 2554static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) 2555{ 2556 map->def.type = def->map_type; 2557 map->def.key_size = def->key_size; 2558 map->def.value_size = def->value_size; 2559 map->def.max_entries = def->max_entries; 2560 map->def.map_flags = def->map_flags; 2561 map->map_extra = def->map_extra; 2562 2563 map->numa_node = def->numa_node; 2564 map->btf_key_type_id = def->key_type_id; 2565 map->btf_value_type_id = def->value_type_id; 2566 2567 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ 2568 if (map_is_ringbuf(map)) 2569 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); 2570 2571 if (def->parts & MAP_DEF_MAP_TYPE) 2572 pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); 2573 2574 if (def->parts & MAP_DEF_KEY_TYPE) 2575 pr_debug("map '%s': found key [%u], sz = %u.\n", 2576 map->name, def->key_type_id, def->key_size); 2577 else if (def->parts & MAP_DEF_KEY_SIZE) 2578 pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size); 2579 2580 if (def->parts & MAP_DEF_VALUE_TYPE) 2581 pr_debug("map '%s': found value [%u], sz = %u.\n", 2582 map->name, def->value_type_id, def->value_size); 2583 else if (def->parts & MAP_DEF_VALUE_SIZE) 2584 pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size); 2585 2586 if (def->parts & MAP_DEF_MAX_ENTRIES) 2587 pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries); 2588 if (def->parts & MAP_DEF_MAP_FLAGS) 2589 pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags); 2590 if (def->parts & MAP_DEF_MAP_EXTRA) 2591 pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name, 2592 (unsigned long long)def->map_extra); 2593 if (def->parts & MAP_DEF_PINNING) 2594 pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning); 2595 if (def->parts & MAP_DEF_NUMA_NODE) 2596 pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node); 2597 2598 if (def->parts & MAP_DEF_INNER_MAP) 2599 pr_debug("map '%s': found inner map definition.\n", map->name); 2600} 2601 2602static const char *btf_var_linkage_str(__u32 linkage) 2603{ 2604 switch (linkage) { 2605 case BTF_VAR_STATIC: return "static"; 2606 case BTF_VAR_GLOBAL_ALLOCATED: return "global"; 2607 case BTF_VAR_GLOBAL_EXTERN: return "extern"; 2608 default: return "unknown"; 2609 } 2610} 2611 2612static int bpf_object__init_user_btf_map(struct bpf_object *obj, 2613 const struct btf_type *sec, 2614 int var_idx, int sec_idx, 2615 const Elf_Data *data, bool strict, 2616 const char *pin_root_path) 2617{ 2618 struct btf_map_def map_def = {}, inner_def = {}; 2619 const struct btf_type *var, *def; 2620 const struct btf_var_secinfo *vi; 2621 const struct btf_var *var_extra; 2622 const char *map_name; 2623 struct bpf_map *map; 2624 int err; 2625 2626 vi = btf_var_secinfos(sec) + var_idx; 2627 var = btf__type_by_id(obj->btf, vi->type); 2628 var_extra = btf_var(var); 2629 map_name = btf__name_by_offset(obj->btf, var->name_off); 2630 2631 if (map_name == NULL || map_name[0] == '\0') { 2632 pr_warn("map #%d: empty name.\n", var_idx); 2633 return -EINVAL; 2634 } 2635 if ((__u64)vi->offset + vi->size > data->d_size) { 2636 pr_warn("map '%s' BTF data is corrupted.\n", map_name); 2637 return -EINVAL; 2638 } 2639 if (!btf_is_var(var)) { 2640 pr_warn("map '%s': unexpected var kind %s.\n", 2641 map_name, btf_kind_str(var)); 2642 return -EINVAL; 2643 } 2644 if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) { 2645 pr_warn("map '%s': unsupported map linkage %s.\n", 2646 map_name, btf_var_linkage_str(var_extra->linkage)); 2647 return -EOPNOTSUPP; 2648 } 2649 2650 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 2651 if (!btf_is_struct(def)) { 2652 pr_warn("map '%s': unexpected def kind %s.\n", 2653 map_name, btf_kind_str(var)); 2654 return -EINVAL; 2655 } 2656 if (def->size > vi->size) { 2657 pr_warn("map '%s': invalid def size.\n", map_name); 2658 return -EINVAL; 2659 } 2660 2661 map = bpf_object__add_map(obj); 2662 if (IS_ERR(map)) 2663 return PTR_ERR(map); 2664 map->name = strdup(map_name); 2665 if (!map->name) { 2666 pr_warn("map '%s': failed to alloc map name.\n", map_name); 2667 return -ENOMEM; 2668 } 2669 map->libbpf_type = LIBBPF_MAP_UNSPEC; 2670 map->def.type = BPF_MAP_TYPE_UNSPEC; 2671 map->sec_idx = sec_idx; 2672 map->sec_offset = vi->offset; 2673 map->btf_var_idx = var_idx; 2674 pr_debug("map '%s': at sec_idx %d, offset %zu.\n", 2675 map_name, map->sec_idx, map->sec_offset); 2676 2677 err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def); 2678 if (err) 2679 return err; 2680 2681 fill_map_from_def(map, &map_def); 2682 2683 if (map_def.pinning == LIBBPF_PIN_BY_NAME) { 2684 err = build_map_pin_path(map, pin_root_path); 2685 if (err) { 2686 pr_warn("map '%s': couldn't build pin path.\n", map->name); 2687 return err; 2688 } 2689 } 2690 2691 if (map_def.parts & MAP_DEF_INNER_MAP) { 2692 map->inner_map = calloc(1, sizeof(*map->inner_map)); 2693 if (!map->inner_map) 2694 return -ENOMEM; 2695 map->inner_map->fd = -1; 2696 map->inner_map->sec_idx = sec_idx; 2697 map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1); 2698 if (!map->inner_map->name) 2699 return -ENOMEM; 2700 sprintf(map->inner_map->name, "%s.inner", map_name); 2701 2702 fill_map_from_def(map->inner_map, &inner_def); 2703 } 2704 2705 err = map_fill_btf_type_info(obj, map); 2706 if (err) 2707 return err; 2708 2709 return 0; 2710} 2711 2712static int bpf_object__init_user_btf_maps(struct bpf_object *obj, bool strict, 2713 const char *pin_root_path) 2714{ 2715 const struct btf_type *sec = NULL; 2716 int nr_types, i, vlen, err; 2717 const struct btf_type *t; 2718 const char *name; 2719 Elf_Data *data; 2720#ifdef HAVE_LIBELF 2721 Elf_Scn *scn; 2722#endif 2723 2724 if (obj->efile.btf_maps_shndx < 0) 2725 return 0; 2726#if defined HAVE_LIBELF 2727 scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx); 2728 data = elf_sec_data(obj, scn); 2729 if (!scn || !data) { 2730#elif defined HAVE_ELFIO 2731 Elf_Data realdata; 2732 data = elf_sec_data_by_idx(obj, obj->efile.btf_maps_shndx, &realdata); 2733 if (!data) { 2734#endif 2735 pr_warn("elf: failed to get %s map definitions for %s\n", 2736 MAPS_ELF_SEC, obj->path); 2737 return -EINVAL; 2738 } 2739 2740 nr_types = btf__type_cnt(obj->btf); 2741 for (i = 1; i < nr_types; i++) { 2742 t = btf__type_by_id(obj->btf, i); 2743 if (!btf_is_datasec(t)) 2744 continue; 2745 name = btf__name_by_offset(obj->btf, t->name_off); 2746 if (strcmp(name, MAPS_ELF_SEC) == 0) { 2747 sec = t; 2748 obj->efile.btf_maps_sec_btf_id = i; 2749 break; 2750 } 2751 } 2752 2753 if (!sec) { 2754 pr_warn("DATASEC '%s' not found.\n", MAPS_ELF_SEC); 2755 return -ENOENT; 2756 } 2757 2758 vlen = btf_vlen(sec); 2759 for (i = 0; i < vlen; i++) { 2760 err = bpf_object__init_user_btf_map(obj, sec, i, 2761 obj->efile.btf_maps_shndx, 2762 data, strict, 2763 pin_root_path); 2764 if (err) 2765 return err; 2766 } 2767 2768 return 0; 2769} 2770 2771static int bpf_object__init_maps(struct bpf_object *obj, 2772 const struct bpf_object_open_opts *opts) 2773{ 2774 const char *pin_root_path; 2775 bool strict; 2776 int err = 0; 2777 2778 strict = !OPTS_GET(opts, relaxed_maps, false); 2779 pin_root_path = OPTS_GET(opts, pin_root_path, NULL); 2780 2781 err = bpf_object__init_user_btf_maps(obj, strict, pin_root_path); 2782 err = err ?: bpf_object__init_global_data_maps(obj); 2783 err = err ?: bpf_object__init_kconfig_map(obj); 2784 err = err ?: bpf_object_init_struct_ops(obj); 2785 2786 return err; 2787} 2788 2789static bool section_have_execinstr(struct bpf_object *obj, int idx) 2790{ 2791 Elf64_Shdr *sh; 2792#if defined HAVE_LIBELF 2793 sh = elf_sec_hdr(obj, elf_sec_by_idx(obj, idx)); 2794#elif defined HAVE_ELFIO 2795 Elf64_Shdr header; 2796 sh = elf_sec_hdr_by_idx(obj, idx, &header); 2797#endif 2798 if (!sh) 2799 return false; 2800 2801 return sh->sh_flags & SHF_EXECINSTR; 2802} 2803 2804static bool btf_needs_sanitization(struct bpf_object *obj) 2805{ 2806 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2807 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2808 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2809 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2810 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); 2811 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); 2812 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); 2813 2814 return !has_func || !has_datasec || !has_func_global || !has_float || 2815 !has_decl_tag || !has_type_tag || !has_enum64; 2816} 2817 2818static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) 2819{ 2820 bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); 2821 bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); 2822 bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); 2823 bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); 2824 bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); 2825 bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); 2826 bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); 2827 int enum64_placeholder_id = 0; 2828 struct btf_type *t; 2829 int i, j, vlen; 2830 2831 for (i = 1; i < btf__type_cnt(btf); i++) { 2832 t = (struct btf_type *)btf__type_by_id(btf, i); 2833 2834 if ((!has_datasec && btf_is_var(t)) || (!has_decl_tag && btf_is_decl_tag(t))) { 2835 /* replace VAR/DECL_TAG with INT */ 2836 t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0); 2837 /* 2838 * using size = 1 is the safest choice, 4 will be too 2839 * big and cause kernel BTF validation failure if 2840 * original variable took less than 4 bytes 2841 */ 2842 t->size = 1; 2843 *(int *)(t + 1) = BTF_INT_ENC(0, 0, 8); 2844 } else if (!has_datasec && btf_is_datasec(t)) { 2845 /* replace DATASEC with STRUCT */ 2846 const struct btf_var_secinfo *v = btf_var_secinfos(t); 2847 struct btf_member *m = btf_members(t); 2848 struct btf_type *vt; 2849 char *name; 2850 2851 name = (char *)btf__name_by_offset(btf, t->name_off); 2852 while (*name) { 2853 if (*name == '.') 2854 *name = '_'; 2855 name++; 2856 } 2857 2858 vlen = btf_vlen(t); 2859 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen); 2860 for (j = 0; j < vlen; j++, v++, m++) { 2861 /* order of field assignments is important */ 2862 m->offset = v->offset * 8; 2863 m->type = v->type; 2864 /* preserve variable name as member name */ 2865 vt = (void *)btf__type_by_id(btf, v->type); 2866 m->name_off = vt->name_off; 2867 } 2868 } else if (!has_func && btf_is_func_proto(t)) { 2869 /* replace FUNC_PROTO with ENUM */ 2870 vlen = btf_vlen(t); 2871 t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen); 2872 t->size = sizeof(__u32); /* kernel enforced */ 2873 } else if (!has_func && btf_is_func(t)) { 2874 /* replace FUNC with TYPEDEF */ 2875 t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0); 2876 } else if (!has_func_global && btf_is_func(t)) { 2877 /* replace BTF_FUNC_GLOBAL with BTF_FUNC_STATIC */ 2878 t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0); 2879 } else if (!has_float && btf_is_float(t)) { 2880 /* replace FLOAT with an equally-sized empty STRUCT; 2881 * since C compilers do not accept e.g. "float" as a 2882 * valid struct name, make it anonymous 2883 */ 2884 t->name_off = 0; 2885 t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); 2886 } else if (!has_type_tag && btf_is_type_tag(t)) { 2887 /* replace TYPE_TAG with a CONST */ 2888 t->name_off = 0; 2889 t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0); 2890 } else if (!has_enum64 && btf_is_enum(t)) { 2891 /* clear the kflag */ 2892 t->info = btf_type_info(btf_kind(t), btf_vlen(t), false); 2893 } else if (!has_enum64 && btf_is_enum64(t)) { 2894 /* replace ENUM64 with a union */ 2895 struct btf_member *m; 2896 2897 if (enum64_placeholder_id == 0) { 2898 enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0); 2899 if (enum64_placeholder_id < 0) 2900 return enum64_placeholder_id; 2901 2902 t = (struct btf_type *)btf__type_by_id(btf, i); 2903 } 2904 2905 m = btf_members(t); 2906 vlen = btf_vlen(t); 2907 t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen); 2908 for (j = 0; j < vlen; j++, m++) { 2909 m->type = enum64_placeholder_id; 2910 m->offset = 0; 2911 } 2912 } 2913 } 2914 2915 return 0; 2916} 2917 2918static bool libbpf_needs_btf(const struct bpf_object *obj) 2919{ 2920 return obj->efile.btf_maps_shndx >= 0 || 2921 obj->efile.st_ops_shndx >= 0 || 2922 obj->efile.st_ops_link_shndx >= 0 || 2923 obj->nr_extern > 0; 2924} 2925 2926static bool kernel_needs_btf(const struct bpf_object *obj) 2927{ 2928 return obj->efile.st_ops_shndx >= 0 || obj->efile.st_ops_link_shndx >= 0; 2929} 2930 2931static int bpf_object__init_btf(struct bpf_object *obj, 2932 Elf_Data *btf_data, 2933 Elf_Data *btf_ext_data) 2934{ 2935 int err = -ENOENT; 2936 2937 if (btf_data) { 2938 obj->btf = btf__new(btf_data->d_buf, btf_data->d_size); 2939 err = libbpf_get_error(obj->btf); 2940 if (err) { 2941 obj->btf = NULL; 2942 pr_warn("Error loading ELF section %s: %d.\n", BTF_ELF_SEC, err); 2943 goto out; 2944 } 2945 /* enforce 8-byte pointers for BPF-targeted BTFs */ 2946 btf__set_pointer_size(obj->btf, 8); 2947 } 2948 if (btf_ext_data) { 2949 struct btf_ext_info *ext_segs[3]; 2950 int seg_num, sec_num; 2951 2952 if (!obj->btf) { 2953 pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", 2954 BTF_EXT_ELF_SEC, BTF_ELF_SEC); 2955 goto out; 2956 } 2957 obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size); 2958 err = libbpf_get_error(obj->btf_ext); 2959 if (err) { 2960 pr_warn("Error loading ELF section %s: %d. Ignored and continue.\n", 2961 BTF_EXT_ELF_SEC, err); 2962 obj->btf_ext = NULL; 2963 goto out; 2964 } 2965 2966 /* setup .BTF.ext to ELF section mapping */ 2967 ext_segs[0] = &obj->btf_ext->func_info; 2968 ext_segs[1] = &obj->btf_ext->line_info; 2969 ext_segs[2] = &obj->btf_ext->core_relo_info; 2970 for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) { 2971 struct btf_ext_info *seg = ext_segs[seg_num]; 2972 const struct btf_ext_info_sec *sec; 2973 const char *sec_name; 2974#ifdef HAVE_LIBELF 2975 Elf_Scn *scn; 2976#elif defined HAVE_ELFIO 2977 psection_t sec_obj; 2978#endif 2979 2980 if (seg->sec_cnt == 0) 2981 continue; 2982 2983 seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs)); 2984 if (!seg->sec_idxs) { 2985 err = -ENOMEM; 2986 goto out; 2987 } 2988 2989 sec_num = 0; 2990 for_each_btf_ext_sec(seg, sec) { 2991 /* preventively increment index to avoid doing 2992 * this before every continue below 2993 */ 2994 sec_num++; 2995 2996 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 2997 if (str_is_empty(sec_name)) 2998 continue; 2999#ifdef HAVE_LIBELF 3000 scn = elf_sec_by_name(obj, sec_name); 3001 if (!scn) 3002 continue; 3003#elif defined HAVE_ELFIO 3004 pelfio_t elf = obj->efile.elf; 3005 sec_obj = elfio_get_section_by_name(elf, sec_name); 3006 if (!sec_obj) 3007 continue; 3008#endif 3009#ifdef HAVE_LIBELF 3010 seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn); 3011#elif defined HAVE_ELFIO 3012 seg->sec_idxs[sec_num - 1] = elfio_section_get_index(sec_obj); 3013#endif 3014 } 3015 } 3016 } 3017out: 3018 if (err && libbpf_needs_btf(obj)) { 3019 pr_warn("BTF is required, but is missing or corrupted.\n"); 3020 return err; 3021 } 3022 return 0; 3023} 3024 3025static int compare_vsi_off(const void *_a, const void *_b) 3026{ 3027 const struct btf_var_secinfo *a = _a; 3028 const struct btf_var_secinfo *b = _b; 3029 3030 return a->offset - b->offset; 3031} 3032 3033static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, 3034 struct btf_type *t) 3035{ 3036 __u32 size = 0, i, vars = btf_vlen(t); 3037 const char *sec_name = btf__name_by_offset(btf, t->name_off); 3038 struct btf_var_secinfo *vsi; 3039 bool fixup_offsets = false; 3040 int err; 3041 3042 if (!sec_name) { 3043 pr_debug("No name found in string section for DATASEC kind.\n"); 3044 return -ENOENT; 3045 } 3046 3047 /* Extern-backing datasecs (.ksyms, .kconfig) have their size and 3048 * variable offsets set at the previous step. Further, not every 3049 * extern BTF VAR has corresponding ELF symbol preserved, so we skip 3050 * all fixups altogether for such sections and go straight to sorting 3051 * VARs within their DATASEC. 3052 */ 3053 if (strcmp(sec_name, KCONFIG_SEC) == 0 || strcmp(sec_name, KSYMS_SEC) == 0) 3054 goto sort_vars; 3055 3056 /* Clang leaves DATASEC size and VAR offsets as zeroes, so we need to 3057 * fix this up. But BPF static linker already fixes this up and fills 3058 * all the sizes and offsets during static linking. So this step has 3059 * to be optional. But the STV_HIDDEN handling is non-optional for any 3060 * non-extern DATASEC, so the variable fixup loop below handles both 3061 * functions at the same time, paying the cost of BTF VAR <-> ELF 3062 * symbol matching just once. 3063 */ 3064 if (t->size == 0) { 3065 err = find_elf_sec_sz(obj, sec_name, &size); 3066 if (err || !size) { 3067 pr_debug("sec '%s': failed to determine size from ELF: size %u, err %d\n", 3068 sec_name, size, err); 3069 return -ENOENT; 3070 } 3071 3072 t->size = size; 3073 fixup_offsets = true; 3074 } 3075 3076 for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) { 3077 const struct btf_type *t_var; 3078 struct btf_var *var; 3079 const char *var_name; 3080 Elf64_Sym *sym; 3081 3082 t_var = btf__type_by_id(btf, vsi->type); 3083 if (!t_var || !btf_is_var(t_var)) { 3084 pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name); 3085 return -EINVAL; 3086 } 3087 3088 var = btf_var(t_var); 3089 if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN) 3090 continue; 3091 3092 var_name = btf__name_by_offset(btf, t_var->name_off); 3093 if (!var_name) { 3094 pr_debug("sec '%s': failed to find name of DATASEC's member #%d\n", 3095 sec_name, i); 3096 return -ENOENT; 3097 } 3098 3099 sym = find_elf_var_sym(obj, var_name); 3100 if (IS_ERR(sym)) { 3101 pr_debug("sec '%s': failed to find ELF symbol for VAR '%s'\n", 3102 sec_name, var_name); 3103 return -ENOENT; 3104 } 3105 3106 if (fixup_offsets) 3107 vsi->offset = sym->st_value; 3108 3109 /* if variable is a global/weak symbol, but has restricted 3110 * (STV_HIDDEN or STV_INTERNAL) visibility, mark its BTF VAR 3111 * as static. This follows similar logic for functions (BPF 3112 * subprogs) and influences libbpf's further decisions about 3113 * whether to make global data BPF array maps as 3114 * BPF_F_MMAPABLE. 3115 */ 3116 if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN 3117 || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL) 3118 var->linkage = BTF_VAR_STATIC; 3119 } 3120 3121sort_vars: 3122 qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off); 3123 return 0; 3124} 3125 3126static int bpf_object_fixup_btf(struct bpf_object *obj) 3127{ 3128 int i, n, err = 0; 3129 3130 if (!obj->btf) 3131 return 0; 3132 3133 n = btf__type_cnt(obj->btf); 3134 for (i = 1; i < n; i++) { 3135 struct btf_type *t = btf_type_by_id(obj->btf, i); 3136 3137 /* Loader needs to fix up some of the things compiler 3138 * couldn't get its hands on while emitting BTF. This 3139 * is section size and global variable offset. We use 3140 * the info from the ELF itself for this purpose. 3141 */ 3142 if (btf_is_datasec(t)) { 3143 err = btf_fixup_datasec(obj, obj->btf, t); 3144 if (err) 3145 return err; 3146 } 3147 } 3148 3149 return 0; 3150} 3151 3152static bool prog_needs_vmlinux_btf(struct bpf_program *prog) 3153{ 3154 if (prog->type == BPF_PROG_TYPE_STRUCT_OPS || 3155 prog->type == BPF_PROG_TYPE_LSM) 3156 return true; 3157 3158 /* BPF_PROG_TYPE_TRACING programs which do not attach to other programs 3159 * also need vmlinux BTF 3160 */ 3161 if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd) 3162 return true; 3163 3164 return false; 3165} 3166 3167static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) 3168{ 3169 struct bpf_program *prog; 3170 int i; 3171 3172 /* CO-RE relocations need kernel BTF, only when btf_custom_path 3173 * is not specified 3174 */ 3175 if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path) 3176 return true; 3177 3178 /* Support for typed ksyms needs kernel BTF */ 3179 for (i = 0; i < obj->nr_extern; i++) { 3180 const struct extern_desc *ext; 3181 3182 ext = &obj->externs[i]; 3183 if (ext->type == EXT_KSYM && ext->ksym.type_id) 3184 return true; 3185 } 3186 3187 bpf_object__for_each_program(prog, obj) { 3188 if (!prog->autoload) 3189 continue; 3190 if (prog_needs_vmlinux_btf(prog)) 3191 return true; 3192 } 3193 3194 return false; 3195} 3196 3197static int bpf_object__load_vmlinux_btf(struct bpf_object *obj, bool force) 3198{ 3199 int err; 3200 3201 /* btf_vmlinux could be loaded earlier */ 3202 if (obj->btf_vmlinux || obj->gen_loader) 3203 return 0; 3204 3205 if (!force && !obj_needs_vmlinux_btf(obj)) 3206 return 0; 3207 3208 obj->btf_vmlinux = btf__load_vmlinux_btf(); 3209 err = libbpf_get_error(obj->btf_vmlinux); 3210 if (err) { 3211 pr_warn("Error loading vmlinux BTF: %d\n", err); 3212 obj->btf_vmlinux = NULL; 3213 return err; 3214 } 3215 return 0; 3216} 3217 3218static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) 3219{ 3220 struct btf *kern_btf = obj->btf; 3221 bool btf_mandatory, sanitize; 3222 int i, err = 0; 3223 3224 if (!obj->btf) 3225 return 0; 3226 3227 if (!kernel_supports(obj, FEAT_BTF)) { 3228 if (kernel_needs_btf(obj)) { 3229 err = -EOPNOTSUPP; 3230 goto report; 3231 } 3232 pr_debug("Kernel doesn't support BTF, skipping uploading it.\n"); 3233 return 0; 3234 } 3235 3236 /* Even though some subprogs are global/weak, user might prefer more 3237 * permissive BPF verification process that BPF verifier performs for 3238 * static functions, taking into account more context from the caller 3239 * functions. In such case, they need to mark such subprogs with 3240 * __attribute__((visibility("hidden"))) and libbpf will adjust 3241 * corresponding FUNC BTF type to be marked as static and trigger more 3242 * involved BPF verification process. 3243 */ 3244 for (i = 0; i < obj->nr_programs; i++) { 3245 struct bpf_program *prog = &obj->programs[i]; 3246 struct btf_type *t; 3247 const char *name; 3248 int j, n; 3249 3250 if (!prog->mark_btf_static || !prog_is_subprog(obj, prog)) 3251 continue; 3252 3253 n = btf__type_cnt(obj->btf); 3254 for (j = 1; j < n; j++) { 3255 t = btf_type_by_id(obj->btf, j); 3256 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) 3257 continue; 3258 3259 name = btf__str_by_offset(obj->btf, t->name_off); 3260 if (strcmp(name, prog->name) != 0) 3261 continue; 3262 3263 t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0); 3264 break; 3265 } 3266 } 3267 3268 if (!kernel_supports(obj, FEAT_BTF_DECL_TAG)) 3269 goto skip_exception_cb; 3270 for (i = 0; i < obj->nr_programs; i++) { 3271 struct bpf_program *prog = &obj->programs[i]; 3272 int j, k, n; 3273 3274 if (prog_is_subprog(obj, prog)) 3275 continue; 3276 n = btf__type_cnt(obj->btf); 3277 for (j = 1; j < n; j++) { 3278 const char *str = "exception_callback:", *name; 3279 size_t len = strlen(str); 3280 struct btf_type *t; 3281 3282 t = btf_type_by_id(obj->btf, j); 3283 if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1) 3284 continue; 3285 3286 name = btf__str_by_offset(obj->btf, t->name_off); 3287 if (strncmp(name, str, len)) 3288 continue; 3289 3290 t = btf_type_by_id(obj->btf, t->type); 3291 if (!btf_is_func(t) || btf_func_linkage(t) != BTF_FUNC_GLOBAL) { 3292 pr_warn("prog '%s': exception_callback:<value> decl tag not applied to the main program\n", 3293 prog->name); 3294 return -EINVAL; 3295 } 3296 if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off))) 3297 continue; 3298 /* Multiple callbacks are specified for the same prog, 3299 * the verifier will eventually return an error for this 3300 * case, hence simply skip appending a subprog. 3301 */ 3302 if (prog->exception_cb_idx >= 0) { 3303 prog->exception_cb_idx = -1; 3304 break; 3305 } 3306 3307 name += len; 3308 if (str_is_empty(name)) { 3309 pr_warn("prog '%s': exception_callback:<value> decl tag contains empty value\n", 3310 prog->name); 3311 return -EINVAL; 3312 } 3313 3314 for (k = 0; k < obj->nr_programs; k++) { 3315 struct bpf_program *subprog = &obj->programs[k]; 3316 3317 if (!prog_is_subprog(obj, subprog)) 3318 continue; 3319 if (strcmp(name, subprog->name)) 3320 continue; 3321 /* Enforce non-hidden, as from verifier point of 3322 * view it expects global functions, whereas the 3323 * mark_btf_static fixes up linkage as static. 3324 */ 3325 if (!subprog->sym_global || subprog->mark_btf_static) { 3326 pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n", 3327 prog->name, subprog->name); 3328 return -EINVAL; 3329 } 3330 /* Let's see if we already saw a static exception callback with the same name */ 3331 if (prog->exception_cb_idx >= 0) { 3332 pr_warn("prog '%s': multiple subprogs with same name as exception callback '%s'\n", 3333 prog->name, subprog->name); 3334 return -EINVAL; 3335 } 3336 prog->exception_cb_idx = k; 3337 break; 3338 } 3339 3340 if (prog->exception_cb_idx >= 0) 3341 continue; 3342 pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name); 3343 return -ENOENT; 3344 } 3345 } 3346skip_exception_cb: 3347 3348 sanitize = btf_needs_sanitization(obj); 3349 if (sanitize) { 3350 const void *raw_data; 3351 __u32 sz; 3352 3353 /* clone BTF to sanitize a copy and leave the original intact */ 3354 raw_data = btf__raw_data(obj->btf, &sz); 3355 kern_btf = btf__new(raw_data, sz); 3356 err = libbpf_get_error(kern_btf); 3357 if (err) 3358 return err; 3359 3360 /* enforce 8-byte pointers for BPF-targeted BTFs */ 3361 btf__set_pointer_size(obj->btf, 8); 3362 err = bpf_object__sanitize_btf(obj, kern_btf); 3363 if (err) 3364 return err; 3365 } 3366 3367 if (obj->gen_loader) { 3368 __u32 raw_size = 0; 3369 const void *raw_data = btf__raw_data(kern_btf, &raw_size); 3370 3371 if (!raw_data) 3372 return -ENOMEM; 3373 bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size); 3374 /* Pretend to have valid FD to pass various fd >= 0 checks. 3375 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 3376 */ 3377 btf__set_fd(kern_btf, 0); 3378 } else { 3379 /* currently BPF_BTF_LOAD only supports log_level 1 */ 3380 err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size, 3381 obj->log_level ? 1 : 0); 3382 } 3383 if (sanitize) { 3384 if (!err) { 3385 /* move fd to libbpf's BTF */ 3386 btf__set_fd(obj->btf, btf__fd(kern_btf)); 3387 btf__set_fd(kern_btf, -1); 3388 } 3389 btf__free(kern_btf); 3390 } 3391report: 3392 if (err) { 3393 btf_mandatory = kernel_needs_btf(obj); 3394 pr_warn("Error loading .BTF into kernel: %d. %s\n", err, 3395 btf_mandatory ? "BTF is mandatory, can't proceed." 3396 : "BTF is optional, ignoring."); 3397 if (!btf_mandatory) 3398 err = 0; 3399 } 3400 return err; 3401} 3402 3403static const char *elf_sym_str(const struct bpf_object *obj, size_t off) 3404{ 3405 const char *name; 3406#if defined HAVE_LIBELF 3407 name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off); 3408#elif defined HAVE_ELFIO 3409 name = elfio_string_get_string(obj->efile.strstring, off); 3410#endif 3411 if (!name) { 3412 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 3413 off, obj->path, elf_errmsg(-1)); 3414 return NULL; 3415 } 3416 3417 return name; 3418} 3419 3420static const char *elf_sec_str(const struct bpf_object *obj, size_t off) 3421{ 3422 const char *name; 3423#if defined HAVE_LIBELF 3424 name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off); 3425#elif defined HAVE_ELFIO 3426 name = elfio_string_get_string(obj->efile.shstring, off); 3427#endif 3428 3429 if (!name) { 3430 pr_warn("elf: failed to get section name string at offset %zu from %s: %s\n", 3431 off, obj->path, elf_errmsg(-1)); 3432 return NULL; 3433 } 3434 3435 return name; 3436} 3437 3438#ifdef HAVE_LIBELF 3439static Elf_Scn *elf_sec_by_idx(const struct bpf_object *obj, size_t idx) 3440{ 3441 Elf_Scn *scn; 3442 3443 scn = elf_getscn(obj->efile.elf, idx); 3444 if (!scn) { 3445 pr_warn("elf: failed to get section(%zu) from %s: %s\n", 3446 idx, obj->path, elf_errmsg(-1)); 3447 return NULL; 3448 } 3449 return scn; 3450} 3451 3452static Elf_Scn *elf_sec_by_name(const struct bpf_object *obj, const char *name) 3453{ 3454 Elf_Scn *scn = NULL; 3455 Elf *elf = obj->efile.elf; 3456 const char *sec_name; 3457 3458 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3459 sec_name = elf_sec_name(obj, scn); 3460 if (!sec_name) 3461 return NULL; 3462 3463 if (strcmp(sec_name, name) != 0) 3464 continue; 3465 3466 return scn; 3467 } 3468 return NULL; 3469} 3470 3471static Elf64_Shdr *elf_sec_hdr(const struct bpf_object *obj, Elf_Scn *scn) 3472{ 3473 Elf64_Shdr *shdr; 3474 3475 if (!scn) 3476 return NULL; 3477 3478 shdr = elf64_getshdr(scn); 3479 if (!shdr) { 3480 pr_warn("elf: failed to get section(%zu) header from %s: %s\n", 3481 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 3482 return NULL; 3483 } 3484 3485 return shdr; 3486} 3487 3488static const char *elf_sec_name(const struct bpf_object *obj, Elf_Scn *scn) 3489{ 3490 const char *name; 3491 Elf64_Shdr *sh; 3492 3493 if (!scn) 3494 return NULL; 3495 3496 sh = elf_sec_hdr(obj, scn); 3497 if (!sh) 3498 return NULL; 3499 3500 name = elf_sec_str(obj, sh->sh_name); 3501 if (!name) { 3502 pr_warn("elf: failed to get section(%zu) name from %s: %s\n", 3503 elf_ndxscn(scn), obj->path, elf_errmsg(-1)); 3504 return NULL; 3505 } 3506 3507 return name; 3508} 3509#elif defined HAVE_ELFIO 3510static Elf64_Shdr *elf_sec_hdr_by_idx(const struct bpf_object *obj, size_t idx, Elf64_Shdr *sheader) 3511{ 3512 psection_t psection = elfio_get_section_by_index(obj->efile.elf, idx); 3513 3514 sheader->sh_name = elfio_section_get_name_string_offset(psection); 3515 sheader->sh_type = elfio_section_get_type(psection); 3516 sheader->sh_flags = elfio_section_get_flags(psection); 3517 sheader->sh_addr = elfio_section_get_address(psection); 3518 sheader->sh_offset = elfio_section_get_offset(psection); 3519 sheader->sh_size = elfio_section_get_size(psection); 3520 sheader->sh_link = elfio_section_get_link(psection); 3521 sheader->sh_info = elfio_section_get_info(psection); 3522 sheader->sh_addralign = elfio_section_get_addr_align(psection); 3523 sheader->sh_entsize = elfio_section_get_entry_size(psection); 3524 3525 return sheader; 3526} 3527 3528static const char *elf_sec_name_by_idx(const struct bpf_object *obj, size_t idx) 3529{ 3530 const char *name; 3531 Elf64_Shdr sh; 3532 3533 elf_sec_hdr_by_idx(obj, idx, &sh); 3534 3535 name = elf_sec_str(obj, sh.sh_name); 3536 if (!name) { 3537 pr_warn("elf: failed to get section(%zu) name from %s: %s\n", 3538 idx, obj->path, elf_errmsg(-1)); 3539 return NULL; 3540 } 3541 3542 return name; 3543} 3544#endif 3545 3546#if defined HAVE_LIBELF 3547static Elf_Data *elf_sec_data(const struct bpf_object *obj, Elf_Scn *scn) 3548{ 3549 Elf_Data *data; 3550 3551 if (!scn) 3552 return NULL; 3553 3554 data = elf_getdata(scn, 0); 3555 if (!data) { 3556 pr_warn("elf: failed to get section(%zu) %s data from %s: %s\n", 3557 elf_ndxscn(scn), elf_sec_name(obj, scn) ?: "<?>", 3558 obj->path, elf_errmsg(-1)); 3559 return NULL; 3560 } 3561 3562 return data; 3563} 3564#elif defined HAVE_ELFIO 3565static Elf_Data *elf_sec_data_by_name(const struct bpf_object *obj, const char *name, Elf_Data *data) 3566{ 3567 pelfio_t elf = obj->efile.elf; 3568 psection_t psection_name = elfio_get_section_by_name(elf, name); 3569 data->d_buf = (void*)elfio_section_get_data(psection_name); 3570 data->d_size = elfio_section_get_size(psection_name); 3571 3572 return data; 3573} 3574 3575static Elf_Data *elf_sec_data_by_idx(const struct bpf_object *obj, size_t idx, Elf_Data *data) 3576{ 3577 pelfio_t elf = obj->efile.elf; 3578 psection_t psection_index = elfio_get_section_by_index(elf, idx); 3579 data->d_buf = (void*)elfio_section_get_data(psection_index); 3580 data->d_size = elfio_section_get_size(psection_index); 3581 3582 return data; 3583} 3584#endif 3585 3586static Elf64_Sym *elf_sym_by_idx(const struct bpf_object *obj, size_t idx) 3587{ 3588 if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym)) 3589 return NULL; 3590 3591 return (Elf64_Sym *)obj->efile.symbols->d_buf + idx; 3592} 3593 3594static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx) 3595{ 3596 if (idx >= data->d_size / sizeof(Elf64_Rel)) 3597 return NULL; 3598 3599 return (Elf64_Rel *)data->d_buf + idx; 3600} 3601 3602static bool is_sec_name_dwarf(const char *name) 3603{ 3604 /* approximation, but the actual list is too long */ 3605 return str_has_pfx(name, ".debug_"); 3606} 3607 3608static bool ignore_elf_section(Elf64_Shdr *hdr, const char *name) 3609{ 3610 /* no special handling of .strtab */ 3611 if (hdr->sh_type == SHT_STRTAB) 3612 return true; 3613 3614 /* ignore .llvm_addrsig section as well */ 3615 if (hdr->sh_type == SHT_LLVM_ADDRSIG) 3616 return true; 3617 3618 /* no subprograms will lead to an empty .text section, ignore it */ 3619 if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 && 3620 strcmp(name, ".text") == 0) 3621 return true; 3622 3623 /* DWARF sections */ 3624 if (is_sec_name_dwarf(name)) 3625 return true; 3626 3627 if (str_has_pfx(name, ".rel")) { 3628 name += sizeof(".rel") - 1; 3629 /* DWARF section relocations */ 3630 if (is_sec_name_dwarf(name)) 3631 return true; 3632 3633 /* .BTF and .BTF.ext don't need relocations */ 3634 if (strcmp(name, BTF_ELF_SEC) == 0 || 3635 strcmp(name, BTF_EXT_ELF_SEC) == 0) 3636 return true; 3637 } 3638 3639 return false; 3640} 3641 3642static int cmp_progs(const void *_a, const void *_b) 3643{ 3644 const struct bpf_program *a = _a; 3645 const struct bpf_program *b = _b; 3646 3647 if (a->sec_idx != b->sec_idx) 3648 return a->sec_idx < b->sec_idx ? -1 : 1; 3649 3650 /* sec_insn_off can't be the same within the section */ 3651 return a->sec_insn_off < b->sec_insn_off ? -1 : 1; 3652} 3653 3654static int bpf_object__elf_collect(struct bpf_object *obj) 3655{ 3656 struct elf_sec_desc *sec_desc; 3657#if defined HAVE_LIBELF 3658 Elf *elf = obj->efile.elf; 3659#elif defined HAVE_ELFIO 3660 pelfio_t elf = obj->efile.elf; 3661#endif 3662 Elf_Data *btf_ext_data = NULL; 3663 Elf_Data *btf_data = NULL; 3664 int idx = 0, err = 0; 3665 const char *name; 3666 Elf_Data *data; 3667#ifdef HAVE_LIBELF 3668 Elf_Scn *scn; 3669#endif 3670 Elf64_Shdr *sh; 3671#ifdef HAVE_ELFIO 3672 Elf64_Shdr secHeader = {0}; 3673 sh = &secHeader; 3674#endif 3675 3676 /* ELF section indices are 0-based, but sec #0 is special "invalid" 3677 * section. Since section count retrieved by elf_getshdrnum() does 3678 * include sec #0, it is already the necessary size of an array to keep 3679 * all the sections. 3680 */ 3681#ifdef HAVE_LIBELF 3682 if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) { 3683 pr_warn("elf: failed to get the number of sections for %s: %s\n", 3684 obj->path, elf_errmsg(-1)); 3685 return -LIBBPF_ERRNO__FORMAT; 3686 } 3687#elif defined HAVE_ELFIO 3688 obj->efile.sec_cnt = elfio_get_sections_num(elf); 3689#endif 3690 obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs)); 3691 if (!obj->efile.secs) 3692 return -ENOMEM; 3693 3694 /* a bunch of ELF parsing functionality depends on processing symbols, 3695 * so do the first pass and find the symbol table 3696 */ 3697#if defined HAVE_LIBELF 3698 scn = NULL; 3699 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3700 sh = elf_sec_hdr(obj, scn); 3701#elif defined HAVE_ELFIO 3702 int secno = elfio_get_sections_num(elf); 3703 for ( int i = 0; i < secno; i++ ) { 3704 Elf_Data realdata; 3705 sh = elf_sec_hdr_by_idx(obj, i, sh); 3706#endif 3707 if (!sh) 3708 return -LIBBPF_ERRNO__FORMAT; 3709 3710 if (sh->sh_type == SHT_SYMTAB) { 3711 if (obj->efile.symbols) { 3712 pr_warn("elf: multiple symbol tables in %s\n", obj->path); 3713 return -LIBBPF_ERRNO__FORMAT; 3714 } 3715#if defined HAVE_LIBELF 3716 data = elf_sec_data(obj, scn); 3717#elif defined HAVE_ELFIO 3718 data = elf_sec_data_by_idx(obj, i, &realdata); 3719#endif 3720 if (!data) 3721 return -LIBBPF_ERRNO__FORMAT; 3722#ifdef HAVE_LIBELF 3723 idx = elf_ndxscn(scn); 3724#endif 3725 3726#if defined HAVE_LIBELF 3727 obj->efile.symbols = data; 3728#elif defined HAVE_ELFIO 3729 obj->efile.realsymbols.d_buf = data->d_buf; 3730 obj->efile.realsymbols.d_size = data->d_size; 3731 obj->efile.symbols = &(obj->efile.realsymbols); 3732#endif 3733 3734#if defined HAVE_LIBELF 3735 obj->efile.symbols_shndx = idx; 3736#elif defined HAVE_ELFIO 3737 obj->efile.symbols_shndx = i; 3738#endif 3739 obj->efile.strtabidx = sh->sh_link; 3740 } 3741 } 3742 3743#ifdef HAVE_ELFIO 3744 pstring_t shstring; 3745 pstring_t strstring; 3746 3747 psection_t psection = elfio_get_section_by_index(elf, obj->efile.strtabidx); 3748 if (!psection) 3749 return -LIBBPF_ERRNO__FORMAT; 3750 strstring = elfio_string_section_accessor_new(psection); 3751 3752 psection = elfio_get_section_by_index(elf, obj->efile.shstrndx); 3753 if (!psection) 3754 return -LIBBPF_ERRNO__FORMAT; 3755 shstring = elfio_string_section_accessor_new(psection); 3756 3757 if (!strstring || !shstring) 3758 return -LIBBPF_ERRNO__FORMAT; 3759 obj->efile.strstring = strstring; 3760 obj->efile.shstring = shstring; 3761#endif 3762 3763 if (!obj->efile.symbols) { 3764 pr_warn("elf: couldn't find symbol table in %s, stripped object file?\n", 3765 obj->path); 3766 return -ENOENT; 3767 } 3768 3769#ifdef HAVE_LIBELF 3770 scn = NULL; 3771 while ((scn = elf_nextscn(elf, scn)) != NULL) { 3772#elif defined HAVE_ELFIO 3773 for ( int i = 0; i < secno; i++ ) { 3774 psection_t ptmpsection = elfio_get_section_by_index(elf, i); 3775 elf_sec_hdr_by_idx(obj, i, sh); 3776#endif 3777 3778#if defined HAVE_LIBELF 3779 idx = elf_ndxscn(scn); 3780#elif defined HAVE_ELFIO 3781 idx = i; 3782#endif 3783 sec_desc = &obj->efile.secs[idx]; 3784 3785#if defined HAVE_LIBELF 3786 sh = elf_sec_hdr(obj, scn); 3787#elif defined HAVE_ELFIO 3788 sh = elf_sec_hdr_by_idx(obj, i, sh); 3789#endif 3790 3791 if (!sh) 3792 return -LIBBPF_ERRNO__FORMAT; 3793 3794 name = elf_sec_str(obj, sh->sh_name); 3795 if (!name) 3796 return -LIBBPF_ERRNO__FORMAT; 3797 3798 if (ignore_elf_section(sh, name)) 3799 continue; 3800 3801#if defined HAVE_LIBELF 3802 data = elf_sec_data(obj, scn); 3803#elif defined HAVE_ELFIO 3804 data = elf_sec_data_by_idx(obj, i, &sec_desc->realdata); 3805#endif 3806 if (!data) 3807 return -LIBBPF_ERRNO__FORMAT; 3808 3809 pr_debug("elf: section(%d) %s, size %ld, link %d, flags %lx, type=%d\n", 3810 idx, name, (unsigned long)data->d_size, 3811 (int)sh->sh_link, (unsigned long)sh->sh_flags, 3812 (int)sh->sh_type); 3813 3814 if (strcmp(name, "license") == 0) { 3815 err = bpf_object__init_license(obj, data->d_buf, data->d_size); 3816 if (err) 3817 return err; 3818 } else if (strcmp(name, "version") == 0) { 3819 err = bpf_object__init_kversion(obj, data->d_buf, data->d_size); 3820 if (err) 3821 return err; 3822 } else if (strcmp(name, "maps") == 0) { 3823 pr_warn("elf: legacy map definitions in 'maps' section are not supported by libbpf v1.0+\n"); 3824 return -ENOTSUP; 3825 } else if (strcmp(name, MAPS_ELF_SEC) == 0) { 3826 obj->efile.btf_maps_shndx = idx; 3827 } else if (strcmp(name, BTF_ELF_SEC) == 0) { 3828 if (sh->sh_type != SHT_PROGBITS) 3829 return -LIBBPF_ERRNO__FORMAT; 3830 btf_data = data; 3831 } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { 3832 if (sh->sh_type != SHT_PROGBITS) 3833 return -LIBBPF_ERRNO__FORMAT; 3834 btf_ext_data = data; 3835 } else if (sh->sh_type == SHT_SYMTAB) { 3836 /* already processed during the first pass above */ 3837 } else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) { 3838 if (sh->sh_flags & SHF_EXECINSTR) { 3839 if (strcmp(name, ".text") == 0) 3840 obj->efile.text_shndx = idx; 3841 err = bpf_object__add_programs(obj, data, name, idx); 3842 if (err) 3843 return err; 3844 } else if (strcmp(name, DATA_SEC) == 0 || 3845 str_has_pfx(name, DATA_SEC ".")) { 3846 sec_desc->sec_type = SEC_DATA; 3847#if defined HAVE_LIBELF 3848 sec_desc->shdr = sh; 3849 sec_desc->data = data; 3850#elif defined HAVE_ELFIO 3851 sec_desc->psection = ptmpsection; 3852 sec_desc->realdata.d_buf = data->d_buf; 3853 sec_desc->realdata.d_size = data->d_size; 3854 sec_desc->data = &(sec_desc->realdata); 3855#endif 3856 } else if (strcmp(name, RODATA_SEC) == 0 || 3857 str_has_pfx(name, RODATA_SEC ".")) { 3858 sec_desc->sec_type = SEC_RODATA; 3859#if defined HAVE_LIBELF 3860 sec_desc->shdr = sh; 3861 sec_desc->data = data; 3862#elif defined HAVE_ELFIO 3863 sec_desc->psection = ptmpsection; 3864 sec_desc->realdata.d_buf = data->d_buf; 3865 sec_desc->realdata.d_size = data->d_size; 3866 sec_desc->data = &(sec_desc->realdata); 3867#endif 3868 3869 } else if (strcmp(name, STRUCT_OPS_SEC) == 0) { 3870#if defined HAVE_LIBELF 3871 obj->efile.st_ops_data = data; 3872#elif defined HAVE_ELFIO 3873 obj->efile.realst_ops_data.d_buf = data->d_buf; 3874 obj->efile.realst_ops_data.d_size = data->d_size; 3875 obj->efile.st_ops_data = &(obj->efile.realst_ops_data); 3876#endif 3877 obj->efile.st_ops_shndx = idx; 3878 } else { 3879 pr_info("elf: skipping unrecognized data section(%d) %s\n", 3880 idx, name); 3881 } 3882 } else if (sh->sh_type == SHT_REL) { 3883 int targ_sec_idx = sh->sh_info; /* points to other section */ 3884 3885 if (sh->sh_entsize != sizeof(Elf64_Rel) || 3886 targ_sec_idx >= obj->efile.sec_cnt) 3887 return -LIBBPF_ERRNO__FORMAT; 3888 3889 /* Only do relo for section with exec instructions */ 3890 if (!section_have_execinstr(obj, targ_sec_idx) && 3891 strcmp(name, ".rel" STRUCT_OPS_SEC) && 3892 strcmp(name, ".rel" MAPS_ELF_SEC)) { 3893#if defined HAVE_LIBELF 3894 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", 3895 idx, name, targ_sec_idx, 3896 elf_sec_name(obj, elf_sec_by_idx(obj, targ_sec_idx)) ?: "<?>"); 3897#elif defined HAVE_ELFIO 3898 pr_info("elf: skipping relo section(%d) %s for section(%d) %s\n", 3899 idx, name, targ_sec_idx, 3900 elf_sec_name_by_idx(obj, targ_sec_idx) ?: "<?>"); 3901#endif 3902 continue; 3903 } 3904 3905 sec_desc->sec_type = SEC_RELO; 3906#if defined HAVE_LIBELF 3907 sec_desc->shdr = sh; 3908#elif defined HAVE_ELFIO 3909 sec_desc->psection = ptmpsection; 3910#endif 3911 sec_desc->data = data; 3912 } else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 || 3913 str_has_pfx(name, BSS_SEC "."))) { 3914 sec_desc->sec_type = SEC_BSS; 3915#if defined HAVE_LIBELF 3916 sec_desc->shdr = sh; 3917#elif defined HAVE_ELFIO 3918 sec_desc->psection = ptmpsection; 3919#endif 3920 sec_desc->data = data; 3921 } else { 3922 pr_info("elf: skipping section(%d) %s (size %zu)\n", idx, name, 3923 (size_t)sh->sh_size); 3924 } 3925 } 3926 3927 if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) { 3928 pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path); 3929 return -LIBBPF_ERRNO__FORMAT; 3930 } 3931 3932 /* sort BPF programs by section name and in-section instruction offset 3933 * for faster search 3934 */ 3935 if (obj->nr_programs) 3936 qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs); 3937 3938 return bpf_object__init_btf(obj, btf_data, btf_ext_data); 3939} 3940 3941static bool sym_is_extern(const Elf64_Sym *sym) 3942{ 3943 int bind = ELF64_ST_BIND(sym->st_info); 3944 /* externs are symbols w/ type=NOTYPE, bind=GLOBAL|WEAK, section=UND */ 3945 return sym->st_shndx == SHN_UNDEF && 3946 (bind == STB_GLOBAL || bind == STB_WEAK) && 3947 ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE; 3948} 3949 3950static bool sym_is_subprog(const Elf64_Sym *sym, int text_shndx) 3951{ 3952 int bind = ELF64_ST_BIND(sym->st_info); 3953 int type = ELF64_ST_TYPE(sym->st_info); 3954 3955 /* in .text section */ 3956 if (sym->st_shndx != text_shndx) 3957 return false; 3958 3959 /* local function */ 3960 if (bind == STB_LOCAL && type == STT_SECTION) 3961 return true; 3962 3963 /* global function */ 3964 return bind == STB_GLOBAL && type == STT_FUNC; 3965} 3966 3967static int find_extern_btf_id(const struct btf *btf, const char *ext_name) 3968{ 3969 const struct btf_type *t; 3970 const char *tname; 3971 int i, n; 3972 3973 if (!btf) 3974 return -ESRCH; 3975 3976 n = btf__type_cnt(btf); 3977 for (i = 1; i < n; i++) { 3978 t = btf__type_by_id(btf, i); 3979 3980 if (!btf_is_var(t) && !btf_is_func(t)) 3981 continue; 3982 3983 tname = btf__name_by_offset(btf, t->name_off); 3984 if (strcmp(tname, ext_name)) 3985 continue; 3986 3987 if (btf_is_var(t) && 3988 btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN) 3989 return -EINVAL; 3990 3991 if (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_EXTERN) 3992 return -EINVAL; 3993 3994 return i; 3995 } 3996 3997 return -ENOENT; 3998} 3999 4000static int find_extern_sec_btf_id(struct btf *btf, int ext_btf_id) { 4001 const struct btf_var_secinfo *vs; 4002 const struct btf_type *t; 4003 int i, j, n; 4004 4005 if (!btf) 4006 return -ESRCH; 4007 4008 n = btf__type_cnt(btf); 4009 for (i = 1; i < n; i++) { 4010 t = btf__type_by_id(btf, i); 4011 4012 if (!btf_is_datasec(t)) 4013 continue; 4014 4015 vs = btf_var_secinfos(t); 4016 for (j = 0; j < btf_vlen(t); j++, vs++) { 4017 if (vs->type == ext_btf_id) 4018 return i; 4019 } 4020 } 4021 4022 return -ENOENT; 4023} 4024 4025static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, 4026 bool *is_signed) 4027{ 4028 const struct btf_type *t; 4029 const char *name; 4030 4031 t = skip_mods_and_typedefs(btf, id, NULL); 4032 name = btf__name_by_offset(btf, t->name_off); 4033 4034 if (is_signed) 4035 *is_signed = false; 4036 switch (btf_kind(t)) { 4037 case BTF_KIND_INT: { 4038 int enc = btf_int_encoding(t); 4039 4040 if (enc & BTF_INT_BOOL) 4041 return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN; 4042 if (is_signed) 4043 *is_signed = enc & BTF_INT_SIGNED; 4044 if (t->size == 1) 4045 return KCFG_CHAR; 4046 if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1))) 4047 return KCFG_UNKNOWN; 4048 return KCFG_INT; 4049 } 4050 case BTF_KIND_ENUM: 4051 if (t->size != 4) 4052 return KCFG_UNKNOWN; 4053 if (strcmp(name, "libbpf_tristate")) 4054 return KCFG_UNKNOWN; 4055 return KCFG_TRISTATE; 4056 case BTF_KIND_ENUM64: 4057 if (strcmp(name, "libbpf_tristate")) 4058 return KCFG_UNKNOWN; 4059 return KCFG_TRISTATE; 4060 case BTF_KIND_ARRAY: 4061 if (btf_array(t)->nelems == 0) 4062 return KCFG_UNKNOWN; 4063 if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR) 4064 return KCFG_UNKNOWN; 4065 return KCFG_CHAR_ARR; 4066 default: 4067 return KCFG_UNKNOWN; 4068 } 4069} 4070 4071static int cmp_externs(const void *_a, const void *_b) 4072{ 4073 const struct extern_desc *a = _a; 4074 const struct extern_desc *b = _b; 4075 4076 if (a->type != b->type) 4077 return a->type < b->type ? -1 : 1; 4078 4079 if (a->type == EXT_KCFG) { 4080 /* descending order by alignment requirements */ 4081 if (a->kcfg.align != b->kcfg.align) 4082 return a->kcfg.align > b->kcfg.align ? -1 : 1; 4083 /* ascending order by size, within same alignment class */ 4084 if (a->kcfg.sz != b->kcfg.sz) 4085 return a->kcfg.sz < b->kcfg.sz ? -1 : 1; 4086 } 4087 4088 /* resolve ties by name */ 4089 return strcmp(a->name, b->name); 4090} 4091 4092static int find_int_btf_id(const struct btf *btf) 4093{ 4094 const struct btf_type *t; 4095 int i, n; 4096 4097 n = btf__type_cnt(btf); 4098 for (i = 1; i < n; i++) { 4099 t = btf__type_by_id(btf, i); 4100 4101 if (btf_is_int(t) && btf_int_bits(t) == 32) 4102 return i; 4103 } 4104 4105 return 0; 4106} 4107 4108static int add_dummy_ksym_var(struct btf *btf) 4109{ 4110 int i, int_btf_id, sec_btf_id, dummy_var_btf_id; 4111 const struct btf_var_secinfo *vs; 4112 const struct btf_type *sec; 4113 4114 if (!btf) 4115 return 0; 4116 4117 sec_btf_id = btf__find_by_name_kind(btf, KSYMS_SEC, 4118 BTF_KIND_DATASEC); 4119 if (sec_btf_id < 0) 4120 return 0; 4121 4122 sec = btf__type_by_id(btf, sec_btf_id); 4123 vs = btf_var_secinfos(sec); 4124 for (i = 0; i < btf_vlen(sec); i++, vs++) { 4125 const struct btf_type *vt; 4126 4127 vt = btf__type_by_id(btf, vs->type); 4128 if (btf_is_func(vt)) 4129 break; 4130 } 4131 4132 /* No func in ksyms sec. No need to add dummy var. */ 4133 if (i == btf_vlen(sec)) 4134 return 0; 4135 4136 int_btf_id = find_int_btf_id(btf); 4137 dummy_var_btf_id = btf__add_var(btf, 4138 "dummy_ksym", 4139 BTF_VAR_GLOBAL_ALLOCATED, 4140 int_btf_id); 4141 if (dummy_var_btf_id < 0) 4142 pr_warn("cannot create a dummy_ksym var\n"); 4143 4144 return dummy_var_btf_id; 4145} 4146 4147static int bpf_object__collect_externs(struct bpf_object *obj) 4148{ 4149 struct btf_type *sec, *kcfg_sec = NULL, *ksym_sec = NULL; 4150 const struct btf_type *t; 4151 struct extern_desc *ext; 4152 int i, n, off, dummy_var_btf_id; 4153 const char *ext_name, *sec_name; 4154 size_t ext_essent_len; 4155#ifdef HAVE_LIBELF 4156 Elf_Scn *scn; 4157#endif 4158 Elf64_Shdr *sh; 4159 Elf64_Shdr shheader; 4160 4161 if (!obj->efile.symbols) 4162 return 0; 4163 4164#if defined HAVE_LIBELF 4165 scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); 4166 sh = elf_sec_hdr(obj, scn); 4167#elif defined HAVE_ELFIO 4168 sh = &shheader; 4169 sh = elf_sec_hdr_by_idx(obj, obj->efile.symbols_shndx, sh); 4170#endif 4171 4172 if (!sh || sh->sh_entsize != sizeof(Elf64_Sym)) 4173 return -LIBBPF_ERRNO__FORMAT; 4174 4175 dummy_var_btf_id = add_dummy_ksym_var(obj->btf); 4176 if (dummy_var_btf_id < 0) 4177 return dummy_var_btf_id; 4178 4179 n = sh->sh_size / sh->sh_entsize; 4180 pr_debug("looking for externs among %d symbols...\n", n); 4181 4182 for (i = 0; i < n; i++) { 4183 Elf64_Sym *sym = elf_sym_by_idx(obj, i); 4184 4185 if (!sym) 4186 return -LIBBPF_ERRNO__FORMAT; 4187 if (!sym_is_extern(sym)) 4188 continue; 4189 ext_name = elf_sym_str(obj, sym->st_name); 4190 if (!ext_name || !ext_name[0]) 4191 continue; 4192 4193 ext = obj->externs; 4194 ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext)); 4195 if (!ext) 4196 return -ENOMEM; 4197 obj->externs = ext; 4198 ext = &ext[obj->nr_extern]; 4199 memset(ext, 0, sizeof(*ext)); 4200 obj->nr_extern++; 4201 4202 ext->btf_id = find_extern_btf_id(obj->btf, ext_name); 4203 if (ext->btf_id <= 0) { 4204 pr_warn("failed to find BTF for extern '%s': %d\n", 4205 ext_name, ext->btf_id); 4206 return ext->btf_id; 4207 } 4208 t = btf__type_by_id(obj->btf, ext->btf_id); 4209 ext->name = btf__name_by_offset(obj->btf, t->name_off); 4210 ext->sym_idx = i; 4211 ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK; 4212 4213 ext_essent_len = bpf_core_essential_name_len(ext->name); 4214 ext->essent_name = NULL; 4215 if (ext_essent_len != strlen(ext->name)) { 4216 ext->essent_name = strndup(ext->name, ext_essent_len); 4217 if (!ext->essent_name) 4218 return -ENOMEM; 4219 } 4220 4221 ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id); 4222 if (ext->sec_btf_id <= 0) { 4223 pr_warn("failed to find BTF for extern '%s' [%d] section: %d\n", 4224 ext_name, ext->btf_id, ext->sec_btf_id); 4225 return ext->sec_btf_id; 4226 } 4227 sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id); 4228 sec_name = btf__name_by_offset(obj->btf, sec->name_off); 4229 4230 if (strcmp(sec_name, KCONFIG_SEC) == 0) { 4231 if (btf_is_func(t)) { 4232 pr_warn("extern function %s is unsupported under %s section\n", 4233 ext->name, KCONFIG_SEC); 4234 return -ENOTSUP; 4235 } 4236 kcfg_sec = sec; 4237 ext->type = EXT_KCFG; 4238 ext->kcfg.sz = btf__resolve_size(obj->btf, t->type); 4239 if (ext->kcfg.sz <= 0) { 4240 pr_warn("failed to resolve size of extern (kcfg) '%s': %d\n", 4241 ext_name, ext->kcfg.sz); 4242 return ext->kcfg.sz; 4243 } 4244 ext->kcfg.align = btf__align_of(obj->btf, t->type); 4245 if (ext->kcfg.align <= 0) { 4246 pr_warn("failed to determine alignment of extern (kcfg) '%s': %d\n", 4247 ext_name, ext->kcfg.align); 4248 return -EINVAL; 4249 } 4250 ext->kcfg.type = find_kcfg_type(obj->btf, t->type, 4251 &ext->kcfg.is_signed); 4252 if (ext->kcfg.type == KCFG_UNKNOWN) { 4253 pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name); 4254 return -ENOTSUP; 4255 } 4256 } else if (strcmp(sec_name, KSYMS_SEC) == 0) { 4257 ksym_sec = sec; 4258 ext->type = EXT_KSYM; 4259 skip_mods_and_typedefs(obj->btf, t->type, 4260 &ext->ksym.type_id); 4261 } else { 4262 pr_warn("unrecognized extern section '%s'\n", sec_name); 4263 return -ENOTSUP; 4264 } 4265 } 4266 pr_debug("collected %d externs total\n", obj->nr_extern); 4267 4268 if (!obj->nr_extern) 4269 return 0; 4270 4271 /* sort externs by type, for kcfg ones also by (align, size, name) */ 4272 qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs); 4273 4274 /* for .ksyms section, we need to turn all externs into allocated 4275 * variables in BTF to pass kernel verification; we do this by 4276 * pretending that each extern is a 8-byte variable 4277 */ 4278 if (ksym_sec) { 4279 /* find existing 4-byte integer type in BTF to use for fake 4280 * extern variables in DATASEC 4281 */ 4282 int int_btf_id = find_int_btf_id(obj->btf); 4283 /* For extern function, a dummy_var added earlier 4284 * will be used to replace the vs->type and 4285 * its name string will be used to refill 4286 * the missing param's name. 4287 */ 4288 const struct btf_type *dummy_var; 4289 4290 dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id); 4291 for (i = 0; i < obj->nr_extern; i++) { 4292 ext = &obj->externs[i]; 4293 if (ext->type != EXT_KSYM) 4294 continue; 4295 pr_debug("extern (ksym) #%d: symbol %d, name %s\n", 4296 i, ext->sym_idx, ext->name); 4297 } 4298 4299 sec = ksym_sec; 4300 n = btf_vlen(sec); 4301 for (i = 0, off = 0; i < n; i++, off += sizeof(int)) { 4302 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 4303 struct btf_type *vt; 4304 4305 vt = (void *)btf__type_by_id(obj->btf, vs->type); 4306 ext_name = btf__name_by_offset(obj->btf, vt->name_off); 4307 ext = find_extern_by_name(obj, ext_name); 4308 if (!ext) { 4309 pr_warn("failed to find extern definition for BTF %s '%s'\n", 4310 btf_kind_str(vt), ext_name); 4311 return -ESRCH; 4312 } 4313 if (btf_is_func(vt)) { 4314 const struct btf_type *func_proto; 4315 struct btf_param *param; 4316 int j; 4317 4318 func_proto = btf__type_by_id(obj->btf, 4319 vt->type); 4320 param = btf_params(func_proto); 4321 /* Reuse the dummy_var string if the 4322 * func proto does not have param name. 4323 */ 4324 for (j = 0; j < btf_vlen(func_proto); j++) 4325 if (param[j].type && !param[j].name_off) 4326 param[j].name_off = 4327 dummy_var->name_off; 4328 vs->type = dummy_var_btf_id; 4329 vt->info &= ~0xffff; 4330 vt->info |= BTF_FUNC_GLOBAL; 4331 } else { 4332 btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 4333 vt->type = int_btf_id; 4334 } 4335 vs->offset = off; 4336 vs->size = sizeof(int); 4337 } 4338 sec->size = off; 4339 } 4340 4341 if (kcfg_sec) { 4342 sec = kcfg_sec; 4343 /* for kcfg externs calculate their offsets within a .kconfig map */ 4344 off = 0; 4345 for (i = 0; i < obj->nr_extern; i++) { 4346 ext = &obj->externs[i]; 4347 if (ext->type != EXT_KCFG) 4348 continue; 4349 4350 ext->kcfg.data_off = roundup(off, ext->kcfg.align); 4351 off = ext->kcfg.data_off + ext->kcfg.sz; 4352 pr_debug("extern (kcfg) #%d: symbol %d, off %u, name %s\n", 4353 i, ext->sym_idx, ext->kcfg.data_off, ext->name); 4354 } 4355 sec->size = off; 4356 n = btf_vlen(sec); 4357 for (i = 0; i < n; i++) { 4358 struct btf_var_secinfo *vs = btf_var_secinfos(sec) + i; 4359 4360 t = btf__type_by_id(obj->btf, vs->type); 4361 ext_name = btf__name_by_offset(obj->btf, t->name_off); 4362 ext = find_extern_by_name(obj, ext_name); 4363 if (!ext) { 4364 pr_warn("failed to find extern definition for BTF var '%s'\n", 4365 ext_name); 4366 return -ESRCH; 4367 } 4368 btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 4369 vs->offset = ext->kcfg.data_off; 4370 } 4371 } 4372 return 0; 4373} 4374 4375static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog) 4376{ 4377 return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; 4378} 4379 4380struct bpf_program * 4381bpf_object__find_program_by_name(const struct bpf_object *obj, 4382 const char *name) 4383{ 4384 struct bpf_program *prog; 4385 4386 bpf_object__for_each_program(prog, obj) { 4387 if (prog_is_subprog(obj, prog)) 4388 continue; 4389 if (!strcmp(prog->name, name)) 4390 return prog; 4391 } 4392 return errno = ENOENT, NULL; 4393} 4394 4395static bool bpf_object__shndx_is_data(const struct bpf_object *obj, 4396 int shndx) 4397{ 4398 switch (obj->efile.secs[shndx].sec_type) { 4399 case SEC_BSS: 4400 case SEC_DATA: 4401 case SEC_RODATA: 4402 return true; 4403 default: 4404 return false; 4405 } 4406} 4407 4408static bool bpf_object__shndx_is_maps(const struct bpf_object *obj, 4409 int shndx) 4410{ 4411 return shndx == obj->efile.btf_maps_shndx; 4412} 4413 4414static enum libbpf_map_type 4415bpf_object__section_to_libbpf_map_type(const struct bpf_object *obj, int shndx) 4416{ 4417 if (shndx == obj->efile.symbols_shndx) 4418 return LIBBPF_MAP_KCONFIG; 4419 4420 switch (obj->efile.secs[shndx].sec_type) { 4421 case SEC_BSS: 4422 return LIBBPF_MAP_BSS; 4423 case SEC_DATA: 4424 return LIBBPF_MAP_DATA; 4425 case SEC_RODATA: 4426 return LIBBPF_MAP_RODATA; 4427 default: 4428 return LIBBPF_MAP_UNSPEC; 4429 } 4430} 4431 4432static int bpf_program__record_reloc(struct bpf_program *prog, 4433 struct reloc_desc *reloc_desc, 4434 __u32 insn_idx, const char *sym_name, 4435 const Elf64_Sym *sym, const Elf64_Rel *rel) 4436{ 4437 struct bpf_insn *insn = &prog->insns[insn_idx]; 4438 size_t map_idx, nr_maps = prog->obj->nr_maps; 4439 struct bpf_object *obj = prog->obj; 4440 __u32 shdr_idx = sym->st_shndx; 4441 enum libbpf_map_type type; 4442 const char *sym_sec_name; 4443 struct bpf_map *map; 4444 4445 if (!is_call_insn(insn) && !is_ldimm64_insn(insn)) { 4446 pr_warn("prog '%s': invalid relo against '%s' for insns[%d].code 0x%x\n", 4447 prog->name, sym_name, insn_idx, insn->code); 4448 return -LIBBPF_ERRNO__RELOC; 4449 } 4450 4451 if (sym_is_extern(sym)) { 4452 int sym_idx = ELF64_R_SYM(rel->r_info); 4453 int i, n = obj->nr_extern; 4454 struct extern_desc *ext; 4455 4456 for (i = 0; i < n; i++) { 4457 ext = &obj->externs[i]; 4458 if (ext->sym_idx == sym_idx) 4459 break; 4460 } 4461 if (i >= n) { 4462 pr_warn("prog '%s': extern relo failed to find extern for '%s' (%d)\n", 4463 prog->name, sym_name, sym_idx); 4464 return -LIBBPF_ERRNO__RELOC; 4465 } 4466 pr_debug("prog '%s': found extern #%d '%s' (sym %d) for insn #%u\n", 4467 prog->name, i, ext->name, ext->sym_idx, insn_idx); 4468 if (insn->code == (BPF_JMP | BPF_CALL)) 4469 reloc_desc->type = RELO_EXTERN_CALL; 4470 else 4471 reloc_desc->type = RELO_EXTERN_LD64; 4472 reloc_desc->insn_idx = insn_idx; 4473 reloc_desc->ext_idx = i; 4474 return 0; 4475 } 4476 4477 /* sub-program call relocation */ 4478 if (is_call_insn(insn)) { 4479 if (insn->src_reg != BPF_PSEUDO_CALL) { 4480 pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name); 4481 return -LIBBPF_ERRNO__RELOC; 4482 } 4483 /* text_shndx can be 0, if no default "main" program exists */ 4484 if (!shdr_idx || shdr_idx != obj->efile.text_shndx) { 4485#if defined HAVE_LIBELF 4486 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 4487#elif defined HAVE_ELFIO 4488 sym_sec_name = elf_sec_name_by_idx(obj, shdr_idx); 4489#endif 4490 pr_warn("prog '%s': bad call relo against '%s' in section '%s'\n", 4491 prog->name, sym_name, sym_sec_name); 4492 return -LIBBPF_ERRNO__RELOC; 4493 } 4494 if (sym->st_value % BPF_INSN_SZ) { 4495 pr_warn("prog '%s': bad call relo against '%s' at offset %zu\n", 4496 prog->name, sym_name, (size_t)sym->st_value); 4497 return -LIBBPF_ERRNO__RELOC; 4498 } 4499 reloc_desc->type = RELO_CALL; 4500 reloc_desc->insn_idx = insn_idx; 4501 reloc_desc->sym_off = sym->st_value; 4502 return 0; 4503 } 4504 4505 if (!shdr_idx || shdr_idx >= SHN_LORESERVE) { 4506 pr_warn("prog '%s': invalid relo against '%s' in special section 0x%x; forgot to initialize global var?..\n", 4507 prog->name, sym_name, shdr_idx); 4508 return -LIBBPF_ERRNO__RELOC; 4509 } 4510 4511 /* loading subprog addresses */ 4512 if (sym_is_subprog(sym, obj->efile.text_shndx)) { 4513 /* global_func: sym->st_value = offset in the section, insn->imm = 0. 4514 * local_func: sym->st_value = 0, insn->imm = offset in the section. 4515 */ 4516 if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) { 4517 pr_warn("prog '%s': bad subprog addr relo against '%s' at offset %zu+%d\n", 4518 prog->name, sym_name, (size_t)sym->st_value, insn->imm); 4519 return -LIBBPF_ERRNO__RELOC; 4520 } 4521 4522 reloc_desc->type = RELO_SUBPROG_ADDR; 4523 reloc_desc->insn_idx = insn_idx; 4524 reloc_desc->sym_off = sym->st_value; 4525 return 0; 4526 } 4527 4528 type = bpf_object__section_to_libbpf_map_type(obj, shdr_idx); 4529#if defined HAVE_LIBELF 4530 sym_sec_name = elf_sec_name(obj, elf_sec_by_idx(obj, shdr_idx)); 4531#elif defined HAVE_ELFIO 4532 sym_sec_name = elf_sec_name_by_idx(obj, shdr_idx); 4533#endif 4534 /* generic map reference relocation */ 4535 if (type == LIBBPF_MAP_UNSPEC) { 4536 if (!bpf_object__shndx_is_maps(obj, shdr_idx)) { 4537 pr_warn("prog '%s': bad map relo against '%s' in section '%s'\n", 4538 prog->name, sym_name, sym_sec_name); 4539 return -LIBBPF_ERRNO__RELOC; 4540 } 4541 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 4542 map = &obj->maps[map_idx]; 4543 if (map->libbpf_type != type || 4544 map->sec_idx != sym->st_shndx || 4545 map->sec_offset != sym->st_value) 4546 continue; 4547 pr_debug("prog '%s': found map %zd (%s, sec %d, off %zu) for insn #%u\n", 4548 prog->name, map_idx, map->name, map->sec_idx, 4549 map->sec_offset, insn_idx); 4550 break; 4551 } 4552 if (map_idx >= nr_maps) { 4553 pr_warn("prog '%s': map relo failed to find map for section '%s', off %zu\n", 4554 prog->name, sym_sec_name, (size_t)sym->st_value); 4555 return -LIBBPF_ERRNO__RELOC; 4556 } 4557 reloc_desc->type = RELO_LD64; 4558 reloc_desc->insn_idx = insn_idx; 4559 reloc_desc->map_idx = map_idx; 4560 reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */ 4561 return 0; 4562 } 4563 4564 /* global data map relocation */ 4565 if (!bpf_object__shndx_is_data(obj, shdr_idx)) { 4566 pr_warn("prog '%s': bad data relo against section '%s'\n", 4567 prog->name, sym_sec_name); 4568 return -LIBBPF_ERRNO__RELOC; 4569 } 4570 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 4571 map = &obj->maps[map_idx]; 4572 if (map->libbpf_type != type || map->sec_idx != sym->st_shndx) 4573 continue; 4574 pr_debug("prog '%s': found data map %zd (%s, sec %d, off %zu) for insn %u\n", 4575 prog->name, map_idx, map->name, map->sec_idx, 4576 map->sec_offset, insn_idx); 4577 break; 4578 } 4579 if (map_idx >= nr_maps) { 4580 pr_warn("prog '%s': data relo failed to find map for section '%s'\n", 4581 prog->name, sym_sec_name); 4582 return -LIBBPF_ERRNO__RELOC; 4583 } 4584 4585 reloc_desc->type = RELO_DATA; 4586 reloc_desc->insn_idx = insn_idx; 4587 reloc_desc->map_idx = map_idx; 4588 reloc_desc->sym_off = sym->st_value; 4589 return 0; 4590} 4591 4592static bool prog_contains_insn(const struct bpf_program *prog, size_t insn_idx) 4593{ 4594 return insn_idx >= prog->sec_insn_off && 4595 insn_idx < prog->sec_insn_off + prog->sec_insn_cnt; 4596} 4597 4598static struct bpf_program *find_prog_by_sec_insn(const struct bpf_object *obj, 4599 size_t sec_idx, size_t insn_idx) 4600{ 4601 int l = 0, r = obj->nr_programs - 1, m; 4602 struct bpf_program *prog; 4603 4604 if (!obj->nr_programs) 4605 return NULL; 4606 4607 while (l < r) { 4608 m = l + (r - l + 1) / 2; 4609 prog = &obj->programs[m]; 4610 4611 if (prog->sec_idx < sec_idx || 4612 (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx)) 4613 l = m; 4614 else 4615 r = m - 1; 4616 } 4617 /* matching program could be at index l, but it still might be the 4618 * wrong one, so we need to double check conditions for the last time 4619 */ 4620 prog = &obj->programs[l]; 4621 if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx)) 4622 return prog; 4623 return NULL; 4624} 4625 4626static int 4627bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) 4628{ 4629 const char *relo_sec_name, *sec_name; 4630 size_t sec_idx = shdr->sh_info, sym_idx; 4631 struct bpf_program *prog; 4632 struct reloc_desc *relos; 4633 int err, i, nrels; 4634 const char *sym_name; 4635 __u32 insn_idx; 4636#ifdef HAVE_LIBELF 4637 Elf_Scn *scn; 4638#endif 4639 Elf_Data *scn_data; 4640 Elf64_Sym *sym; 4641 Elf64_Rel *rel; 4642 4643 if (sec_idx >= obj->efile.sec_cnt) 4644 return -EINVAL; 4645 4646#if defined HAVE_LIBELF 4647 scn = elf_sec_by_idx(obj, sec_idx); 4648 scn_data = elf_sec_data(obj, scn); 4649 4650 relo_sec_name = elf_sec_str(obj, shdr->sh_name); 4651 sec_name = elf_sec_name(obj, scn); 4652 if (!relo_sec_name || !sec_name) 4653 return -EINVAL; 4654#elif defined HAVE_ELFIO 4655 Elf_Data realdata; 4656 scn_data = elf_sec_data_by_idx(obj, sec_idx, &realdata); 4657 4658 relo_sec_name = elf_sec_str(obj, shdr->sh_name); 4659 sec_name = elf_sec_name_by_idx(obj, sec_idx); 4660 if (!relo_sec_name || !sec_name) 4661 return -EINVAL; 4662#endif 4663 4664 pr_debug("sec '%s': collecting relocation for section(%zu) '%s'\n", 4665 relo_sec_name, sec_idx, sec_name); 4666 nrels = shdr->sh_size / shdr->sh_entsize; 4667 4668 for (i = 0; i < nrels; i++) { 4669 rel = elf_rel_by_idx(data, i); 4670 if (!rel) { 4671 pr_warn("sec '%s': failed to get relo #%d\n", relo_sec_name, i); 4672 return -LIBBPF_ERRNO__FORMAT; 4673 } 4674 4675 sym_idx = ELF64_R_SYM(rel->r_info); 4676 sym = elf_sym_by_idx(obj, sym_idx); 4677 if (!sym) { 4678 pr_warn("sec '%s': symbol #%zu not found for relo #%d\n", 4679 relo_sec_name, sym_idx, i); 4680 return -LIBBPF_ERRNO__FORMAT; 4681 } 4682 4683 if (sym->st_shndx >= obj->efile.sec_cnt) { 4684 pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n", 4685 relo_sec_name, sym_idx, (size_t)sym->st_shndx, i); 4686 return -LIBBPF_ERRNO__FORMAT; 4687 } 4688 4689 if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) { 4690 pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", 4691 relo_sec_name, (size_t)rel->r_offset, i); 4692 return -LIBBPF_ERRNO__FORMAT; 4693 } 4694 4695 insn_idx = rel->r_offset / BPF_INSN_SZ; 4696 /* relocations against static functions are recorded as 4697 * relocations against the section that contains a function; 4698 * in such case, symbol will be STT_SECTION and sym.st_name 4699 * will point to empty string (0), so fetch section name 4700 * instead 4701 */ 4702 if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0) 4703#if defined HAVE_LIBELF 4704 sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx)); 4705#elif defined HAVE_ELFIO 4706 sym_name = elf_sec_name_by_idx(obj, sym->st_shndx); 4707#endif 4708 else 4709 sym_name = elf_sym_str(obj, sym->st_name); 4710 sym_name = sym_name ?: "<?"; 4711 4712 pr_debug("sec '%s': relo #%d: insn #%u against '%s'\n", 4713 relo_sec_name, i, insn_idx, sym_name); 4714 4715 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 4716 if (!prog) { 4717 pr_debug("sec '%s': relo #%d: couldn't find program in section '%s' for insn #%u, probably overridden weak function, skipping...\n", 4718 relo_sec_name, i, sec_name, insn_idx); 4719 continue; 4720 } 4721 4722 relos = libbpf_reallocarray(prog->reloc_desc, 4723 prog->nr_reloc + 1, sizeof(*relos)); 4724 if (!relos) 4725 return -ENOMEM; 4726 prog->reloc_desc = relos; 4727 4728 /* adjust insn_idx to local BPF program frame of reference */ 4729 insn_idx -= prog->sec_insn_off; 4730 err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc], 4731 insn_idx, sym_name, sym, rel); 4732 if (err) 4733 return err; 4734 4735 prog->nr_reloc++; 4736 } 4737 return 0; 4738} 4739 4740static int map_fill_btf_type_info(struct bpf_object *obj, struct bpf_map *map) 4741{ 4742 int id; 4743 4744 if (!obj->btf) 4745 return -ENOENT; 4746 4747 /* if it's BTF-defined map, we don't need to search for type IDs. 4748 * For struct_ops map, it does not need btf_key_type_id and 4749 * btf_value_type_id. 4750 */ 4751 if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map)) 4752 return 0; 4753 4754 /* 4755 * LLVM annotates global data differently in BTF, that is, 4756 * only as '.data', '.bss' or '.rodata'. 4757 */ 4758 if (!bpf_map__is_internal(map)) 4759 return -ENOENT; 4760 4761 id = btf__find_by_name(obj->btf, map->real_name); 4762 if (id < 0) 4763 return id; 4764 4765 map->btf_key_type_id = 0; 4766 map->btf_value_type_id = id; 4767 return 0; 4768} 4769 4770static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) 4771{ 4772 char file[PATH_MAX], buff[4096]; 4773 FILE *fp; 4774 __u32 val; 4775 int err; 4776 4777 snprintf(file, sizeof(file), "/proc/%d/fdinfo/%d", getpid(), fd); 4778 memset(info, 0, sizeof(*info)); 4779 4780 fp = fopen(file, "re"); 4781 if (!fp) { 4782 err = -errno; 4783 pr_warn("failed to open %s: %d. No procfs support?\n", file, 4784 err); 4785 return err; 4786 } 4787 4788 while (fgets(buff, sizeof(buff), fp)) { 4789 if (sscanf(buff, "map_type:\t%u", &val) == 1) 4790 info->type = val; 4791 else if (sscanf(buff, "key_size:\t%u", &val) == 1) 4792 info->key_size = val; 4793 else if (sscanf(buff, "value_size:\t%u", &val) == 1) 4794 info->value_size = val; 4795 else if (sscanf(buff, "max_entries:\t%u", &val) == 1) 4796 info->max_entries = val; 4797 else if (sscanf(buff, "map_flags:\t%i", &val) == 1) 4798 info->map_flags = val; 4799 } 4800 4801 fclose(fp); 4802 4803 return 0; 4804} 4805 4806bool bpf_map__autocreate(const struct bpf_map *map) 4807{ 4808 return map->autocreate; 4809} 4810 4811int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) 4812{ 4813 if (map->obj->loaded) 4814 return libbpf_err(-EBUSY); 4815 4816 map->autocreate = autocreate; 4817 return 0; 4818} 4819 4820int bpf_map__reuse_fd(struct bpf_map *map, int fd) 4821{ 4822 struct bpf_map_info info; 4823 __u32 len = sizeof(info), name_len; 4824 int new_fd, err; 4825 char *new_name; 4826 4827 memset(&info, 0, len); 4828 err = bpf_map_get_info_by_fd(fd, &info, &len); 4829 if (err && errno == EINVAL) 4830 err = bpf_get_map_info_from_fdinfo(fd, &info); 4831 if (err) 4832 return libbpf_err(err); 4833 4834 name_len = strlen(info.name); 4835 if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0) 4836 new_name = strdup(map->name); 4837 else 4838 new_name = strdup(info.name); 4839 4840 if (!new_name) 4841 return libbpf_err(-errno); 4842 4843 /* 4844 * Like dup(), but make sure new FD is >= 3 and has O_CLOEXEC set. 4845 * This is similar to what we do in ensure_good_fd(), but without 4846 * closing original FD. 4847 */ 4848 new_fd = fcntl(fd, F_DUPFD_CLOEXEC, 3); 4849 if (new_fd < 0) { 4850 err = -errno; 4851 goto err_free_new_name; 4852 } 4853 4854 err = zclose(map->fd); 4855 if (err) { 4856 err = -errno; 4857 goto err_close_new_fd; 4858 } 4859 free(map->name); 4860 4861 map->fd = new_fd; 4862 map->name = new_name; 4863 map->def.type = info.type; 4864 map->def.key_size = info.key_size; 4865 map->def.value_size = info.value_size; 4866 map->def.max_entries = info.max_entries; 4867 map->def.map_flags = info.map_flags; 4868 map->btf_key_type_id = info.btf_key_type_id; 4869 map->btf_value_type_id = info.btf_value_type_id; 4870 map->reused = true; 4871 map->map_extra = info.map_extra; 4872 4873 return 0; 4874 4875err_close_new_fd: 4876 close(new_fd); 4877err_free_new_name: 4878 free(new_name); 4879 return libbpf_err(err); 4880} 4881 4882__u32 bpf_map__max_entries(const struct bpf_map *map) 4883{ 4884 return map->def.max_entries; 4885} 4886 4887struct bpf_map *bpf_map__inner_map(struct bpf_map *map) 4888{ 4889 if (!bpf_map_type__is_map_in_map(map->def.type)) 4890 return errno = EINVAL, NULL; 4891 4892 return map->inner_map; 4893} 4894 4895int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) 4896{ 4897 if (map->obj->loaded) 4898 return libbpf_err(-EBUSY); 4899 4900 map->def.max_entries = max_entries; 4901 4902 /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ 4903 if (map_is_ringbuf(map)) 4904 map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); 4905 4906 return 0; 4907} 4908 4909static int 4910bpf_object__probe_loading(struct bpf_object *obj) 4911{ 4912 char *cp, errmsg[STRERR_BUFSIZE]; 4913 struct bpf_insn insns[] = { 4914 BPF_MOV64_IMM(BPF_REG_0, 0), 4915 BPF_EXIT_INSN(), 4916 }; 4917 int ret, insn_cnt = ARRAY_SIZE(insns); 4918 4919 if (obj->gen_loader) 4920 return 0; 4921 4922 ret = bump_rlimit_memlock(); 4923 if (ret) 4924 pr_warn("Failed to bump RLIMIT_MEMLOCK (err = %d), you might need to do it explicitly!\n", ret); 4925 4926 /* make sure basic loading works */ 4927 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); 4928 if (ret < 0) 4929 ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); 4930 if (ret < 0) { 4931 ret = errno; 4932 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4933 pr_warn("Error in %s():%s(%d). Couldn't load trivial BPF " 4934 "program. Make sure your kernel supports BPF " 4935 "(CONFIG_BPF_SYSCALL=y) and/or that RLIMIT_MEMLOCK is " 4936 "set to big enough value.\n", __func__, cp, ret); 4937 return -ret; 4938 } 4939 close(ret); 4940 4941 return 0; 4942} 4943 4944static int probe_fd(int fd) 4945{ 4946 if (fd >= 0) 4947 close(fd); 4948 return fd >= 0; 4949} 4950 4951static int probe_kern_prog_name(void) 4952{ 4953 const size_t attr_sz = offsetofend(union bpf_attr, prog_name); 4954 struct bpf_insn insns[] = { 4955 BPF_MOV64_IMM(BPF_REG_0, 0), 4956 BPF_EXIT_INSN(), 4957 }; 4958 union bpf_attr attr; 4959 int ret; 4960 4961 memset(&attr, 0, attr_sz); 4962 attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 4963 attr.license = ptr_to_u64("GPL"); 4964 attr.insns = ptr_to_u64(insns); 4965 attr.insn_cnt = (__u32)ARRAY_SIZE(insns); 4966 libbpf_strlcpy(attr.prog_name, "libbpf_nametest", sizeof(attr.prog_name)); 4967 4968 /* make sure loading with name works */ 4969 ret = sys_bpf_prog_load(&attr, attr_sz, PROG_LOAD_ATTEMPTS); 4970 return probe_fd(ret); 4971} 4972 4973static int probe_kern_global_data(void) 4974{ 4975 char *cp, errmsg[STRERR_BUFSIZE]; 4976 struct bpf_insn insns[] = { 4977 BPF_LD_MAP_VALUE(BPF_REG_1, 0, 16), 4978 BPF_ST_MEM(BPF_DW, BPF_REG_1, 0, 42), 4979 BPF_MOV64_IMM(BPF_REG_0, 0), 4980 BPF_EXIT_INSN(), 4981 }; 4982 int ret, map, insn_cnt = ARRAY_SIZE(insns); 4983 4984 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_global", sizeof(int), 32, 1, NULL); 4985 if (map < 0) { 4986 ret = -errno; 4987 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 4988 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 4989 __func__, cp, -ret); 4990 return ret; 4991 } 4992 4993 insns[0].imm = map; 4994 4995 ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); 4996 close(map); 4997 return probe_fd(ret); 4998} 4999 5000static int probe_kern_btf(void) 5001{ 5002 static const char strs[] = "\0int"; 5003 __u32 types[] = { 5004 /* int */ 5005 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 5006 }; 5007 5008 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 5009 strs, sizeof(strs))); 5010} 5011 5012static int probe_kern_btf_func(void) 5013{ 5014 static const char strs[] = "\0int\0x\0a"; 5015 /* void x(int a) {} */ 5016 __u32 types[] = { 5017 /* int */ 5018 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 5019 /* FUNC_PROTO */ /* [2] */ 5020 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 5021 BTF_PARAM_ENC(7, 1), 5022 /* FUNC x */ /* [3] */ 5023 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0), 2), 5024 }; 5025 5026 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 5027 strs, sizeof(strs))); 5028} 5029 5030static int probe_kern_btf_func_global(void) 5031{ 5032 static const char strs[] = "\0int\0x\0a"; 5033 /* static void x(int a) {} */ 5034 __u32 types[] = { 5035 /* int */ 5036 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 5037 /* FUNC_PROTO */ /* [2] */ 5038 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_FUNC_PROTO, 0, 1), 0), 5039 BTF_PARAM_ENC(7, 1), 5040 /* FUNC x BTF_FUNC_GLOBAL */ /* [3] */ 5041 BTF_TYPE_ENC(5, BTF_INFO_ENC(BTF_KIND_FUNC, 0, BTF_FUNC_GLOBAL), 2), 5042 }; 5043 5044 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 5045 strs, sizeof(strs))); 5046} 5047 5048static int probe_kern_btf_datasec(void) 5049{ 5050 static const char strs[] = "\0x\0.data"; 5051 /* static int a; */ 5052 __u32 types[] = { 5053 /* int */ 5054 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 5055 /* VAR x */ /* [2] */ 5056 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 5057 BTF_VAR_STATIC, 5058 /* DATASEC val */ /* [3] */ 5059 BTF_TYPE_ENC(3, BTF_INFO_ENC(BTF_KIND_DATASEC, 0, 1), 4), 5060 BTF_VAR_SECINFO_ENC(2, 0, 4), 5061 }; 5062 5063 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 5064 strs, sizeof(strs))); 5065} 5066 5067static int probe_kern_btf_float(void) 5068{ 5069 static const char strs[] = "\0float"; 5070 __u32 types[] = { 5071 /* float */ 5072 BTF_TYPE_FLOAT_ENC(1, 4), 5073 }; 5074 5075 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 5076 strs, sizeof(strs))); 5077} 5078 5079static int probe_kern_btf_decl_tag(void) 5080{ 5081 static const char strs[] = "\0tag"; 5082 __u32 types[] = { 5083 /* int */ 5084 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 5085 /* VAR x */ /* [2] */ 5086 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_VAR, 0, 0), 1), 5087 BTF_VAR_STATIC, 5088 /* attr */ 5089 BTF_TYPE_DECL_TAG_ENC(1, 2, -1), 5090 }; 5091 5092 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 5093 strs, sizeof(strs))); 5094} 5095 5096static int probe_kern_btf_type_tag(void) 5097{ 5098 static const char strs[] = "\0tag"; 5099 __u32 types[] = { 5100 /* int */ 5101 BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ 5102 /* attr */ 5103 BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */ 5104 /* ptr */ 5105 BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */ 5106 }; 5107 5108 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 5109 strs, sizeof(strs))); 5110} 5111 5112static int probe_kern_array_mmap(void) 5113{ 5114 LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = BPF_F_MMAPABLE); 5115 int fd; 5116 5117 fd = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_mmap", sizeof(int), sizeof(int), 1, &opts); 5118 return probe_fd(fd); 5119} 5120 5121static int probe_kern_exp_attach_type(void) 5122{ 5123 LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE); 5124 struct bpf_insn insns[] = { 5125 BPF_MOV64_IMM(BPF_REG_0, 0), 5126 BPF_EXIT_INSN(), 5127 }; 5128 int fd, insn_cnt = ARRAY_SIZE(insns); 5129 5130 /* use any valid combination of program type and (optional) 5131 * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) 5132 * to see if kernel supports expected_attach_type field for 5133 * BPF_PROG_LOAD command 5134 */ 5135 fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts); 5136 return probe_fd(fd); 5137} 5138 5139static int probe_kern_probe_read_kernel(void) 5140{ 5141 struct bpf_insn insns[] = { 5142 BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ 5143 BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ 5144 BPF_MOV64_IMM(BPF_REG_2, 8), /* r2 = 8 */ 5145 BPF_MOV64_IMM(BPF_REG_3, 0), /* r3 = 0 */ 5146 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), 5147 BPF_EXIT_INSN(), 5148 }; 5149 int fd, insn_cnt = ARRAY_SIZE(insns); 5150 5151 fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); 5152 return probe_fd(fd); 5153} 5154 5155static int probe_prog_bind_map(void) 5156{ 5157 char *cp, errmsg[STRERR_BUFSIZE]; 5158 struct bpf_insn insns[] = { 5159 BPF_MOV64_IMM(BPF_REG_0, 0), 5160 BPF_EXIT_INSN(), 5161 }; 5162 int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); 5163 5164 map = bpf_map_create(BPF_MAP_TYPE_ARRAY, "libbpf_det_bind", sizeof(int), 32, 1, NULL); 5165 if (map < 0) { 5166 ret = -errno; 5167 cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); 5168 pr_warn("Error in %s():%s(%d). Couldn't create simple array map.\n", 5169 __func__, cp, -ret); 5170 return ret; 5171 } 5172 5173 prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); 5174 if (prog < 0) { 5175 close(map); 5176 return 0; 5177 } 5178 5179 ret = bpf_prog_bind_map(prog, map, NULL); 5180 5181 close(map); 5182 close(prog); 5183 5184 return ret >= 0; 5185} 5186 5187static int probe_module_btf(void) 5188{ 5189 static const char strs[] = "\0int"; 5190 __u32 types[] = { 5191 /* int */ 5192 BTF_TYPE_INT_ENC(1, BTF_INT_SIGNED, 0, 32, 4), 5193 }; 5194 struct bpf_btf_info info; 5195 __u32 len = sizeof(info); 5196 char name[16]; 5197 int fd, err; 5198 5199 fd = libbpf__load_raw_btf((char *)types, sizeof(types), strs, sizeof(strs)); 5200 if (fd < 0) 5201 return 0; /* BTF not supported at all */ 5202 5203 memset(&info, 0, sizeof(info)); 5204 info.name = ptr_to_u64(name); 5205 info.name_len = sizeof(name); 5206 5207 /* check that BPF_OBJ_GET_INFO_BY_FD supports specifying name pointer; 5208 * kernel's module BTF support coincides with support for 5209 * name/name_len fields in struct bpf_btf_info. 5210 */ 5211 err = bpf_btf_get_info_by_fd(fd, &info, &len); 5212 close(fd); 5213 return !err; 5214} 5215 5216static int probe_perf_link(void) 5217{ 5218 struct bpf_insn insns[] = { 5219 BPF_MOV64_IMM(BPF_REG_0, 0), 5220 BPF_EXIT_INSN(), 5221 }; 5222 int prog_fd, link_fd, err; 5223 5224 prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", 5225 insns, ARRAY_SIZE(insns), NULL); 5226 if (prog_fd < 0) 5227 return -errno; 5228 5229 /* use invalid perf_event FD to get EBADF, if link is supported; 5230 * otherwise EINVAL should be returned 5231 */ 5232 link_fd = bpf_link_create(prog_fd, -1, BPF_PERF_EVENT, NULL); 5233 err = -errno; /* close() can clobber errno */ 5234 5235 if (link_fd >= 0) 5236 close(link_fd); 5237 close(prog_fd); 5238 5239 return link_fd < 0 && err == -EBADF; 5240} 5241 5242static int probe_uprobe_multi_link(void) 5243{ 5244 LIBBPF_OPTS(bpf_prog_load_opts, load_opts, 5245 .expected_attach_type = BPF_TRACE_UPROBE_MULTI, 5246 ); 5247 LIBBPF_OPTS(bpf_link_create_opts, link_opts); 5248 struct bpf_insn insns[] = { 5249 BPF_MOV64_IMM(BPF_REG_0, 0), 5250 BPF_EXIT_INSN(), 5251 }; 5252 int prog_fd, link_fd, err; 5253 unsigned long offset = 0; 5254 5255 prog_fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", 5256 insns, ARRAY_SIZE(insns), &load_opts); 5257 if (prog_fd < 0) 5258 return -errno; 5259 5260 /* Creating uprobe in '/' binary should fail with -EBADF. */ 5261 link_opts.uprobe_multi.path = "/"; 5262 link_opts.uprobe_multi.offsets = &offset; 5263 link_opts.uprobe_multi.cnt = 1; 5264 5265 link_fd = bpf_link_create(prog_fd, -1, BPF_TRACE_UPROBE_MULTI, &link_opts); 5266 err = -errno; /* close() can clobber errno */ 5267 5268 if (link_fd >= 0) 5269 close(link_fd); 5270 close(prog_fd); 5271 5272 return link_fd < 0 && err == -EBADF; 5273} 5274 5275static int probe_kern_bpf_cookie(void) 5276{ 5277 struct bpf_insn insns[] = { 5278 BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie), 5279 BPF_EXIT_INSN(), 5280 }; 5281 int ret, insn_cnt = ARRAY_SIZE(insns); 5282 5283 ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL); 5284 return probe_fd(ret); 5285} 5286 5287static int probe_kern_btf_enum64(void) 5288{ 5289 static const char strs[] = "\0enum64"; 5290 __u32 types[] = { 5291 BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8), 5292 }; 5293 5294 return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), 5295 strs, sizeof(strs))); 5296} 5297 5298static int probe_kern_syscall_wrapper(void); 5299 5300enum kern_feature_result { 5301 FEAT_UNKNOWN = 0, 5302 FEAT_SUPPORTED = 1, 5303 FEAT_MISSING = 2, 5304}; 5305 5306typedef int (*feature_probe_fn)(void); 5307 5308static struct kern_feature_desc { 5309 const char *desc; 5310 feature_probe_fn probe; 5311 enum kern_feature_result res; 5312} feature_probes[__FEAT_CNT] = { 5313 [FEAT_PROG_NAME] = { 5314 "BPF program name", probe_kern_prog_name, 5315 }, 5316 [FEAT_GLOBAL_DATA] = { 5317 "global variables", probe_kern_global_data, 5318 }, 5319 [FEAT_BTF] = { 5320 "minimal BTF", probe_kern_btf, 5321 }, 5322 [FEAT_BTF_FUNC] = { 5323 "BTF functions", probe_kern_btf_func, 5324 }, 5325 [FEAT_BTF_GLOBAL_FUNC] = { 5326 "BTF global function", probe_kern_btf_func_global, 5327 }, 5328 [FEAT_BTF_DATASEC] = { 5329 "BTF data section and variable", probe_kern_btf_datasec, 5330 }, 5331 [FEAT_ARRAY_MMAP] = { 5332 "ARRAY map mmap()", probe_kern_array_mmap, 5333 }, 5334 [FEAT_EXP_ATTACH_TYPE] = { 5335 "BPF_PROG_LOAD expected_attach_type attribute", 5336 probe_kern_exp_attach_type, 5337 }, 5338 [FEAT_PROBE_READ_KERN] = { 5339 "bpf_probe_read_kernel() helper", probe_kern_probe_read_kernel, 5340 }, 5341 [FEAT_PROG_BIND_MAP] = { 5342 "BPF_PROG_BIND_MAP support", probe_prog_bind_map, 5343 }, 5344 [FEAT_MODULE_BTF] = { 5345 "module BTF support", probe_module_btf, 5346 }, 5347 [FEAT_BTF_FLOAT] = { 5348 "BTF_KIND_FLOAT support", probe_kern_btf_float, 5349 }, 5350 [FEAT_PERF_LINK] = { 5351 "BPF perf link support", probe_perf_link, 5352 }, 5353 [FEAT_BTF_DECL_TAG] = { 5354 "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag, 5355 }, 5356 [FEAT_BTF_TYPE_TAG] = { 5357 "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, 5358 }, 5359 [FEAT_MEMCG_ACCOUNT] = { 5360 "memcg-based memory accounting", probe_memcg_account, 5361 }, 5362 [FEAT_BPF_COOKIE] = { 5363 "BPF cookie support", probe_kern_bpf_cookie, 5364 }, 5365 [FEAT_BTF_ENUM64] = { 5366 "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, 5367 }, 5368 [FEAT_SYSCALL_WRAPPER] = { 5369 "Kernel using syscall wrapper", probe_kern_syscall_wrapper, 5370 }, 5371 [FEAT_UPROBE_MULTI_LINK] = { 5372 "BPF multi-uprobe link support", probe_uprobe_multi_link, 5373 }, 5374}; 5375 5376bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) 5377{ 5378 struct kern_feature_desc *feat = &feature_probes[feat_id]; 5379 int ret; 5380 5381 if (obj && obj->gen_loader) 5382 /* To generate loader program assume the latest kernel 5383 * to avoid doing extra prog_load, map_create syscalls. 5384 */ 5385 return true; 5386 5387 if (READ_ONCE(feat->res) == FEAT_UNKNOWN) { 5388 ret = feat->probe(); 5389 if (ret > 0) { 5390 WRITE_ONCE(feat->res, FEAT_SUPPORTED); 5391 } else if (ret == 0) { 5392 WRITE_ONCE(feat->res, FEAT_MISSING); 5393 } else { 5394 pr_warn("Detection of kernel %s support failed: %d\n", feat->desc, ret); 5395 WRITE_ONCE(feat->res, FEAT_MISSING); 5396 } 5397 } 5398 5399 return READ_ONCE(feat->res) == FEAT_SUPPORTED; 5400} 5401 5402static bool map_is_reuse_compat(const struct bpf_map *map, int map_fd) 5403{ 5404 struct bpf_map_info map_info; 5405 char msg[STRERR_BUFSIZE]; 5406 __u32 map_info_len = sizeof(map_info); 5407 int err; 5408 5409 memset(&map_info, 0, map_info_len); 5410 err = bpf_map_get_info_by_fd(map_fd, &map_info, &map_info_len); 5411 if (err && errno == EINVAL) 5412 err = bpf_get_map_info_from_fdinfo(map_fd, &map_info); 5413 if (err) { 5414 pr_warn("failed to get map info for map FD %d: %s\n", map_fd, 5415 libbpf_strerror_r(errno, msg, sizeof(msg))); 5416 return false; 5417 } 5418 5419 return (map_info.type == map->def.type && 5420 map_info.key_size == map->def.key_size && 5421 map_info.value_size == map->def.value_size && 5422 map_info.max_entries == map->def.max_entries && 5423 map_info.map_flags == map->def.map_flags && 5424 map_info.map_extra == map->map_extra); 5425} 5426 5427static int 5428bpf_object__reuse_map(struct bpf_map *map) 5429{ 5430 char *cp, errmsg[STRERR_BUFSIZE]; 5431 int err, pin_fd; 5432 5433 pin_fd = bpf_obj_get(map->pin_path); 5434 if (pin_fd < 0) { 5435 err = -errno; 5436 if (err == -ENOENT) { 5437 pr_debug("found no pinned map to reuse at '%s'\n", 5438 map->pin_path); 5439 return 0; 5440 } 5441 5442 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 5443 pr_warn("couldn't retrieve pinned map '%s': %s\n", 5444 map->pin_path, cp); 5445 return err; 5446 } 5447 5448 if (!map_is_reuse_compat(map, pin_fd)) { 5449 pr_warn("couldn't reuse pinned map at '%s': parameter mismatch\n", 5450 map->pin_path); 5451 close(pin_fd); 5452 return -EINVAL; 5453 } 5454 5455 err = bpf_map__reuse_fd(map, pin_fd); 5456 close(pin_fd); 5457 if (err) 5458 return err; 5459 5460 map->pinned = true; 5461 pr_debug("reused pinned map at '%s'\n", map->pin_path); 5462 5463 return 0; 5464} 5465 5466static int 5467bpf_object__populate_internal_map(struct bpf_object *obj, struct bpf_map *map) 5468{ 5469 enum libbpf_map_type map_type = map->libbpf_type; 5470 char *cp, errmsg[STRERR_BUFSIZE]; 5471 int err, zero = 0; 5472 5473 if (obj->gen_loader) { 5474 bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps, 5475 map->mmaped, map->def.value_size); 5476 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) 5477 bpf_gen__map_freeze(obj->gen_loader, map - obj->maps); 5478 return 0; 5479 } 5480 err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0); 5481 if (err) { 5482 err = -errno; 5483 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 5484 pr_warn("Error setting initial map(%s) contents: %s\n", 5485 map->name, cp); 5486 return err; 5487 } 5488 5489 /* Freeze .rodata and .kconfig map as read-only from syscall side. */ 5490 if (map_type == LIBBPF_MAP_RODATA || map_type == LIBBPF_MAP_KCONFIG) { 5491 err = bpf_map_freeze(map->fd); 5492 if (err) { 5493 err = -errno; 5494 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 5495 pr_warn("Error freezing map(%s) as read-only: %s\n", 5496 map->name, cp); 5497 return err; 5498 } 5499 } 5500 return 0; 5501} 5502 5503static void bpf_map__destroy(struct bpf_map *map); 5504 5505static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, bool is_inner) 5506{ 5507 LIBBPF_OPTS(bpf_map_create_opts, create_attr); 5508 struct bpf_map_def *def = &map->def; 5509 const char *map_name = NULL; 5510 int err = 0; 5511 5512 if (kernel_supports(obj, FEAT_PROG_NAME)) 5513 map_name = map->name; 5514 create_attr.map_ifindex = map->map_ifindex; 5515 create_attr.map_flags = def->map_flags; 5516 create_attr.numa_node = map->numa_node; 5517 create_attr.map_extra = map->map_extra; 5518 5519 if (bpf_map__is_struct_ops(map)) 5520 create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id; 5521 5522 if (obj->btf && btf__fd(obj->btf) >= 0) { 5523 create_attr.btf_fd = btf__fd(obj->btf); 5524 create_attr.btf_key_type_id = map->btf_key_type_id; 5525 create_attr.btf_value_type_id = map->btf_value_type_id; 5526 } 5527 5528 if (bpf_map_type__is_map_in_map(def->type)) { 5529 if (map->inner_map) { 5530 err = bpf_object__create_map(obj, map->inner_map, true); 5531 if (err) { 5532 pr_warn("map '%s': failed to create inner map: %d\n", 5533 map->name, err); 5534 return err; 5535 } 5536 map->inner_map_fd = bpf_map__fd(map->inner_map); 5537 } 5538 if (map->inner_map_fd >= 0) 5539 create_attr.inner_map_fd = map->inner_map_fd; 5540 } 5541 5542 switch (def->type) { 5543 case BPF_MAP_TYPE_PERF_EVENT_ARRAY: 5544 case BPF_MAP_TYPE_CGROUP_ARRAY: 5545 case BPF_MAP_TYPE_STACK_TRACE: 5546 case BPF_MAP_TYPE_ARRAY_OF_MAPS: 5547 case BPF_MAP_TYPE_HASH_OF_MAPS: 5548 case BPF_MAP_TYPE_DEVMAP: 5549 case BPF_MAP_TYPE_DEVMAP_HASH: 5550 case BPF_MAP_TYPE_CPUMAP: 5551 case BPF_MAP_TYPE_XSKMAP: 5552 case BPF_MAP_TYPE_SOCKMAP: 5553 case BPF_MAP_TYPE_SOCKHASH: 5554 case BPF_MAP_TYPE_QUEUE: 5555 case BPF_MAP_TYPE_STACK: 5556 create_attr.btf_fd = 0; 5557 create_attr.btf_key_type_id = 0; 5558 create_attr.btf_value_type_id = 0; 5559 map->btf_key_type_id = 0; 5560 map->btf_value_type_id = 0; 5561 break; 5562 default: 5563 break; 5564 } 5565 5566 if (obj->gen_loader) { 5567 bpf_gen__map_create(obj->gen_loader, def->type, map_name, 5568 def->key_size, def->value_size, def->max_entries, 5569 &create_attr, is_inner ? -1 : map - obj->maps); 5570 /* Pretend to have valid FD to pass various fd >= 0 checks. 5571 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually. 5572 */ 5573 map->fd = 0; 5574 } else { 5575 map->fd = bpf_map_create(def->type, map_name, 5576 def->key_size, def->value_size, 5577 def->max_entries, &create_attr); 5578 } 5579 if (map->fd < 0 && (create_attr.btf_key_type_id || 5580 create_attr.btf_value_type_id)) { 5581 char *cp, errmsg[STRERR_BUFSIZE]; 5582 5583 err = -errno; 5584 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 5585 pr_warn("Error in bpf_create_map_xattr(%s):%s(%d). Retrying without BTF.\n", 5586 map->name, cp, err); 5587 create_attr.btf_fd = 0; 5588 create_attr.btf_key_type_id = 0; 5589 create_attr.btf_value_type_id = 0; 5590 map->btf_key_type_id = 0; 5591 map->btf_value_type_id = 0; 5592 map->fd = bpf_map_create(def->type, map_name, 5593 def->key_size, def->value_size, 5594 def->max_entries, &create_attr); 5595 } 5596 5597 err = map->fd < 0 ? -errno : 0; 5598 5599 if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) { 5600 if (obj->gen_loader) 5601 map->inner_map->fd = -1; 5602 bpf_map__destroy(map->inner_map); 5603 zfree(&map->inner_map); 5604 } 5605 5606 return err; 5607} 5608 5609static int init_map_in_map_slots(struct bpf_object *obj, struct bpf_map *map) 5610{ 5611 const struct bpf_map *targ_map; 5612 unsigned int i; 5613 int fd, err = 0; 5614 5615 for (i = 0; i < map->init_slots_sz; i++) { 5616 if (!map->init_slots[i]) 5617 continue; 5618 5619 targ_map = map->init_slots[i]; 5620 fd = bpf_map__fd(targ_map); 5621 5622 if (obj->gen_loader) { 5623 bpf_gen__populate_outer_map(obj->gen_loader, 5624 map - obj->maps, i, 5625 targ_map - obj->maps); 5626 } else { 5627 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 5628 } 5629 if (err) { 5630 err = -errno; 5631 pr_warn("map '%s': failed to initialize slot [%d] to map '%s' fd=%d: %d\n", 5632 map->name, i, targ_map->name, fd, err); 5633 return err; 5634 } 5635 pr_debug("map '%s': slot [%d] set to map '%s' fd=%d\n", 5636 map->name, i, targ_map->name, fd); 5637 } 5638 5639 zfree(&map->init_slots); 5640 map->init_slots_sz = 0; 5641 5642 return 0; 5643} 5644 5645static int init_prog_array_slots(struct bpf_object *obj, struct bpf_map *map) 5646{ 5647 const struct bpf_program *targ_prog; 5648 unsigned int i; 5649 int fd, err; 5650 5651 if (obj->gen_loader) 5652 return -ENOTSUP; 5653 5654 for (i = 0; i < map->init_slots_sz; i++) { 5655 if (!map->init_slots[i]) 5656 continue; 5657 5658 targ_prog = map->init_slots[i]; 5659 fd = bpf_program__fd(targ_prog); 5660 5661 err = bpf_map_update_elem(map->fd, &i, &fd, 0); 5662 if (err) { 5663 err = -errno; 5664 pr_warn("map '%s': failed to initialize slot [%d] to prog '%s' fd=%d: %d\n", 5665 map->name, i, targ_prog->name, fd, err); 5666 return err; 5667 } 5668 pr_debug("map '%s': slot [%d] set to prog '%s' fd=%d\n", 5669 map->name, i, targ_prog->name, fd); 5670 } 5671 5672 zfree(&map->init_slots); 5673 map->init_slots_sz = 0; 5674 5675 return 0; 5676} 5677 5678static int bpf_object_init_prog_arrays(struct bpf_object *obj) 5679{ 5680 struct bpf_map *map; 5681 int i, err; 5682 5683 for (i = 0; i < obj->nr_maps; i++) { 5684 map = &obj->maps[i]; 5685 5686 if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY) 5687 continue; 5688 5689 err = init_prog_array_slots(obj, map); 5690 if (err < 0) { 5691 zclose(map->fd); 5692 return err; 5693 } 5694 } 5695 return 0; 5696} 5697 5698static int map_set_def_max_entries(struct bpf_map *map) 5699{ 5700 if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) { 5701 int nr_cpus; 5702 5703 nr_cpus = libbpf_num_possible_cpus(); 5704 if (nr_cpus < 0) { 5705 pr_warn("map '%s': failed to determine number of system CPUs: %d\n", 5706 map->name, nr_cpus); 5707 return nr_cpus; 5708 } 5709 pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus); 5710 map->def.max_entries = nr_cpus; 5711 } 5712 5713 return 0; 5714} 5715 5716static int 5717bpf_object__create_maps(struct bpf_object *obj) 5718{ 5719 struct bpf_map *map; 5720 char *cp, errmsg[STRERR_BUFSIZE]; 5721 unsigned int i, j; 5722 int err; 5723 bool retried; 5724 5725 for (i = 0; i < obj->nr_maps; i++) { 5726 map = &obj->maps[i]; 5727 5728 /* To support old kernels, we skip creating global data maps 5729 * (.rodata, .data, .kconfig, etc); later on, during program 5730 * loading, if we detect that at least one of the to-be-loaded 5731 * programs is referencing any global data map, we'll error 5732 * out with program name and relocation index logged. 5733 * This approach allows to accommodate Clang emitting 5734 * unnecessary .rodata.str1.1 sections for string literals, 5735 * but also it allows to have CO-RE applications that use 5736 * global variables in some of BPF programs, but not others. 5737 * If those global variable-using programs are not loaded at 5738 * runtime due to bpf_program__set_autoload(prog, false), 5739 * bpf_object loading will succeed just fine even on old 5740 * kernels. 5741 */ 5742 if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA)) 5743 map->autocreate = false; 5744 5745 if (!map->autocreate) { 5746 pr_debug("map '%s': skipped auto-creating...\n", map->name); 5747 continue; 5748 } 5749 5750 err = map_set_def_max_entries(map); 5751 if (err) 5752 goto err_out; 5753 5754 retried = false; 5755retry: 5756 if (map->pin_path) { 5757 err = bpf_object__reuse_map(map); 5758 if (err) { 5759 pr_warn("map '%s': error reusing pinned map\n", 5760 map->name); 5761 goto err_out; 5762 } 5763 if (retried && map->fd < 0) { 5764 pr_warn("map '%s': cannot find pinned map\n", 5765 map->name); 5766 err = -ENOENT; 5767 goto err_out; 5768 } 5769 } 5770 5771 if (map->fd >= 0) { 5772 pr_debug("map '%s': skipping creation (preset fd=%d)\n", 5773 map->name, map->fd); 5774 } else { 5775 err = bpf_object__create_map(obj, map, false); 5776 if (err) 5777 goto err_out; 5778 5779 pr_debug("map '%s': created successfully, fd=%d\n", 5780 map->name, map->fd); 5781 5782 if (bpf_map__is_internal(map)) { 5783 err = bpf_object__populate_internal_map(obj, map); 5784 if (err < 0) { 5785 zclose(map->fd); 5786 goto err_out; 5787 } 5788 } 5789 5790 if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) { 5791 err = init_map_in_map_slots(obj, map); 5792 if (err < 0) { 5793 zclose(map->fd); 5794 goto err_out; 5795 } 5796 } 5797 } 5798 5799 if (map->pin_path && !map->pinned) { 5800 err = bpf_map__pin(map, NULL); 5801 if (err) { 5802 zclose(map->fd); 5803 if (!retried && err == -EEXIST) { 5804 retried = true; 5805 goto retry; 5806 } 5807 pr_warn("map '%s': failed to auto-pin at '%s': %d\n", 5808 map->name, map->pin_path, err); 5809 goto err_out; 5810 } 5811 } 5812 } 5813 5814 return 0; 5815 5816err_out: 5817 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 5818 pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err); 5819 pr_perm_msg(err); 5820 for (j = 0; j < i; j++) 5821 zclose(obj->maps[j].fd); 5822 return err; 5823} 5824 5825static bool bpf_core_is_flavor_sep(const char *s) 5826{ 5827 /* check X___Y name pattern, where X and Y are not underscores */ 5828 return s[0] != '_' && /* X */ 5829 s[1] == '_' && s[2] == '_' && s[3] == '_' && /* ___ */ 5830 s[4] != '_'; /* Y */ 5831} 5832 5833/* Given 'some_struct_name___with_flavor' return the length of a name prefix 5834 * before last triple underscore. Struct name part after last triple 5835 * underscore is ignored by BPF CO-RE relocation during relocation matching. 5836 */ 5837size_t bpf_core_essential_name_len(const char *name) 5838{ 5839 size_t n = strlen(name); 5840 int i; 5841 5842 for (i = n - 5; i >= 0; i--) { 5843 if (bpf_core_is_flavor_sep(name + i)) 5844 return i + 1; 5845 } 5846 return n; 5847} 5848 5849void bpf_core_free_cands(struct bpf_core_cand_list *cands) 5850{ 5851 if (!cands) 5852 return; 5853 5854 free(cands->cands); 5855 free(cands); 5856} 5857 5858int bpf_core_add_cands(struct bpf_core_cand *local_cand, 5859 size_t local_essent_len, 5860 const struct btf *targ_btf, 5861 const char *targ_btf_name, 5862 int targ_start_id, 5863 struct bpf_core_cand_list *cands) 5864{ 5865 struct bpf_core_cand *new_cands, *cand; 5866 const struct btf_type *t, *local_t; 5867 const char *targ_name, *local_name; 5868 size_t targ_essent_len; 5869 int n, i; 5870 5871 local_t = btf__type_by_id(local_cand->btf, local_cand->id); 5872 local_name = btf__str_by_offset(local_cand->btf, local_t->name_off); 5873 5874 n = btf__type_cnt(targ_btf); 5875 for (i = targ_start_id; i < n; i++) { 5876 t = btf__type_by_id(targ_btf, i); 5877 if (!btf_kind_core_compat(t, local_t)) 5878 continue; 5879 5880 targ_name = btf__name_by_offset(targ_btf, t->name_off); 5881 if (str_is_empty(targ_name)) 5882 continue; 5883 5884 targ_essent_len = bpf_core_essential_name_len(targ_name); 5885 if (targ_essent_len != local_essent_len) 5886 continue; 5887 5888 if (strncmp(local_name, targ_name, local_essent_len) != 0) 5889 continue; 5890 5891 pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n", 5892 local_cand->id, btf_kind_str(local_t), 5893 local_name, i, btf_kind_str(t), targ_name, 5894 targ_btf_name); 5895 new_cands = libbpf_reallocarray(cands->cands, cands->len + 1, 5896 sizeof(*cands->cands)); 5897 if (!new_cands) 5898 return -ENOMEM; 5899 5900 cand = &new_cands[cands->len]; 5901 cand->btf = targ_btf; 5902 cand->id = i; 5903 5904 cands->cands = new_cands; 5905 cands->len++; 5906 } 5907 return 0; 5908} 5909 5910static int load_module_btfs(struct bpf_object *obj) 5911{ 5912 struct bpf_btf_info info; 5913 struct module_btf *mod_btf; 5914 struct btf *btf; 5915 char name[64]; 5916 __u32 id = 0, len; 5917 int err, fd; 5918 5919 if (obj->btf_modules_loaded) 5920 return 0; 5921 5922 if (obj->gen_loader) 5923 return 0; 5924 5925 /* don't do this again, even if we find no module BTFs */ 5926 obj->btf_modules_loaded = true; 5927 5928 /* kernel too old to support module BTFs */ 5929 if (!kernel_supports(obj, FEAT_MODULE_BTF)) 5930 return 0; 5931 5932 while (true) { 5933 err = bpf_btf_get_next_id(id, &id); 5934 if (err && errno == ENOENT) 5935 return 0; 5936 if (err && errno == EPERM) { 5937 pr_debug("skipping module BTFs loading, missing privileges\n"); 5938 return 0; 5939 } 5940 if (err) { 5941 err = -errno; 5942 pr_warn("failed to iterate BTF objects: %d\n", err); 5943 return err; 5944 } 5945 5946 fd = bpf_btf_get_fd_by_id(id); 5947 if (fd < 0) { 5948 if (errno == ENOENT) 5949 continue; /* expected race: BTF was unloaded */ 5950 err = -errno; 5951 pr_warn("failed to get BTF object #%d FD: %d\n", id, err); 5952 return err; 5953 } 5954 5955 len = sizeof(info); 5956 memset(&info, 0, sizeof(info)); 5957 info.name = ptr_to_u64(name); 5958 info.name_len = sizeof(name); 5959 5960 err = bpf_btf_get_info_by_fd(fd, &info, &len); 5961 if (err) { 5962 err = -errno; 5963 pr_warn("failed to get BTF object #%d info: %d\n", id, err); 5964 goto err_out; 5965 } 5966 5967 /* ignore non-module BTFs */ 5968 if (!info.kernel_btf || strcmp(name, "vmlinux") == 0) { 5969 close(fd); 5970 continue; 5971 } 5972 5973 btf = btf_get_from_fd(fd, obj->btf_vmlinux); 5974 err = libbpf_get_error(btf); 5975 if (err) { 5976 pr_warn("failed to load module [%s]'s BTF object #%d: %d\n", 5977 name, id, err); 5978 goto err_out; 5979 } 5980 5981 err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap, 5982 sizeof(*obj->btf_modules), obj->btf_module_cnt + 1); 5983 if (err) 5984 goto err_out; 5985 5986 mod_btf = &obj->btf_modules[obj->btf_module_cnt++]; 5987 5988 mod_btf->btf = btf; 5989 mod_btf->id = id; 5990 mod_btf->fd = fd; 5991 mod_btf->name = strdup(name); 5992 if (!mod_btf->name) { 5993 err = -ENOMEM; 5994 goto err_out; 5995 } 5996 continue; 5997 5998err_out: 5999 close(fd); 6000 return err; 6001 } 6002 6003 return 0; 6004} 6005 6006static struct bpf_core_cand_list * 6007bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 local_type_id) 6008{ 6009 struct bpf_core_cand local_cand = {}; 6010 struct bpf_core_cand_list *cands; 6011 const struct btf *main_btf; 6012 const struct btf_type *local_t; 6013 const char *local_name; 6014 size_t local_essent_len; 6015 int err, i; 6016 6017 local_cand.btf = local_btf; 6018 local_cand.id = local_type_id; 6019 local_t = btf__type_by_id(local_btf, local_type_id); 6020 if (!local_t) 6021 return ERR_PTR(-EINVAL); 6022 6023 local_name = btf__name_by_offset(local_btf, local_t->name_off); 6024 if (str_is_empty(local_name)) 6025 return ERR_PTR(-EINVAL); 6026 local_essent_len = bpf_core_essential_name_len(local_name); 6027 6028 cands = calloc(1, sizeof(*cands)); 6029 if (!cands) 6030 return ERR_PTR(-ENOMEM); 6031 6032 /* Attempt to find target candidates in vmlinux BTF first */ 6033 main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux; 6034 err = bpf_core_add_cands(&local_cand, local_essent_len, main_btf, "vmlinux", 1, cands); 6035 if (err) 6036 goto err_out; 6037 6038 /* if vmlinux BTF has any candidate, don't got for module BTFs */ 6039 if (cands->len) 6040 return cands; 6041 6042 /* if vmlinux BTF was overridden, don't attempt to load module BTFs */ 6043 if (obj->btf_vmlinux_override) 6044 return cands; 6045 6046 /* now look through module BTFs, trying to still find candidates */ 6047 err = load_module_btfs(obj); 6048 if (err) 6049 goto err_out; 6050 6051 for (i = 0; i < obj->btf_module_cnt; i++) { 6052 err = bpf_core_add_cands(&local_cand, local_essent_len, 6053 obj->btf_modules[i].btf, 6054 obj->btf_modules[i].name, 6055 btf__type_cnt(obj->btf_vmlinux), 6056 cands); 6057 if (err) 6058 goto err_out; 6059 } 6060 6061 return cands; 6062err_out: 6063 bpf_core_free_cands(cands); 6064 return ERR_PTR(err); 6065} 6066 6067/* Check local and target types for compatibility. This check is used for 6068 * type-based CO-RE relocations and follow slightly different rules than 6069 * field-based relocations. This function assumes that root types were already 6070 * checked for name match. Beyond that initial root-level name check, names 6071 * are completely ignored. Compatibility rules are as follows: 6072 * - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but 6073 * kind should match for local and target types (i.e., STRUCT is not 6074 * compatible with UNION); 6075 * - for ENUMs, the size is ignored; 6076 * - for INT, size and signedness are ignored; 6077 * - for ARRAY, dimensionality is ignored, element types are checked for 6078 * compatibility recursively; 6079 * - CONST/VOLATILE/RESTRICT modifiers are ignored; 6080 * - TYPEDEFs/PTRs are compatible if types they pointing to are compatible; 6081 * - FUNC_PROTOs are compatible if they have compatible signature: same 6082 * number of input args and compatible return and argument types. 6083 * These rules are not set in stone and probably will be adjusted as we get 6084 * more experience with using BPF CO-RE relocations. 6085 */ 6086int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, 6087 const struct btf *targ_btf, __u32 targ_id) 6088{ 6089 return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32); 6090} 6091 6092int bpf_core_types_match(const struct btf *local_btf, __u32 local_id, 6093 const struct btf *targ_btf, __u32 targ_id) 6094{ 6095 return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32); 6096} 6097 6098static size_t bpf_core_hash_fn(const long key, void *ctx) 6099{ 6100 return key; 6101} 6102 6103static bool bpf_core_equal_fn(const long k1, const long k2, void *ctx) 6104{ 6105 return k1 == k2; 6106} 6107 6108static int record_relo_core(struct bpf_program *prog, 6109 const struct bpf_core_relo *core_relo, int insn_idx) 6110{ 6111 struct reloc_desc *relos, *relo; 6112 6113 relos = libbpf_reallocarray(prog->reloc_desc, 6114 prog->nr_reloc + 1, sizeof(*relos)); 6115 if (!relos) 6116 return -ENOMEM; 6117 relo = &relos[prog->nr_reloc]; 6118 relo->type = RELO_CORE; 6119 relo->insn_idx = insn_idx; 6120 relo->core_relo = core_relo; 6121 prog->reloc_desc = relos; 6122 prog->nr_reloc++; 6123 return 0; 6124} 6125 6126static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx) 6127{ 6128 struct reloc_desc *relo; 6129 int i; 6130 6131 for (i = 0; i < prog->nr_reloc; i++) { 6132 relo = &prog->reloc_desc[i]; 6133 if (relo->type != RELO_CORE || relo->insn_idx != insn_idx) 6134 continue; 6135 6136 return relo->core_relo; 6137 } 6138 6139 return NULL; 6140} 6141 6142static int bpf_core_resolve_relo(struct bpf_program *prog, 6143 const struct bpf_core_relo *relo, 6144 int relo_idx, 6145 const struct btf *local_btf, 6146 struct hashmap *cand_cache, 6147 struct bpf_core_relo_res *targ_res) 6148{ 6149 struct bpf_core_spec specs_scratch[3] = {}; 6150 struct bpf_core_cand_list *cands = NULL; 6151 const char *prog_name = prog->name; 6152 const struct btf_type *local_type; 6153 const char *local_name; 6154 __u32 local_id = relo->type_id; 6155 int err; 6156 6157 local_type = btf__type_by_id(local_btf, local_id); 6158 if (!local_type) 6159 return -EINVAL; 6160 6161 local_name = btf__name_by_offset(local_btf, local_type->name_off); 6162 if (!local_name) 6163 return -EINVAL; 6164 6165 if (relo->kind != BPF_CORE_TYPE_ID_LOCAL && 6166 !hashmap__find(cand_cache, local_id, &cands)) { 6167 cands = bpf_core_find_cands(prog->obj, local_btf, local_id); 6168 if (IS_ERR(cands)) { 6169 pr_warn("prog '%s': relo #%d: target candidate search failed for [%d] %s %s: %ld\n", 6170 prog_name, relo_idx, local_id, btf_kind_str(local_type), 6171 local_name, PTR_ERR(cands)); 6172 return PTR_ERR(cands); 6173 } 6174 err = hashmap__set(cand_cache, local_id, cands, NULL, NULL); 6175 if (err) { 6176 bpf_core_free_cands(cands); 6177 return err; 6178 } 6179 } 6180 6181 return bpf_core_calc_relo_insn(prog_name, relo, relo_idx, local_btf, cands, specs_scratch, 6182 targ_res); 6183} 6184 6185static int 6186bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) 6187{ 6188 const struct btf_ext_info_sec *sec; 6189 struct bpf_core_relo_res targ_res; 6190 const struct bpf_core_relo *rec; 6191 const struct btf_ext_info *seg; 6192 struct hashmap_entry *entry; 6193 struct hashmap *cand_cache = NULL; 6194 struct bpf_program *prog; 6195 struct bpf_insn *insn; 6196 const char *sec_name; 6197 int i, err = 0, insn_idx, sec_idx, sec_num; 6198 6199 if (obj->btf_ext->core_relo_info.len == 0) 6200 return 0; 6201 6202 if (targ_btf_path) { 6203#ifdef HAVE_LIBELF 6204 obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL); 6205#endif 6206 err = libbpf_get_error(obj->btf_vmlinux_override); 6207 if (err) { 6208 pr_warn("failed to parse target BTF: %d\n", err); 6209 return err; 6210 } 6211 } 6212 6213 cand_cache = hashmap__new(bpf_core_hash_fn, bpf_core_equal_fn, NULL); 6214 if (IS_ERR(cand_cache)) { 6215 err = PTR_ERR(cand_cache); 6216 goto out; 6217 } 6218 6219 seg = &obj->btf_ext->core_relo_info; 6220 sec_num = 0; 6221 for_each_btf_ext_sec(seg, sec) { 6222 sec_idx = seg->sec_idxs[sec_num]; 6223 sec_num++; 6224 6225 sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); 6226 if (str_is_empty(sec_name)) { 6227 err = -EINVAL; 6228 goto out; 6229 } 6230 6231 pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); 6232 6233 for_each_btf_ext_rec(seg, sec, i, rec) { 6234 if (rec->insn_off % BPF_INSN_SZ) 6235 return -EINVAL; 6236 insn_idx = rec->insn_off / BPF_INSN_SZ; 6237 prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); 6238 if (!prog) { 6239 /* When __weak subprog is "overridden" by another instance 6240 * of the subprog from a different object file, linker still 6241 * appends all the .BTF.ext info that used to belong to that 6242 * eliminated subprogram. 6243 * This is similar to what x86-64 linker does for relocations. 6244 * So just ignore such relocations just like we ignore 6245 * subprog instructions when discovering subprograms. 6246 */ 6247 pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n", 6248 sec_name, i, insn_idx); 6249 continue; 6250 } 6251 /* no need to apply CO-RE relocation if the program is 6252 * not going to be loaded 6253 */ 6254 if (!prog->autoload) 6255 continue; 6256 6257 /* adjust insn_idx from section frame of reference to the local 6258 * program's frame of reference; (sub-)program code is not yet 6259 * relocated, so it's enough to just subtract in-section offset 6260 */ 6261 insn_idx = insn_idx - prog->sec_insn_off; 6262 if (insn_idx >= prog->insns_cnt) 6263 return -EINVAL; 6264 insn = &prog->insns[insn_idx]; 6265 6266 err = record_relo_core(prog, rec, insn_idx); 6267 if (err) { 6268 pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n", 6269 prog->name, i, err); 6270 goto out; 6271 } 6272 6273 if (prog->obj->gen_loader) 6274 continue; 6275 6276 err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); 6277 if (err) { 6278 pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", 6279 prog->name, i, err); 6280 goto out; 6281 } 6282 6283 err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res); 6284 if (err) { 6285 pr_warn("prog '%s': relo #%d: failed to patch insn #%u: %d\n", 6286 prog->name, i, insn_idx, err); 6287 goto out; 6288 } 6289 } 6290 } 6291 6292out: 6293 /* obj->btf_vmlinux and module BTFs are freed after object load */ 6294 btf__free(obj->btf_vmlinux_override); 6295 obj->btf_vmlinux_override = NULL; 6296 6297 if (!IS_ERR_OR_NULL(cand_cache)) { 6298 hashmap__for_each_entry(cand_cache, entry, i) { 6299 bpf_core_free_cands(entry->pvalue); 6300 } 6301 hashmap__free(cand_cache); 6302 } 6303 return err; 6304} 6305 6306/* base map load ldimm64 special constant, used also for log fixup logic */ 6307#define POISON_LDIMM64_MAP_BASE 2001000000 6308#define POISON_LDIMM64_MAP_PFX "200100" 6309 6310static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx, 6311 int insn_idx, struct bpf_insn *insn, 6312 int map_idx, const struct bpf_map *map) 6313{ 6314 int i; 6315 6316 pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n", 6317 prog->name, relo_idx, insn_idx, map_idx, map->name); 6318 6319 /* we turn single ldimm64 into two identical invalid calls */ 6320 for (i = 0; i < 2; i++) { 6321 insn->code = BPF_JMP | BPF_CALL; 6322 insn->dst_reg = 0; 6323 insn->src_reg = 0; 6324 insn->off = 0; 6325 /* if this instruction is reachable (not a dead code), 6326 * verifier will complain with something like: 6327 * invalid func unknown#2001000123 6328 * where lower 123 is map index into obj->maps[] array 6329 */ 6330 insn->imm = POISON_LDIMM64_MAP_BASE + map_idx; 6331 6332 insn++; 6333 } 6334} 6335 6336/* unresolved kfunc call special constant, used also for log fixup logic */ 6337#define POISON_CALL_KFUNC_BASE 2002000000 6338#define POISON_CALL_KFUNC_PFX "2002" 6339 6340static void poison_kfunc_call(struct bpf_program *prog, int relo_idx, 6341 int insn_idx, struct bpf_insn *insn, 6342 int ext_idx, const struct extern_desc *ext) 6343{ 6344 pr_debug("prog '%s': relo #%d: poisoning insn #%d that calls kfunc '%s'\n", 6345 prog->name, relo_idx, insn_idx, ext->name); 6346 6347 /* we turn kfunc call into invalid helper call with identifiable constant */ 6348 insn->code = BPF_JMP | BPF_CALL; 6349 insn->dst_reg = 0; 6350 insn->src_reg = 0; 6351 insn->off = 0; 6352 /* if this instruction is reachable (not a dead code), 6353 * verifier will complain with something like: 6354 * invalid func unknown#2001000123 6355 * where lower 123 is extern index into obj->externs[] array 6356 */ 6357 insn->imm = POISON_CALL_KFUNC_BASE + ext_idx; 6358} 6359 6360/* Relocate data references within program code: 6361 * - map references; 6362 * - global variable references; 6363 * - extern references. 6364 */ 6365static int 6366bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) 6367{ 6368 int i; 6369 6370 for (i = 0; i < prog->nr_reloc; i++) { 6371 struct reloc_desc *relo = &prog->reloc_desc[i]; 6372 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 6373 const struct bpf_map *map; 6374 struct extern_desc *ext; 6375 6376 switch (relo->type) { 6377 case RELO_LD64: 6378 map = &obj->maps[relo->map_idx]; 6379 if (obj->gen_loader) { 6380 insn[0].src_reg = BPF_PSEUDO_MAP_IDX; 6381 insn[0].imm = relo->map_idx; 6382 } else if (map->autocreate) { 6383 insn[0].src_reg = BPF_PSEUDO_MAP_FD; 6384 insn[0].imm = map->fd; 6385 } else { 6386 poison_map_ldimm64(prog, i, relo->insn_idx, insn, 6387 relo->map_idx, map); 6388 } 6389 break; 6390 case RELO_DATA: 6391 map = &obj->maps[relo->map_idx]; 6392 insn[1].imm = insn[0].imm + relo->sym_off; 6393 if (obj->gen_loader) { 6394 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 6395 insn[0].imm = relo->map_idx; 6396 } else if (map->autocreate) { 6397 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 6398 insn[0].imm = map->fd; 6399 } else { 6400 poison_map_ldimm64(prog, i, relo->insn_idx, insn, 6401 relo->map_idx, map); 6402 } 6403 break; 6404 case RELO_EXTERN_LD64: 6405 ext = &obj->externs[relo->ext_idx]; 6406 if (ext->type == EXT_KCFG) { 6407 if (obj->gen_loader) { 6408 insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; 6409 insn[0].imm = obj->kconfig_map_idx; 6410 } else { 6411 insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; 6412 insn[0].imm = obj->maps[obj->kconfig_map_idx].fd; 6413 } 6414 insn[1].imm = ext->kcfg.data_off; 6415 } else /* EXT_KSYM */ { 6416 if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */ 6417 insn[0].src_reg = BPF_PSEUDO_BTF_ID; 6418 insn[0].imm = ext->ksym.kernel_btf_id; 6419 insn[1].imm = ext->ksym.kernel_btf_obj_fd; 6420 } else { /* typeless ksyms or unresolved typed ksyms */ 6421 insn[0].imm = (__u32)ext->ksym.addr; 6422 insn[1].imm = ext->ksym.addr >> 32; 6423 } 6424 } 6425 break; 6426 case RELO_EXTERN_CALL: 6427 ext = &obj->externs[relo->ext_idx]; 6428 insn[0].src_reg = BPF_PSEUDO_KFUNC_CALL; 6429 if (ext->is_set) { 6430 insn[0].imm = ext->ksym.kernel_btf_id; 6431 insn[0].off = ext->ksym.btf_fd_idx; 6432 } else { /* unresolved weak kfunc call */ 6433 poison_kfunc_call(prog, i, relo->insn_idx, insn, 6434 relo->ext_idx, ext); 6435 } 6436 break; 6437 case RELO_SUBPROG_ADDR: 6438 if (insn[0].src_reg != BPF_PSEUDO_FUNC) { 6439 pr_warn("prog '%s': relo #%d: bad insn\n", 6440 prog->name, i); 6441 return -EINVAL; 6442 } 6443 /* handled already */ 6444 break; 6445 case RELO_CALL: 6446 /* handled already */ 6447 break; 6448 case RELO_CORE: 6449 /* will be handled by bpf_program_record_relos() */ 6450 break; 6451 default: 6452 pr_warn("prog '%s': relo #%d: bad relo type %d\n", 6453 prog->name, i, relo->type); 6454 return -EINVAL; 6455 } 6456 } 6457 6458 return 0; 6459} 6460 6461static int adjust_prog_btf_ext_info(const struct bpf_object *obj, 6462 const struct bpf_program *prog, 6463 const struct btf_ext_info *ext_info, 6464 void **prog_info, __u32 *prog_rec_cnt, 6465 __u32 *prog_rec_sz) 6466{ 6467 void *copy_start = NULL, *copy_end = NULL; 6468 void *rec, *rec_end, *new_prog_info; 6469 const struct btf_ext_info_sec *sec; 6470 size_t old_sz, new_sz; 6471 int i, sec_num, sec_idx, off_adj; 6472 6473 sec_num = 0; 6474 for_each_btf_ext_sec(ext_info, sec) { 6475 sec_idx = ext_info->sec_idxs[sec_num]; 6476 sec_num++; 6477 if (prog->sec_idx != sec_idx) 6478 continue; 6479 6480 for_each_btf_ext_rec(ext_info, sec, i, rec) { 6481 __u32 insn_off = *(__u32 *)rec / BPF_INSN_SZ; 6482 6483 if (insn_off < prog->sec_insn_off) 6484 continue; 6485 if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt) 6486 break; 6487 6488 if (!copy_start) 6489 copy_start = rec; 6490 copy_end = rec + ext_info->rec_size; 6491 } 6492 6493 if (!copy_start) 6494 return -ENOENT; 6495 6496 /* append func/line info of a given (sub-)program to the main 6497 * program func/line info 6498 */ 6499 old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size; 6500 new_sz = old_sz + (copy_end - copy_start); 6501 new_prog_info = realloc(*prog_info, new_sz); 6502 if (!new_prog_info) 6503 return -ENOMEM; 6504 *prog_info = new_prog_info; 6505 *prog_rec_cnt = new_sz / ext_info->rec_size; 6506 memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start); 6507 6508 /* Kernel instruction offsets are in units of 8-byte 6509 * instructions, while .BTF.ext instruction offsets generated 6510 * by Clang are in units of bytes. So convert Clang offsets 6511 * into kernel offsets and adjust offset according to program 6512 * relocated position. 6513 */ 6514 off_adj = prog->sub_insn_off - prog->sec_insn_off; 6515 rec = new_prog_info + old_sz; 6516 rec_end = new_prog_info + new_sz; 6517 for (; rec < rec_end; rec += ext_info->rec_size) { 6518 __u32 *insn_off = rec; 6519 6520 *insn_off = *insn_off / BPF_INSN_SZ + off_adj; 6521 } 6522 *prog_rec_sz = ext_info->rec_size; 6523 return 0; 6524 } 6525 6526 return -ENOENT; 6527} 6528 6529static int 6530reloc_prog_func_and_line_info(const struct bpf_object *obj, 6531 struct bpf_program *main_prog, 6532 const struct bpf_program *prog) 6533{ 6534 int err; 6535 6536 /* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't 6537 * supprot func/line info 6538 */ 6539 if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC)) 6540 return 0; 6541 6542 /* only attempt func info relocation if main program's func_info 6543 * relocation was successful 6544 */ 6545 if (main_prog != prog && !main_prog->func_info) 6546 goto line_info; 6547 6548 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info, 6549 &main_prog->func_info, 6550 &main_prog->func_info_cnt, 6551 &main_prog->func_info_rec_size); 6552 if (err) { 6553 if (err != -ENOENT) { 6554 pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n", 6555 prog->name, err); 6556 return err; 6557 } 6558 if (main_prog->func_info) { 6559 /* 6560 * Some info has already been found but has problem 6561 * in the last btf_ext reloc. Must have to error out. 6562 */ 6563 pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name); 6564 return err; 6565 } 6566 /* Have problem loading the very first info. Ignore the rest. */ 6567 pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext func info.\n", 6568 prog->name); 6569 } 6570 6571line_info: 6572 /* don't relocate line info if main program's relocation failed */ 6573 if (main_prog != prog && !main_prog->line_info) 6574 return 0; 6575 6576 err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info, 6577 &main_prog->line_info, 6578 &main_prog->line_info_cnt, 6579 &main_prog->line_info_rec_size); 6580 if (err) { 6581 if (err != -ENOENT) { 6582 pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n", 6583 prog->name, err); 6584 return err; 6585 } 6586 if (main_prog->line_info) { 6587 /* 6588 * Some info has already been found but has problem 6589 * in the last btf_ext reloc. Must have to error out. 6590 */ 6591 pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name); 6592 return err; 6593 } 6594 /* Have problem loading the very first info. Ignore the rest. */ 6595 pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line info.\n", 6596 prog->name); 6597 } 6598 return 0; 6599} 6600 6601static int cmp_relo_by_insn_idx(const void *key, const void *elem) 6602{ 6603 size_t insn_idx = *(const size_t *)key; 6604 const struct reloc_desc *relo = elem; 6605 6606 if (insn_idx == relo->insn_idx) 6607 return 0; 6608 return insn_idx < relo->insn_idx ? -1 : 1; 6609} 6610 6611static struct reloc_desc *find_prog_insn_relo(const struct bpf_program *prog, size_t insn_idx) 6612{ 6613 if (!prog->nr_reloc) 6614 return NULL; 6615 return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc, 6616 sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx); 6617} 6618 6619static int append_subprog_relos(struct bpf_program *main_prog, struct bpf_program *subprog) 6620{ 6621 int new_cnt = main_prog->nr_reloc + subprog->nr_reloc; 6622 struct reloc_desc *relos; 6623 int i; 6624 6625 if (main_prog == subprog) 6626 return 0; 6627 relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos)); 6628 /* if new count is zero, reallocarray can return a valid NULL result; 6629 * in this case the previous pointer will be freed, so we *have to* 6630 * reassign old pointer to the new value (even if it's NULL) 6631 */ 6632 if (!relos && new_cnt) 6633 return -ENOMEM; 6634 if (subprog->nr_reloc) 6635 memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc, 6636 sizeof(*relos) * subprog->nr_reloc); 6637 6638 for (i = main_prog->nr_reloc; i < new_cnt; i++) 6639 relos[i].insn_idx += subprog->sub_insn_off; 6640 /* After insn_idx adjustment the 'relos' array is still sorted 6641 * by insn_idx and doesn't break bsearch. 6642 */ 6643 main_prog->reloc_desc = relos; 6644 main_prog->nr_reloc = new_cnt; 6645 return 0; 6646} 6647 6648static int 6649bpf_object__append_subprog_code(struct bpf_object *obj, struct bpf_program *main_prog, 6650 struct bpf_program *subprog) 6651{ 6652 struct bpf_insn *insns; 6653 size_t new_cnt; 6654 int err; 6655 6656 subprog->sub_insn_off = main_prog->insns_cnt; 6657 6658 new_cnt = main_prog->insns_cnt + subprog->insns_cnt; 6659 insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns)); 6660 if (!insns) { 6661 pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name); 6662 return -ENOMEM; 6663 } 6664 main_prog->insns = insns; 6665 main_prog->insns_cnt = new_cnt; 6666 6667 memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns, 6668 subprog->insns_cnt * sizeof(*insns)); 6669 6670 pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n", 6671 main_prog->name, subprog->insns_cnt, subprog->name); 6672 6673 /* The subprog insns are now appended. Append its relos too. */ 6674 err = append_subprog_relos(main_prog, subprog); 6675 if (err) 6676 return err; 6677 return 0; 6678} 6679 6680static int 6681bpf_object__reloc_code(struct bpf_object *obj, struct bpf_program *main_prog, 6682 struct bpf_program *prog) 6683{ 6684 size_t sub_insn_idx, insn_idx; 6685 struct bpf_program *subprog; 6686 struct reloc_desc *relo; 6687 struct bpf_insn *insn; 6688 int err; 6689 6690 err = reloc_prog_func_and_line_info(obj, main_prog, prog); 6691 if (err) 6692 return err; 6693 6694 for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) { 6695 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6696 if (!insn_is_subprog_call(insn) && !insn_is_pseudo_func(insn)) 6697 continue; 6698 6699 relo = find_prog_insn_relo(prog, insn_idx); 6700 if (relo && relo->type == RELO_EXTERN_CALL) 6701 /* kfunc relocations will be handled later 6702 * in bpf_object__relocate_data() 6703 */ 6704 continue; 6705 if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) { 6706 pr_warn("prog '%s': unexpected relo for insn #%zu, type %d\n", 6707 prog->name, insn_idx, relo->type); 6708 return -LIBBPF_ERRNO__RELOC; 6709 } 6710 if (relo) { 6711 /* sub-program instruction index is a combination of 6712 * an offset of a symbol pointed to by relocation and 6713 * call instruction's imm field; for global functions, 6714 * call always has imm = -1, but for static functions 6715 * relocation is against STT_SECTION and insn->imm 6716 * points to a start of a static function 6717 * 6718 * for subprog addr relocation, the relo->sym_off + insn->imm is 6719 * the byte offset in the corresponding section. 6720 */ 6721 if (relo->type == RELO_CALL) 6722 sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1; 6723 else 6724 sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ; 6725 } else if (insn_is_pseudo_func(insn)) { 6726 /* 6727 * RELO_SUBPROG_ADDR relo is always emitted even if both 6728 * functions are in the same section, so it shouldn't reach here. 6729 */ 6730 pr_warn("prog '%s': missing subprog addr relo for insn #%zu\n", 6731 prog->name, insn_idx); 6732 return -LIBBPF_ERRNO__RELOC; 6733 } else { 6734 /* if subprogram call is to a static function within 6735 * the same ELF section, there won't be any relocation 6736 * emitted, but it also means there is no additional 6737 * offset necessary, insns->imm is relative to 6738 * instruction's original position within the section 6739 */ 6740 sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1; 6741 } 6742 6743 /* we enforce that sub-programs should be in .text section */ 6744 subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx); 6745 if (!subprog) { 6746 pr_warn("prog '%s': no .text section found yet sub-program call exists\n", 6747 prog->name); 6748 return -LIBBPF_ERRNO__RELOC; 6749 } 6750 6751 /* if it's the first call instruction calling into this 6752 * subprogram (meaning this subprog hasn't been processed 6753 * yet) within the context of current main program: 6754 * - append it at the end of main program's instructions blog; 6755 * - process is recursively, while current program is put on hold; 6756 * - if that subprogram calls some other not yet processes 6757 * subprogram, same thing will happen recursively until 6758 * there are no more unprocesses subprograms left to append 6759 * and relocate. 6760 */ 6761 if (subprog->sub_insn_off == 0) { 6762 err = bpf_object__append_subprog_code(obj, main_prog, subprog); 6763 if (err) 6764 return err; 6765 err = bpf_object__reloc_code(obj, main_prog, subprog); 6766 if (err) 6767 return err; 6768 } 6769 6770 /* main_prog->insns memory could have been re-allocated, so 6771 * calculate pointer again 6772 */ 6773 insn = &main_prog->insns[prog->sub_insn_off + insn_idx]; 6774 /* calculate correct instruction position within current main 6775 * prog; each main prog can have a different set of 6776 * subprograms appended (potentially in different order as 6777 * well), so position of any subprog can be different for 6778 * different main programs 6779 */ 6780 insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1; 6781 6782 pr_debug("prog '%s': insn #%zu relocated, imm %d points to subprog '%s' (now at %zu offset)\n", 6783 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off); 6784 } 6785 6786 return 0; 6787} 6788 6789/* 6790 * Relocate sub-program calls. 6791 * 6792 * Algorithm operates as follows. Each entry-point BPF program (referred to as 6793 * main prog) is processed separately. For each subprog (non-entry functions, 6794 * that can be called from either entry progs or other subprogs) gets their 6795 * sub_insn_off reset to zero. This serves as indicator that this subprogram 6796 * hasn't been yet appended and relocated within current main prog. Once its 6797 * relocated, sub_insn_off will point at the position within current main prog 6798 * where given subprog was appended. This will further be used to relocate all 6799 * the call instructions jumping into this subprog. 6800 * 6801 * We start with main program and process all call instructions. If the call 6802 * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off 6803 * is zero), subprog instructions are appended at the end of main program's 6804 * instruction array. Then main program is "put on hold" while we recursively 6805 * process newly appended subprogram. If that subprogram calls into another 6806 * subprogram that hasn't been appended, new subprogram is appended again to 6807 * the *main* prog's instructions (subprog's instructions are always left 6808 * untouched, as they need to be in unmodified state for subsequent main progs 6809 * and subprog instructions are always sent only as part of a main prog) and 6810 * the process continues recursively. Once all the subprogs called from a main 6811 * prog or any of its subprogs are appended (and relocated), all their 6812 * positions within finalized instructions array are known, so it's easy to 6813 * rewrite call instructions with correct relative offsets, corresponding to 6814 * desired target subprog. 6815 * 6816 * Its important to realize that some subprogs might not be called from some 6817 * main prog and any of its called/used subprogs. Those will keep their 6818 * subprog->sub_insn_off as zero at all times and won't be appended to current 6819 * main prog and won't be relocated within the context of current main prog. 6820 * They might still be used from other main progs later. 6821 * 6822 * Visually this process can be shown as below. Suppose we have two main 6823 * programs mainA and mainB and BPF object contains three subprogs: subA, 6824 * subB, and subC. mainA calls only subA, mainB calls only subC, but subA and 6825 * subC both call subB: 6826 * 6827 * +--------+ +-------+ 6828 * | v v | 6829 * +--+---+ +--+-+-+ +---+--+ 6830 * | subA | | subB | | subC | 6831 * +--+---+ +------+ +---+--+ 6832 * ^ ^ 6833 * | | 6834 * +---+-------+ +------+----+ 6835 * | mainA | | mainB | 6836 * +-----------+ +-----------+ 6837 * 6838 * We'll start relocating mainA, will find subA, append it and start 6839 * processing sub A recursively: 6840 * 6841 * +-----------+------+ 6842 * | mainA | subA | 6843 * +-----------+------+ 6844 * 6845 * At this point we notice that subB is used from subA, so we append it and 6846 * relocate (there are no further subcalls from subB): 6847 * 6848 * +-----------+------+------+ 6849 * | mainA | subA | subB | 6850 * +-----------+------+------+ 6851 * 6852 * At this point, we relocate subA calls, then go one level up and finish with 6853 * relocatin mainA calls. mainA is done. 6854 * 6855 * For mainB process is similar but results in different order. We start with 6856 * mainB and skip subA and subB, as mainB never calls them (at least 6857 * directly), but we see subC is needed, so we append and start processing it: 6858 * 6859 * +-----------+------+ 6860 * | mainB | subC | 6861 * +-----------+------+ 6862 * Now we see subC needs subB, so we go back to it, append and relocate it: 6863 * 6864 * +-----------+------+------+ 6865 * | mainB | subC | subB | 6866 * +-----------+------+------+ 6867 * 6868 * At this point we unwind recursion, relocate calls in subC, then in mainB. 6869 */ 6870static int 6871bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) 6872{ 6873 struct bpf_program *subprog; 6874 int i, err; 6875 6876 /* mark all subprogs as not relocated (yet) within the context of 6877 * current main program 6878 */ 6879 for (i = 0; i < obj->nr_programs; i++) { 6880 subprog = &obj->programs[i]; 6881 if (!prog_is_subprog(obj, subprog)) 6882 continue; 6883 6884 subprog->sub_insn_off = 0; 6885 } 6886 6887 err = bpf_object__reloc_code(obj, prog, prog); 6888 if (err) 6889 return err; 6890 6891 return 0; 6892} 6893 6894static void 6895bpf_object__free_relocs(struct bpf_object *obj) 6896{ 6897 struct bpf_program *prog; 6898 int i; 6899 6900 /* free up relocation descriptors */ 6901 for (i = 0; i < obj->nr_programs; i++) { 6902 prog = &obj->programs[i]; 6903 zfree(&prog->reloc_desc); 6904 prog->nr_reloc = 0; 6905 } 6906} 6907 6908static int cmp_relocs(const void *_a, const void *_b) 6909{ 6910 const struct reloc_desc *a = _a; 6911 const struct reloc_desc *b = _b; 6912 6913 if (a->insn_idx != b->insn_idx) 6914 return a->insn_idx < b->insn_idx ? -1 : 1; 6915 6916 /* no two relocations should have the same insn_idx, but ... */ 6917 if (a->type != b->type) 6918 return a->type < b->type ? -1 : 1; 6919 6920 return 0; 6921} 6922 6923static void bpf_object__sort_relos(struct bpf_object *obj) 6924{ 6925 int i; 6926 6927 for (i = 0; i < obj->nr_programs; i++) { 6928 struct bpf_program *p = &obj->programs[i]; 6929 6930 if (!p->nr_reloc) 6931 continue; 6932 6933 qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs); 6934 } 6935} 6936 6937static int 6938bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) 6939{ 6940 struct bpf_program *prog; 6941 size_t i, j; 6942 int err; 6943 6944 if (obj->btf_ext) { 6945 err = bpf_object__relocate_core(obj, targ_btf_path); 6946 if (err) { 6947 pr_warn("failed to perform CO-RE relocations: %d\n", 6948 err); 6949 return err; 6950 } 6951 bpf_object__sort_relos(obj); 6952 } 6953 6954 /* Before relocating calls pre-process relocations and mark 6955 * few ld_imm64 instructions that points to subprogs. 6956 * Otherwise bpf_object__reloc_code() later would have to consider 6957 * all ld_imm64 insns as relocation candidates. That would 6958 * reduce relocation speed, since amount of find_prog_insn_relo() 6959 * would increase and most of them will fail to find a relo. 6960 */ 6961 for (i = 0; i < obj->nr_programs; i++) { 6962 prog = &obj->programs[i]; 6963 for (j = 0; j < prog->nr_reloc; j++) { 6964 struct reloc_desc *relo = &prog->reloc_desc[j]; 6965 struct bpf_insn *insn = &prog->insns[relo->insn_idx]; 6966 6967 /* mark the insn, so it's recognized by insn_is_pseudo_func() */ 6968 if (relo->type == RELO_SUBPROG_ADDR) 6969 insn[0].src_reg = BPF_PSEUDO_FUNC; 6970 } 6971 } 6972 6973 /* relocate subprogram calls and append used subprograms to main 6974 * programs; each copy of subprogram code needs to be relocated 6975 * differently for each main program, because its code location might 6976 * have changed. 6977 * Append subprog relos to main programs to allow data relos to be 6978 * processed after text is completely relocated. 6979 */ 6980 for (i = 0; i < obj->nr_programs; i++) { 6981 prog = &obj->programs[i]; 6982 /* sub-program's sub-calls are relocated within the context of 6983 * its main program only 6984 */ 6985 if (prog_is_subprog(obj, prog)) 6986 continue; 6987 if (!prog->autoload) 6988 continue; 6989 6990 err = bpf_object__relocate_calls(obj, prog); 6991 if (err) { 6992 pr_warn("prog '%s': failed to relocate calls: %d\n", 6993 prog->name, err); 6994 return err; 6995 } 6996 6997 /* Now, also append exception callback if it has not been done already. */ 6998 if (prog->exception_cb_idx >= 0) { 6999 struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx]; 7000 7001 /* Calling exception callback directly is disallowed, which the 7002 * verifier will reject later. In case it was processed already, 7003 * we can skip this step, otherwise for all other valid cases we 7004 * have to append exception callback now. 7005 */ 7006 if (subprog->sub_insn_off == 0) { 7007 err = bpf_object__append_subprog_code(obj, prog, subprog); 7008 if (err) 7009 return err; 7010 err = bpf_object__reloc_code(obj, prog, subprog); 7011 if (err) 7012 return err; 7013 } 7014 } 7015 } 7016 /* Process data relos for main programs */ 7017 for (i = 0; i < obj->nr_programs; i++) { 7018 prog = &obj->programs[i]; 7019 if (prog_is_subprog(obj, prog)) 7020 continue; 7021 if (!prog->autoload) 7022 continue; 7023 err = bpf_object__relocate_data(obj, prog); 7024 if (err) { 7025 pr_warn("prog '%s': failed to relocate data references: %d\n", 7026 prog->name, err); 7027 return err; 7028 } 7029 } 7030 7031 return 0; 7032} 7033 7034static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 7035 Elf64_Shdr *shdr, Elf_Data *data); 7036 7037static int bpf_object__collect_map_relos(struct bpf_object *obj, 7038 Elf64_Shdr *shdr, Elf_Data *data) 7039{ 7040 const int bpf_ptr_sz = 8, host_ptr_sz = sizeof(void *); 7041 int i, j, nrels, new_sz; 7042 const struct btf_var_secinfo *vi = NULL; 7043 const struct btf_type *sec, *var, *def; 7044 struct bpf_map *map = NULL, *targ_map = NULL; 7045 struct bpf_program *targ_prog = NULL; 7046 bool is_prog_array, is_map_in_map; 7047 const struct btf_member *member; 7048 const char *name, *mname, *type; 7049 unsigned int moff; 7050 Elf64_Sym *sym; 7051 Elf64_Rel *rel; 7052 void *tmp; 7053 7054 if (!obj->efile.btf_maps_sec_btf_id || !obj->btf) 7055 return -EINVAL; 7056 sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id); 7057 if (!sec) 7058 return -EINVAL; 7059 7060 nrels = shdr->sh_size / shdr->sh_entsize; 7061 for (i = 0; i < nrels; i++) { 7062 rel = elf_rel_by_idx(data, i); 7063 if (!rel) { 7064 pr_warn(".maps relo #%d: failed to get ELF relo\n", i); 7065 return -LIBBPF_ERRNO__FORMAT; 7066 } 7067 7068 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); 7069 if (!sym) { 7070 pr_warn(".maps relo #%d: symbol %zx not found\n", 7071 i, (size_t)ELF64_R_SYM(rel->r_info)); 7072 return -LIBBPF_ERRNO__FORMAT; 7073 } 7074 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; 7075 7076 pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n", 7077 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value, 7078 (size_t)rel->r_offset, sym->st_name, name); 7079 7080 for (j = 0; j < obj->nr_maps; j++) { 7081 map = &obj->maps[j]; 7082 if (map->sec_idx != obj->efile.btf_maps_shndx) 7083 continue; 7084 7085 vi = btf_var_secinfos(sec) + map->btf_var_idx; 7086 if (vi->offset <= rel->r_offset && 7087 rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size) 7088 break; 7089 } 7090 if (j == obj->nr_maps) { 7091 pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n", 7092 i, name, (size_t)rel->r_offset); 7093 return -EINVAL; 7094 } 7095 7096 is_map_in_map = bpf_map_type__is_map_in_map(map->def.type); 7097 is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY; 7098 type = is_map_in_map ? "map" : "prog"; 7099 if (is_map_in_map) { 7100 if (sym->st_shndx != obj->efile.btf_maps_shndx) { 7101 pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n", 7102 i, name); 7103 return -LIBBPF_ERRNO__RELOC; 7104 } 7105 if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS && 7106 map->def.key_size != sizeof(int)) { 7107 pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n", 7108 i, map->name, sizeof(int)); 7109 return -EINVAL; 7110 } 7111 targ_map = bpf_object__find_map_by_name(obj, name); 7112 if (!targ_map) { 7113 pr_warn(".maps relo #%d: '%s' isn't a valid map reference\n", 7114 i, name); 7115 return -ESRCH; 7116 } 7117 } else if (is_prog_array) { 7118 targ_prog = bpf_object__find_program_by_name(obj, name); 7119 if (!targ_prog) { 7120 pr_warn(".maps relo #%d: '%s' isn't a valid program reference\n", 7121 i, name); 7122 return -ESRCH; 7123 } 7124 if (targ_prog->sec_idx != sym->st_shndx || 7125 targ_prog->sec_insn_off * 8 != sym->st_value || 7126 prog_is_subprog(obj, targ_prog)) { 7127 pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n", 7128 i, name); 7129 return -LIBBPF_ERRNO__RELOC; 7130 } 7131 } else { 7132 return -EINVAL; 7133 } 7134 7135 var = btf__type_by_id(obj->btf, vi->type); 7136 def = skip_mods_and_typedefs(obj->btf, var->type, NULL); 7137 if (btf_vlen(def) == 0) 7138 return -EINVAL; 7139 member = btf_members(def) + btf_vlen(def) - 1; 7140 mname = btf__name_by_offset(obj->btf, member->name_off); 7141 if (strcmp(mname, "values")) 7142 return -EINVAL; 7143 7144 moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8; 7145 if (rel->r_offset - vi->offset < moff) 7146 return -EINVAL; 7147 7148 moff = rel->r_offset - vi->offset - moff; 7149 /* here we use BPF pointer size, which is always 64 bit, as we 7150 * are parsing ELF that was built for BPF target 7151 */ 7152 if (moff % bpf_ptr_sz) 7153 return -EINVAL; 7154 moff /= bpf_ptr_sz; 7155 if (moff >= map->init_slots_sz) { 7156 new_sz = moff + 1; 7157 tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz); 7158 if (!tmp) 7159 return -ENOMEM; 7160 map->init_slots = tmp; 7161 memset(map->init_slots + map->init_slots_sz, 0, 7162 (new_sz - map->init_slots_sz) * host_ptr_sz); 7163 map->init_slots_sz = new_sz; 7164 } 7165 map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog; 7166 7167 pr_debug(".maps relo #%d: map '%s' slot [%d] points to %s '%s'\n", 7168 i, map->name, moff, type, name); 7169 } 7170 7171 return 0; 7172} 7173 7174static int bpf_object__collect_relos(struct bpf_object *obj) 7175{ 7176 int i, err; 7177 7178 for (i = 0; i < obj->efile.sec_cnt; i++) { 7179 struct elf_sec_desc *sec_desc = &obj->efile.secs[i]; 7180 Elf64_Shdr *shdr; 7181 Elf_Data *data; 7182 int idx; 7183 Elf64_Shdr shdrelf; 7184 7185 if (sec_desc->sec_type != SEC_RELO) 7186 continue; 7187 7188#if defined HAVE_LIBELF 7189 shdr = sec_desc->shdr; 7190#elif defined HAVE_ELFIO 7191 shdr = elf_sec_hdr_by_idx(obj, i, &shdrelf); 7192#endif 7193 data = sec_desc->data; 7194 idx = shdr->sh_info; 7195 7196 if (shdr->sh_type != SHT_REL) { 7197 pr_warn("internal error at %d\n", __LINE__); 7198 return -LIBBPF_ERRNO__INTERNAL; 7199 } 7200 7201 if (idx == obj->efile.st_ops_shndx || idx == obj->efile.st_ops_link_shndx) 7202 err = bpf_object__collect_st_ops_relos(obj, shdr, data); 7203 else if (idx == obj->efile.btf_maps_shndx) 7204 err = bpf_object__collect_map_relos(obj, shdr, data); 7205 else 7206 err = bpf_object__collect_prog_relos(obj, shdr, data); 7207 if (err) 7208 return err; 7209 } 7210 7211 bpf_object__sort_relos(obj); 7212 return 0; 7213} 7214 7215static bool insn_is_helper_call(struct bpf_insn *insn, enum bpf_func_id *func_id) 7216{ 7217 if (BPF_CLASS(insn->code) == BPF_JMP && 7218 BPF_OP(insn->code) == BPF_CALL && 7219 BPF_SRC(insn->code) == BPF_K && 7220 insn->src_reg == 0 && 7221 insn->dst_reg == 0) { 7222 *func_id = insn->imm; 7223 return true; 7224 } 7225 return false; 7226} 7227 7228static int bpf_object__sanitize_prog(struct bpf_object *obj, struct bpf_program *prog) 7229{ 7230 struct bpf_insn *insn = prog->insns; 7231 enum bpf_func_id func_id; 7232 int i; 7233 7234 if (obj->gen_loader) 7235 return 0; 7236 7237 for (i = 0; i < prog->insns_cnt; i++, insn++) { 7238 if (!insn_is_helper_call(insn, &func_id)) 7239 continue; 7240 7241 /* on kernels that don't yet support 7242 * bpf_probe_read_{kernel,user}[_str] helpers, fall back 7243 * to bpf_probe_read() which works well for old kernels 7244 */ 7245 switch (func_id) { 7246 case BPF_FUNC_probe_read_kernel: 7247 case BPF_FUNC_probe_read_user: 7248 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 7249 insn->imm = BPF_FUNC_probe_read; 7250 break; 7251 case BPF_FUNC_probe_read_kernel_str: 7252 case BPF_FUNC_probe_read_user_str: 7253 if (!kernel_supports(obj, FEAT_PROBE_READ_KERN)) 7254 insn->imm = BPF_FUNC_probe_read_str; 7255 break; 7256 default: 7257 break; 7258 } 7259 } 7260 return 0; 7261} 7262 7263static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, 7264 int *btf_obj_fd, int *btf_type_id); 7265 7266/* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */ 7267static int libbpf_prepare_prog_load(struct bpf_program *prog, 7268 struct bpf_prog_load_opts *opts, long cookie) 7269{ 7270 enum sec_def_flags def = cookie; 7271 7272 /* old kernels might not support specifying expected_attach_type */ 7273 if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) 7274 opts->expected_attach_type = 0; 7275 7276 if (def & SEC_SLEEPABLE) 7277 opts->prog_flags |= BPF_F_SLEEPABLE; 7278 7279 if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) 7280 opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; 7281 7282 /* special check for usdt to use uprobe_multi link */ 7283 if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK)) 7284 prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI; 7285 7286 if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { 7287 int btf_obj_fd = 0, btf_type_id = 0, err; 7288 const char *attach_name; 7289 7290 attach_name = strchr(prog->sec_name, '/'); 7291 if (!attach_name) { 7292 /* if BPF program is annotated with just SEC("fentry") 7293 * (or similar) without declaratively specifying 7294 * target, then it is expected that target will be 7295 * specified with bpf_program__set_attach_target() at 7296 * runtime before BPF object load step. If not, then 7297 * there is nothing to load into the kernel as BPF 7298 * verifier won't be able to validate BPF program 7299 * correctness anyways. 7300 */ 7301 pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n", 7302 prog->name); 7303 return -EINVAL; 7304 } 7305 attach_name++; /* skip over / */ 7306 7307 err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id); 7308 if (err) 7309 return err; 7310 7311 /* cache resolved BTF FD and BTF type ID in the prog */ 7312 prog->attach_btf_obj_fd = btf_obj_fd; 7313 prog->attach_btf_id = btf_type_id; 7314 7315 /* but by now libbpf common logic is not utilizing 7316 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because 7317 * this callback is called after opts were populated by 7318 * libbpf, so this callback has to update opts explicitly here 7319 */ 7320 opts->attach_btf_obj_fd = btf_obj_fd; 7321 opts->attach_btf_id = btf_type_id; 7322 } 7323 return 0; 7324} 7325 7326static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); 7327 7328static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, 7329 struct bpf_insn *insns, int insns_cnt, 7330 const char *license, __u32 kern_version, int *prog_fd) 7331{ 7332 LIBBPF_OPTS(bpf_prog_load_opts, load_attr); 7333 const char *prog_name = NULL; 7334 char *cp, errmsg[STRERR_BUFSIZE]; 7335 size_t log_buf_size = 0; 7336 char *log_buf = NULL, *tmp; 7337 int btf_fd, ret, err; 7338 bool own_log_buf = true; 7339 __u32 log_level = prog->log_level; 7340 7341 if (prog->type == BPF_PROG_TYPE_UNSPEC) { 7342 /* 7343 * The program type must be set. Most likely we couldn't find a proper 7344 * section definition at load time, and thus we didn't infer the type. 7345 */ 7346 pr_warn("prog '%s': missing BPF prog type, check ELF section name '%s'\n", 7347 prog->name, prog->sec_name); 7348 return -EINVAL; 7349 } 7350 7351 if (!insns || !insns_cnt) 7352 return -EINVAL; 7353 7354 if (kernel_supports(obj, FEAT_PROG_NAME)) 7355 prog_name = prog->name; 7356 load_attr.attach_prog_fd = prog->attach_prog_fd; 7357 load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; 7358 load_attr.attach_btf_id = prog->attach_btf_id; 7359 load_attr.kern_version = kern_version; 7360 load_attr.prog_ifindex = prog->prog_ifindex; 7361 7362 /* specify func_info/line_info only if kernel supports them */ 7363 btf_fd = bpf_object__btf_fd(obj); 7364 if (btf_fd >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) { 7365 load_attr.prog_btf_fd = btf_fd; 7366 load_attr.func_info = prog->func_info; 7367 load_attr.func_info_rec_size = prog->func_info_rec_size; 7368 load_attr.func_info_cnt = prog->func_info_cnt; 7369 load_attr.line_info = prog->line_info; 7370 load_attr.line_info_rec_size = prog->line_info_rec_size; 7371 load_attr.line_info_cnt = prog->line_info_cnt; 7372 } 7373 load_attr.log_level = log_level; 7374 load_attr.prog_flags = prog->prog_flags; 7375 load_attr.fd_array = obj->fd_array; 7376 7377 /* adjust load_attr if sec_def provides custom preload callback */ 7378 if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) { 7379 err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie); 7380 if (err < 0) { 7381 pr_warn("prog '%s': failed to prepare load attributes: %d\n", 7382 prog->name, err); 7383 return err; 7384 } 7385 insns = prog->insns; 7386 insns_cnt = prog->insns_cnt; 7387 } 7388 7389 /* allow prog_prepare_load_fn to change expected_attach_type */ 7390 load_attr.expected_attach_type = prog->expected_attach_type; 7391 7392 if (obj->gen_loader) { 7393 bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, 7394 license, insns, insns_cnt, &load_attr, 7395 prog - obj->programs); 7396 *prog_fd = -1; 7397 return 0; 7398 } 7399 7400retry_load: 7401 /* if log_level is zero, we don't request logs initially even if 7402 * custom log_buf is specified; if the program load fails, then we'll 7403 * bump log_level to 1 and use either custom log_buf or we'll allocate 7404 * our own and retry the load to get details on what failed 7405 */ 7406 if (log_level) { 7407 if (prog->log_buf) { 7408 log_buf = prog->log_buf; 7409 log_buf_size = prog->log_size; 7410 own_log_buf = false; 7411 } else if (obj->log_buf) { 7412 log_buf = obj->log_buf; 7413 log_buf_size = obj->log_size; 7414 own_log_buf = false; 7415 } else { 7416 log_buf_size = max((size_t)BPF_LOG_BUF_SIZE, log_buf_size * 2); 7417 tmp = realloc(log_buf, log_buf_size); 7418 if (!tmp) { 7419 ret = -ENOMEM; 7420 goto out; 7421 } 7422 log_buf = tmp; 7423 log_buf[0] = '\0'; 7424 own_log_buf = true; 7425 } 7426 } 7427 7428 load_attr.log_buf = log_buf; 7429 load_attr.log_size = log_buf_size; 7430 load_attr.log_level = log_level; 7431 7432 ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); 7433 if (ret >= 0) { 7434 if (log_level && own_log_buf) { 7435 pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", 7436 prog->name, log_buf); 7437 } 7438 7439 if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) { 7440 struct bpf_map *map; 7441 int i; 7442 7443 for (i = 0; i < obj->nr_maps; i++) { 7444 map = &prog->obj->maps[i]; 7445 if (map->libbpf_type != LIBBPF_MAP_RODATA) 7446 continue; 7447 7448 if (bpf_prog_bind_map(ret, bpf_map__fd(map), NULL)) { 7449 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7450 pr_warn("prog '%s': failed to bind map '%s': %s\n", 7451 prog->name, map->real_name, cp); 7452 /* Don't fail hard if can't bind rodata. */ 7453 } 7454 } 7455 } 7456 7457 *prog_fd = ret; 7458 ret = 0; 7459 goto out; 7460 } 7461 7462 if (log_level == 0) { 7463 log_level = 1; 7464 goto retry_load; 7465 } 7466 /* On ENOSPC, increase log buffer size and retry, unless custom 7467 * log_buf is specified. 7468 * Be careful to not overflow u32, though. Kernel's log buf size limit 7469 * isn't part of UAPI so it can always be bumped to full 4GB. So don't 7470 * multiply by 2 unless we are sure we'll fit within 32 bits. 7471 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2). 7472 */ 7473 if (own_log_buf && errno == ENOSPC && log_buf_size <= UINT_MAX / 2) 7474 goto retry_load; 7475 7476 ret = -errno; 7477 7478 /* post-process verifier log to improve error descriptions */ 7479 fixup_verifier_log(prog, log_buf, log_buf_size); 7480 7481 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 7482 pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); 7483 pr_perm_msg(ret); 7484 7485 if (own_log_buf && log_buf && log_buf[0] != '\0') { 7486 pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", 7487 prog->name, log_buf); 7488 } 7489 7490out: 7491 if (own_log_buf) 7492 free(log_buf); 7493 return ret; 7494} 7495 7496static char *find_prev_line(char *buf, char *cur) 7497{ 7498 char *p; 7499 7500 if (cur == buf) /* end of a log buf */ 7501 return NULL; 7502 7503 p = cur - 1; 7504 while (p - 1 >= buf && *(p - 1) != '\n') 7505 p--; 7506 7507 return p; 7508} 7509 7510static void patch_log(char *buf, size_t buf_sz, size_t log_sz, 7511 char *orig, size_t orig_sz, const char *patch) 7512{ 7513 /* size of the remaining log content to the right from the to-be-replaced part */ 7514 size_t rem_sz = (buf + log_sz) - (orig + orig_sz); 7515 size_t patch_sz = strlen(patch); 7516 7517 if (patch_sz != orig_sz) { 7518 /* If patch line(s) are longer than original piece of verifier log, 7519 * shift log contents by (patch_sz - orig_sz) bytes to the right 7520 * starting from after to-be-replaced part of the log. 7521 * 7522 * If patch line(s) are shorter than original piece of verifier log, 7523 * shift log contents by (orig_sz - patch_sz) bytes to the left 7524 * starting from after to-be-replaced part of the log 7525 * 7526 * We need to be careful about not overflowing available 7527 * buf_sz capacity. If that's the case, we'll truncate the end 7528 * of the original log, as necessary. 7529 */ 7530 if (patch_sz > orig_sz) { 7531 if (orig + patch_sz >= buf + buf_sz) { 7532 /* patch is big enough to cover remaining space completely */ 7533 patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1; 7534 rem_sz = 0; 7535 } else if (patch_sz - orig_sz > buf_sz - log_sz) { 7536 /* patch causes part of remaining log to be truncated */ 7537 rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz); 7538 } 7539 } 7540 /* shift remaining log to the right by calculated amount */ 7541 memmove(orig + patch_sz, orig + orig_sz, rem_sz); 7542 } 7543 7544 memcpy(orig, patch, patch_sz); 7545} 7546 7547static void fixup_log_failed_core_relo(struct bpf_program *prog, 7548 char *buf, size_t buf_sz, size_t log_sz, 7549 char *line1, char *line2, char *line3) 7550{ 7551 /* Expected log for failed and not properly guarded CO-RE relocation: 7552 * line1 -> 123: (85) call unknown#195896080 7553 * line2 -> invalid func unknown#195896080 7554 * line3 -> <anything else or end of buffer> 7555 * 7556 * "123" is the index of the instruction that was poisoned. We extract 7557 * instruction index to find corresponding CO-RE relocation and 7558 * replace this part of the log with more relevant information about 7559 * failed CO-RE relocation. 7560 */ 7561 const struct bpf_core_relo *relo; 7562 struct bpf_core_spec spec; 7563 char patch[512], spec_buf[256]; 7564 int insn_idx, err, spec_len; 7565 7566 if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1) 7567 return; 7568 7569 relo = find_relo_core(prog, insn_idx); 7570 if (!relo) 7571 return; 7572 7573 err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); 7574 if (err) 7575 return; 7576 7577 spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec); 7578 snprintf(patch, sizeof(patch), 7579 "%d: <invalid CO-RE relocation>\n" 7580 "failed to resolve CO-RE relocation %s%s\n", 7581 insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : ""); 7582 7583 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 7584} 7585 7586static void fixup_log_missing_map_load(struct bpf_program *prog, 7587 char *buf, size_t buf_sz, size_t log_sz, 7588 char *line1, char *line2, char *line3) 7589{ 7590 /* Expected log for failed and not properly guarded map reference: 7591 * line1 -> 123: (85) call unknown#2001000345 7592 * line2 -> invalid func unknown#2001000345 7593 * line3 -> <anything else or end of buffer> 7594 * 7595 * "123" is the index of the instruction that was poisoned. 7596 * "345" in "2001000345" is a map index in obj->maps to fetch map name. 7597 */ 7598 struct bpf_object *obj = prog->obj; 7599 const struct bpf_map *map; 7600 int insn_idx, map_idx; 7601 char patch[128]; 7602 7603 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2) 7604 return; 7605 7606 map_idx -= POISON_LDIMM64_MAP_BASE; 7607 if (map_idx < 0 || map_idx >= obj->nr_maps) 7608 return; 7609 map = &obj->maps[map_idx]; 7610 7611 snprintf(patch, sizeof(patch), 7612 "%d: <invalid BPF map reference>\n" 7613 "BPF map '%s' is referenced but wasn't created\n", 7614 insn_idx, map->name); 7615 7616 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 7617} 7618 7619static void fixup_log_missing_kfunc_call(struct bpf_program *prog, 7620 char *buf, size_t buf_sz, size_t log_sz, 7621 char *line1, char *line2, char *line3) 7622{ 7623 /* Expected log for failed and not properly guarded kfunc call: 7624 * line1 -> 123: (85) call unknown#2002000345 7625 * line2 -> invalid func unknown#2002000345 7626 * line3 -> <anything else or end of buffer> 7627 * 7628 * "123" is the index of the instruction that was poisoned. 7629 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name. 7630 */ 7631 struct bpf_object *obj = prog->obj; 7632 const struct extern_desc *ext; 7633 int insn_idx, ext_idx; 7634 char patch[128]; 7635 7636 if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &ext_idx) != 2) 7637 return; 7638 7639 ext_idx -= POISON_CALL_KFUNC_BASE; 7640 if (ext_idx < 0 || ext_idx >= obj->nr_extern) 7641 return; 7642 ext = &obj->externs[ext_idx]; 7643 7644 snprintf(patch, sizeof(patch), 7645 "%d: <invalid kfunc call>\n" 7646 "kfunc '%s' is referenced but wasn't resolved\n", 7647 insn_idx, ext->name); 7648 7649 patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); 7650} 7651 7652static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz) 7653{ 7654 /* look for familiar error patterns in last N lines of the log */ 7655 const size_t max_last_line_cnt = 10; 7656 char *prev_line, *cur_line, *next_line; 7657 size_t log_sz; 7658 int i; 7659 7660 if (!buf) 7661 return; 7662 7663 log_sz = strlen(buf) + 1; 7664 next_line = buf + log_sz - 1; 7665 7666 for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) { 7667 cur_line = find_prev_line(buf, next_line); 7668 if (!cur_line) 7669 return; 7670 7671 if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) { 7672 prev_line = find_prev_line(buf, cur_line); 7673 if (!prev_line) 7674 continue; 7675 7676 /* failed CO-RE relocation case */ 7677 fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz, 7678 prev_line, cur_line, next_line); 7679 return; 7680 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_LDIMM64_MAP_PFX)) { 7681 prev_line = find_prev_line(buf, cur_line); 7682 if (!prev_line) 7683 continue; 7684 7685 /* reference to uncreated BPF map */ 7686 fixup_log_missing_map_load(prog, buf, buf_sz, log_sz, 7687 prev_line, cur_line, next_line); 7688 return; 7689 } else if (str_has_pfx(cur_line, "invalid func unknown#"POISON_CALL_KFUNC_PFX)) { 7690 prev_line = find_prev_line(buf, cur_line); 7691 if (!prev_line) 7692 continue; 7693 7694 /* reference to unresolved kfunc */ 7695 fixup_log_missing_kfunc_call(prog, buf, buf_sz, log_sz, 7696 prev_line, cur_line, next_line); 7697 return; 7698 } 7699 } 7700} 7701 7702static int bpf_program_record_relos(struct bpf_program *prog) 7703{ 7704 struct bpf_object *obj = prog->obj; 7705 int i; 7706 7707 for (i = 0; i < prog->nr_reloc; i++) { 7708 struct reloc_desc *relo = &prog->reloc_desc[i]; 7709 struct extern_desc *ext = &obj->externs[relo->ext_idx]; 7710 int kind; 7711 7712 switch (relo->type) { 7713 case RELO_EXTERN_LD64: 7714 if (ext->type != EXT_KSYM) 7715 continue; 7716 kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ? 7717 BTF_KIND_VAR : BTF_KIND_FUNC; 7718 bpf_gen__record_extern(obj->gen_loader, ext->name, 7719 ext->is_weak, !ext->ksym.type_id, 7720 true, kind, relo->insn_idx); 7721 break; 7722 case RELO_EXTERN_CALL: 7723 bpf_gen__record_extern(obj->gen_loader, ext->name, 7724 ext->is_weak, false, false, BTF_KIND_FUNC, 7725 relo->insn_idx); 7726 break; 7727 case RELO_CORE: { 7728 struct bpf_core_relo cr = { 7729 .insn_off = relo->insn_idx * 8, 7730 .type_id = relo->core_relo->type_id, 7731 .access_str_off = relo->core_relo->access_str_off, 7732 .kind = relo->core_relo->kind, 7733 }; 7734 7735 bpf_gen__record_relo_core(obj->gen_loader, &cr); 7736 break; 7737 } 7738 default: 7739 continue; 7740 } 7741 } 7742 return 0; 7743} 7744 7745static int 7746bpf_object__load_progs(struct bpf_object *obj, int log_level) 7747{ 7748 struct bpf_program *prog; 7749 size_t i; 7750 int err; 7751 7752 for (i = 0; i < obj->nr_programs; i++) { 7753 prog = &obj->programs[i]; 7754 err = bpf_object__sanitize_prog(obj, prog); 7755 if (err) 7756 return err; 7757 } 7758 7759 for (i = 0; i < obj->nr_programs; i++) { 7760 prog = &obj->programs[i]; 7761 if (prog_is_subprog(obj, prog)) 7762 continue; 7763 if (!prog->autoload) { 7764 pr_debug("prog '%s': skipped loading\n", prog->name); 7765 continue; 7766 } 7767 prog->log_level |= log_level; 7768 7769 if (obj->gen_loader) 7770 bpf_program_record_relos(prog); 7771 7772 err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, 7773 obj->license, obj->kern_version, &prog->fd); 7774 if (err) { 7775 pr_warn("prog '%s': failed to load: %d\n", prog->name, err); 7776 return err; 7777 } 7778 } 7779 7780 bpf_object__free_relocs(obj); 7781 return 0; 7782} 7783 7784static const struct bpf_sec_def *find_sec_def(const char *sec_name); 7785 7786static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object_open_opts *opts) 7787{ 7788 struct bpf_program *prog; 7789 int err; 7790 7791 bpf_object__for_each_program(prog, obj) { 7792 prog->sec_def = find_sec_def(prog->sec_name); 7793 if (!prog->sec_def) { 7794 /* couldn't guess, but user might manually specify */ 7795 pr_debug("prog '%s': unrecognized ELF section name '%s'\n", 7796 prog->name, prog->sec_name); 7797 continue; 7798 } 7799 7800 prog->type = prog->sec_def->prog_type; 7801 prog->expected_attach_type = prog->sec_def->expected_attach_type; 7802 7803 /* sec_def can have custom callback which should be called 7804 * after bpf_program is initialized to adjust its properties 7805 */ 7806 if (prog->sec_def->prog_setup_fn) { 7807 err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie); 7808 if (err < 0) { 7809 pr_warn("prog '%s': failed to initialize: %d\n", 7810 prog->name, err); 7811 return err; 7812 } 7813 } 7814 } 7815 7816 return 0; 7817} 7818 7819static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, size_t obj_buf_sz, 7820 const struct bpf_object_open_opts *opts) 7821{ 7822 const char *obj_name, *kconfig, *btf_tmp_path; 7823 struct bpf_object *obj; 7824 char tmp_name[64]; 7825 int err; 7826 char *log_buf; 7827 size_t log_size; 7828 __u32 log_level; 7829 7830#ifdef HAVE_LIBELF 7831 if (elf_version(EV_CURRENT) == EV_NONE) { 7832 pr_warn("failed to init libelf for %s\n", 7833 path ? : "(mem buf)"); 7834 return ERR_PTR(-LIBBPF_ERRNO__LIBELF); 7835 } 7836#endif 7837 7838 if (!OPTS_VALID(opts, bpf_object_open_opts)) 7839 return ERR_PTR(-EINVAL); 7840 7841 obj_name = OPTS_GET(opts, object_name, NULL); 7842 if (obj_buf) { 7843 if (!obj_name) { 7844 snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx", 7845 (unsigned long)obj_buf, 7846 (unsigned long)obj_buf_sz); 7847 obj_name = tmp_name; 7848 } 7849 path = obj_name; 7850 pr_debug("loading object '%s' from buffer\n", obj_name); 7851 } 7852 7853 log_buf = OPTS_GET(opts, kernel_log_buf, NULL); 7854 log_size = OPTS_GET(opts, kernel_log_size, 0); 7855 log_level = OPTS_GET(opts, kernel_log_level, 0); 7856 if (log_size > UINT_MAX) 7857 return ERR_PTR(-EINVAL); 7858 if (log_size && !log_buf) 7859 return ERR_PTR(-EINVAL); 7860 7861 obj = bpf_object__new(path, obj_buf, obj_buf_sz, obj_name); 7862 if (IS_ERR(obj)) 7863 return obj; 7864 7865 obj->log_buf = log_buf; 7866 obj->log_size = log_size; 7867 obj->log_level = log_level; 7868 7869 btf_tmp_path = OPTS_GET(opts, btf_custom_path, NULL); 7870 if (btf_tmp_path) { 7871 if (strlen(btf_tmp_path) >= PATH_MAX) { 7872 err = -ENAMETOOLONG; 7873 goto out; 7874 } 7875 obj->btf_custom_path = strdup(btf_tmp_path); 7876 if (!obj->btf_custom_path) { 7877 err = -ENOMEM; 7878 goto out; 7879 } 7880 } 7881 7882 kconfig = OPTS_GET(opts, kconfig, NULL); 7883 if (kconfig) { 7884 obj->kconfig = strdup(kconfig); 7885 if (!obj->kconfig) { 7886 err = -ENOMEM; 7887 goto out; 7888 } 7889 } 7890 7891 err = bpf_object__elf_init(obj); 7892 err = err ? : bpf_object__check_endianness(obj); 7893 err = err ? : bpf_object__elf_collect(obj); 7894 err = err ? : bpf_object__collect_externs(obj); 7895 err = err ? : bpf_object_fixup_btf(obj); 7896 err = err ? : bpf_object__init_maps(obj, opts); 7897 err = err ? : bpf_object_init_progs(obj, opts); 7898 err = err ? : bpf_object__collect_relos(obj); 7899 if (err) 7900 goto out; 7901 7902 bpf_object__elf_finish(obj); 7903 7904 return obj; 7905out: 7906 bpf_object__close(obj); 7907 return ERR_PTR(err); 7908} 7909 7910struct bpf_object * 7911bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) 7912{ 7913 if (!path) 7914 return libbpf_err_ptr(-EINVAL); 7915 7916 pr_debug("loading %s\n", path); 7917 7918 return libbpf_ptr(bpf_object_open(path, NULL, 0, opts)); 7919} 7920 7921struct bpf_object *bpf_object__open(const char *path) 7922{ 7923 return bpf_object__open_file(path, NULL); 7924} 7925 7926struct bpf_object * 7927bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, 7928 const struct bpf_object_open_opts *opts) 7929{ 7930 if (!obj_buf || obj_buf_sz == 0) 7931 return libbpf_err_ptr(-EINVAL); 7932 7933 return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts)); 7934} 7935 7936static int bpf_object_unload(struct bpf_object *obj) 7937{ 7938 size_t i; 7939 7940 if (!obj) 7941 return libbpf_err(-EINVAL); 7942 7943 for (i = 0; i < obj->nr_maps; i++) { 7944 zclose(obj->maps[i].fd); 7945 if (obj->maps[i].st_ops) 7946 zfree(&obj->maps[i].st_ops->kern_vdata); 7947 } 7948 7949 for (i = 0; i < obj->nr_programs; i++) 7950 bpf_program__unload(&obj->programs[i]); 7951 7952 return 0; 7953} 7954 7955static int bpf_object__sanitize_maps(struct bpf_object *obj) 7956{ 7957 struct bpf_map *m; 7958 7959 bpf_object__for_each_map(m, obj) { 7960 if (!bpf_map__is_internal(m)) 7961 continue; 7962 if (!kernel_supports(obj, FEAT_ARRAY_MMAP)) 7963 m->def.map_flags &= ~BPF_F_MMAPABLE; 7964 } 7965 7966 return 0; 7967} 7968 7969int libbpf_kallsyms_parse(kallsyms_cb_t cb, void *ctx) 7970{ 7971 char sym_type, sym_name[500]; 7972 unsigned long long sym_addr; 7973 int ret, err = 0; 7974 FILE *f; 7975 7976 f = fopen("/proc/kallsyms", "re"); 7977 if (!f) { 7978 err = -errno; 7979 pr_warn("failed to open /proc/kallsyms: %d\n", err); 7980 return err; 7981 } 7982 7983 while (true) { 7984 ret = fscanf(f, "%llx %c %499s%*[^\n]\n", 7985 &sym_addr, &sym_type, sym_name); 7986 if (ret == EOF && feof(f)) 7987 break; 7988 if (ret != 3) { 7989 pr_warn("failed to read kallsyms entry: %d\n", ret); 7990 err = -EINVAL; 7991 break; 7992 } 7993 7994 err = cb(sym_addr, sym_type, sym_name, ctx); 7995 if (err) 7996 break; 7997 } 7998 7999 fclose(f); 8000 return err; 8001} 8002 8003static int kallsyms_cb(unsigned long long sym_addr, char sym_type, 8004 const char *sym_name, void *ctx) 8005{ 8006 struct bpf_object *obj = ctx; 8007 const struct btf_type *t; 8008 struct extern_desc *ext; 8009 8010 ext = find_extern_by_name(obj, sym_name); 8011 if (!ext || ext->type != EXT_KSYM) 8012 return 0; 8013 8014 t = btf__type_by_id(obj->btf, ext->btf_id); 8015 if (!btf_is_var(t)) 8016 return 0; 8017 8018 if (ext->is_set && ext->ksym.addr != sym_addr) { 8019 pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n", 8020 sym_name, ext->ksym.addr, sym_addr); 8021 return -EINVAL; 8022 } 8023 if (!ext->is_set) { 8024 ext->is_set = true; 8025 ext->ksym.addr = sym_addr; 8026 pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr); 8027 } 8028 return 0; 8029} 8030 8031static int bpf_object__read_kallsyms_file(struct bpf_object *obj) 8032{ 8033 return libbpf_kallsyms_parse(kallsyms_cb, obj); 8034} 8035 8036static int find_ksym_btf_id(struct bpf_object *obj, const char *ksym_name, 8037 __u16 kind, struct btf **res_btf, 8038 struct module_btf **res_mod_btf) 8039{ 8040 struct module_btf *mod_btf; 8041 struct btf *btf; 8042 int i, id, err; 8043 8044 btf = obj->btf_vmlinux; 8045 mod_btf = NULL; 8046 id = btf__find_by_name_kind(btf, ksym_name, kind); 8047 8048 if (id == -ENOENT) { 8049 err = load_module_btfs(obj); 8050 if (err) 8051 return err; 8052 8053 for (i = 0; i < obj->btf_module_cnt; i++) { 8054 /* we assume module_btf's BTF FD is always >0 */ 8055 mod_btf = &obj->btf_modules[i]; 8056 btf = mod_btf->btf; 8057 id = btf__find_by_name_kind_own(btf, ksym_name, kind); 8058 if (id != -ENOENT) 8059 break; 8060 } 8061 } 8062 if (id <= 0) 8063 return -ESRCH; 8064 8065 *res_btf = btf; 8066 *res_mod_btf = mod_btf; 8067 return id; 8068} 8069 8070static int bpf_object__resolve_ksym_var_btf_id(struct bpf_object *obj, 8071 struct extern_desc *ext) 8072{ 8073 const struct btf_type *targ_var, *targ_type; 8074 __u32 targ_type_id, local_type_id; 8075 struct module_btf *mod_btf = NULL; 8076 const char *targ_var_name; 8077 struct btf *btf = NULL; 8078 int id, err; 8079 8080 id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf); 8081 if (id < 0) { 8082 if (id == -ESRCH && ext->is_weak) 8083 return 0; 8084 pr_warn("extern (var ksym) '%s': not found in kernel BTF\n", 8085 ext->name); 8086 return id; 8087 } 8088 8089 /* find local type_id */ 8090 local_type_id = ext->ksym.type_id; 8091 8092 /* find target type_id */ 8093 targ_var = btf__type_by_id(btf, id); 8094 targ_var_name = btf__name_by_offset(btf, targ_var->name_off); 8095 targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id); 8096 8097 err = bpf_core_types_are_compat(obj->btf, local_type_id, 8098 btf, targ_type_id); 8099 if (err <= 0) { 8100 const struct btf_type *local_type; 8101 const char *targ_name, *local_name; 8102 8103 local_type = btf__type_by_id(obj->btf, local_type_id); 8104 local_name = btf__name_by_offset(obj->btf, local_type->name_off); 8105 targ_name = btf__name_by_offset(btf, targ_type->name_off); 8106 8107 pr_warn("extern (var ksym) '%s': incompatible types, expected [%d] %s %s, but kernel has [%d] %s %s\n", 8108 ext->name, local_type_id, 8109 btf_kind_str(local_type), local_name, targ_type_id, 8110 btf_kind_str(targ_type), targ_name); 8111 return -EINVAL; 8112 } 8113 8114 ext->is_set = true; 8115 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; 8116 ext->ksym.kernel_btf_id = id; 8117 pr_debug("extern (var ksym) '%s': resolved to [%d] %s %s\n", 8118 ext->name, id, btf_kind_str(targ_var), targ_var_name); 8119 8120 return 0; 8121} 8122 8123static int bpf_object__resolve_ksym_func_btf_id(struct bpf_object *obj, 8124 struct extern_desc *ext) 8125{ 8126 int local_func_proto_id, kfunc_proto_id, kfunc_id; 8127 struct module_btf *mod_btf = NULL; 8128 const struct btf_type *kern_func; 8129 struct btf *kern_btf = NULL; 8130 int ret; 8131 8132 local_func_proto_id = ext->ksym.type_id; 8133 8134 kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf, 8135 &mod_btf); 8136 if (kfunc_id < 0) { 8137 if (kfunc_id == -ESRCH && ext->is_weak) 8138 return 0; 8139 pr_warn("extern (func ksym) '%s': not found in kernel or module BTFs\n", 8140 ext->name); 8141 return kfunc_id; 8142 } 8143 8144 kern_func = btf__type_by_id(kern_btf, kfunc_id); 8145 kfunc_proto_id = kern_func->type; 8146 8147 ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id, 8148 kern_btf, kfunc_proto_id); 8149 if (ret <= 0) { 8150 if (ext->is_weak) 8151 return 0; 8152 8153 pr_warn("extern (func ksym) '%s': func_proto [%d] incompatible with %s [%d]\n", 8154 ext->name, local_func_proto_id, 8155 mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id); 8156 return -EINVAL; 8157 } 8158 8159 /* set index for module BTF fd in fd_array, if unset */ 8160 if (mod_btf && !mod_btf->fd_array_idx) { 8161 /* insn->off is s16 */ 8162 if (obj->fd_array_cnt == INT16_MAX) { 8163 pr_warn("extern (func ksym) '%s': module BTF fd index %d too big to fit in bpf_insn offset\n", 8164 ext->name, mod_btf->fd_array_idx); 8165 return -E2BIG; 8166 } 8167 /* Cannot use index 0 for module BTF fd */ 8168 if (!obj->fd_array_cnt) 8169 obj->fd_array_cnt = 1; 8170 8171 ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int), 8172 obj->fd_array_cnt + 1); 8173 if (ret) 8174 return ret; 8175 mod_btf->fd_array_idx = obj->fd_array_cnt; 8176 /* we assume module BTF FD is always >0 */ 8177 obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd; 8178 } 8179 8180 ext->is_set = true; 8181 ext->ksym.kernel_btf_id = kfunc_id; 8182 ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0; 8183 /* Also set kernel_btf_obj_fd to make sure that bpf_object__relocate_data() 8184 * populates FD into ld_imm64 insn when it's used to point to kfunc. 8185 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call. 8186 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64. 8187 */ 8188 ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0; 8189 pr_debug("extern (func ksym) '%s': resolved to %s [%d]\n", 8190 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id); 8191 8192 return 0; 8193} 8194 8195static int bpf_object__resolve_ksyms_btf_id(struct bpf_object *obj) 8196{ 8197 const struct btf_type *t; 8198 struct extern_desc *ext; 8199 int i, err; 8200 8201 for (i = 0; i < obj->nr_extern; i++) { 8202 ext = &obj->externs[i]; 8203 if (ext->type != EXT_KSYM || !ext->ksym.type_id) 8204 continue; 8205 8206 if (obj->gen_loader) { 8207 ext->is_set = true; 8208 ext->ksym.kernel_btf_obj_fd = 0; 8209 ext->ksym.kernel_btf_id = 0; 8210 continue; 8211 } 8212 t = btf__type_by_id(obj->btf, ext->btf_id); 8213 if (btf_is_var(t)) 8214 err = bpf_object__resolve_ksym_var_btf_id(obj, ext); 8215 else 8216 err = bpf_object__resolve_ksym_func_btf_id(obj, ext); 8217 if (err) 8218 return err; 8219 } 8220 return 0; 8221} 8222 8223static int bpf_object__resolve_externs(struct bpf_object *obj, 8224 const char *extra_kconfig) 8225{ 8226 bool need_config = false, need_kallsyms = false; 8227 bool need_vmlinux_btf = false; 8228 struct extern_desc *ext; 8229 void *kcfg_data = NULL; 8230 int err, i; 8231 8232 if (obj->nr_extern == 0) 8233 return 0; 8234 8235 if (obj->kconfig_map_idx >= 0) 8236 kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped; 8237 8238 for (i = 0; i < obj->nr_extern; i++) { 8239 ext = &obj->externs[i]; 8240 8241 if (ext->type == EXT_KSYM) { 8242 if (ext->ksym.type_id) 8243 need_vmlinux_btf = true; 8244 else 8245 need_kallsyms = true; 8246 continue; 8247 } else if (ext->type == EXT_KCFG) { 8248 void *ext_ptr = kcfg_data + ext->kcfg.data_off; 8249 __u64 value = 0; 8250 8251 /* Kconfig externs need actual /proc/config.gz */ 8252 if (str_has_pfx(ext->name, "CONFIG_")) { 8253 need_config = true; 8254 continue; 8255 } 8256 8257 /* Virtual kcfg externs are customly handled by libbpf */ 8258 if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { 8259 value = get_kernel_version(); 8260 if (!value) { 8261 pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name); 8262 return -EINVAL; 8263 } 8264 } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) { 8265 value = kernel_supports(obj, FEAT_BPF_COOKIE); 8266 } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) { 8267 value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER); 8268 } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) { 8269 /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed 8270 * __kconfig externs, where LINUX_ ones are virtual and filled out 8271 * customly by libbpf (their values don't come from Kconfig). 8272 * If LINUX_xxx variable is not recognized by libbpf, but is marked 8273 * __weak, it defaults to zero value, just like for CONFIG_xxx 8274 * externs. 8275 */ 8276 pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name); 8277 return -EINVAL; 8278 } 8279 8280 err = set_kcfg_value_num(ext, ext_ptr, value); 8281 if (err) 8282 return err; 8283 pr_debug("extern (kcfg) '%s': set to 0x%llx\n", 8284 ext->name, (long long)value); 8285 } else { 8286 pr_warn("extern '%s': unrecognized extern kind\n", ext->name); 8287 return -EINVAL; 8288 } 8289 } 8290 if (need_config && extra_kconfig) { 8291 err = bpf_object__read_kconfig_mem(obj, extra_kconfig, kcfg_data); 8292 if (err) 8293 return -EINVAL; 8294 need_config = false; 8295 for (i = 0; i < obj->nr_extern; i++) { 8296 ext = &obj->externs[i]; 8297 if (ext->type == EXT_KCFG && !ext->is_set) { 8298 need_config = true; 8299 break; 8300 } 8301 } 8302 } 8303 if (need_config) { 8304 err = bpf_object__read_kconfig_file(obj, kcfg_data); 8305 if (err) 8306 return -EINVAL; 8307 } 8308 if (need_kallsyms) { 8309 err = bpf_object__read_kallsyms_file(obj); 8310 if (err) 8311 return -EINVAL; 8312 } 8313 if (need_vmlinux_btf) { 8314 err = bpf_object__resolve_ksyms_btf_id(obj); 8315 if (err) 8316 return -EINVAL; 8317 } 8318 for (i = 0; i < obj->nr_extern; i++) { 8319 ext = &obj->externs[i]; 8320 8321 if (!ext->is_set && !ext->is_weak) { 8322 pr_warn("extern '%s' (strong): not resolved\n", ext->name); 8323 return -ESRCH; 8324 } else if (!ext->is_set) { 8325 pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n", 8326 ext->name); 8327 } 8328 } 8329 8330 return 0; 8331} 8332 8333static void bpf_map_prepare_vdata(const struct bpf_map *map) 8334{ 8335 struct bpf_struct_ops *st_ops; 8336 __u32 i; 8337 8338 st_ops = map->st_ops; 8339 for (i = 0; i < btf_vlen(st_ops->type); i++) { 8340 struct bpf_program *prog = st_ops->progs[i]; 8341 void *kern_data; 8342 int prog_fd; 8343 8344 if (!prog) 8345 continue; 8346 8347 prog_fd = bpf_program__fd(prog); 8348 kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i]; 8349 *(unsigned long *)kern_data = prog_fd; 8350 } 8351} 8352 8353static int bpf_object_prepare_struct_ops(struct bpf_object *obj) 8354{ 8355 int i; 8356 8357 for (i = 0; i < obj->nr_maps; i++) 8358 if (bpf_map__is_struct_ops(&obj->maps[i])) 8359 bpf_map_prepare_vdata(&obj->maps[i]); 8360 8361 return 0; 8362} 8363 8364static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const char *target_btf_path) 8365{ 8366 int err, i; 8367 8368 if (!obj) 8369 return libbpf_err(-EINVAL); 8370 8371 if (obj->loaded) { 8372 pr_warn("object '%s': load can't be attempted twice\n", obj->name); 8373 return libbpf_err(-EINVAL); 8374 } 8375 8376 if (obj->gen_loader) 8377 bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps); 8378 8379 err = bpf_object__probe_loading(obj); 8380 err = err ? : bpf_object__load_vmlinux_btf(obj, false); 8381 err = err ? : bpf_object__resolve_externs(obj, obj->kconfig); 8382 err = err ? : bpf_object__sanitize_and_load_btf(obj); 8383 err = err ? : bpf_object__sanitize_maps(obj); 8384 err = err ? : bpf_object__init_kern_struct_ops_maps(obj); 8385 err = err ? : bpf_object__create_maps(obj); 8386 err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path); 8387 err = err ? : bpf_object__load_progs(obj, extra_log_level); 8388 err = err ? : bpf_object_init_prog_arrays(obj); 8389 err = err ? : bpf_object_prepare_struct_ops(obj); 8390 8391 if (obj->gen_loader) { 8392 /* reset FDs */ 8393 if (obj->btf) 8394 btf__set_fd(obj->btf, -1); 8395 for (i = 0; i < obj->nr_maps; i++) 8396 obj->maps[i].fd = -1; 8397 if (!err) 8398 err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps); 8399 } 8400 8401 /* clean up fd_array */ 8402 zfree(&obj->fd_array); 8403 8404 /* clean up module BTFs */ 8405 for (i = 0; i < obj->btf_module_cnt; i++) { 8406 close(obj->btf_modules[i].fd); 8407 btf__free(obj->btf_modules[i].btf); 8408 free(obj->btf_modules[i].name); 8409 } 8410 free(obj->btf_modules); 8411 8412 /* clean up vmlinux BTF */ 8413 btf__free(obj->btf_vmlinux); 8414 obj->btf_vmlinux = NULL; 8415 8416 obj->loaded = true; /* doesn't matter if successfully or not */ 8417 8418 if (err) 8419 goto out; 8420 8421 return 0; 8422out: 8423 /* unpin any maps that were auto-pinned during load */ 8424 for (i = 0; i < obj->nr_maps; i++) 8425 if (obj->maps[i].pinned && !obj->maps[i].reused) 8426 bpf_map__unpin(&obj->maps[i], NULL); 8427 8428 bpf_object_unload(obj); 8429 pr_warn("failed to load object '%s'\n", obj->path); 8430 return libbpf_err(err); 8431} 8432 8433int bpf_object__load(struct bpf_object *obj) 8434{ 8435 return bpf_object_load(obj, 0, NULL); 8436} 8437 8438static int make_parent_dir(const char *path) 8439{ 8440 char *cp, errmsg[STRERR_BUFSIZE]; 8441 char *dname, *dir; 8442 int err = 0; 8443 8444 dname = strdup(path); 8445 if (dname == NULL) 8446 return -ENOMEM; 8447 8448 dir = dirname(dname); 8449 if (mkdir(dir, 0700) && errno != EEXIST) 8450 err = -errno; 8451 8452 free(dname); 8453 if (err) { 8454 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 8455 pr_warn("failed to mkdir %s: %s\n", path, cp); 8456 } 8457 return err; 8458} 8459 8460static int check_path(const char *path) 8461{ 8462 char *cp, errmsg[STRERR_BUFSIZE]; 8463 struct statfs st_fs; 8464 char *dname, *dir; 8465 int err = 0; 8466 8467 if (path == NULL) 8468 return -EINVAL; 8469 8470 dname = strdup(path); 8471 if (dname == NULL) 8472 return -ENOMEM; 8473 8474 dir = dirname(dname); 8475 if (statfs(dir, &st_fs)) { 8476 cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); 8477 pr_warn("failed to statfs %s: %s\n", dir, cp); 8478 err = -errno; 8479 } 8480 free(dname); 8481 8482 if (!err && st_fs.f_type != BPF_FS_MAGIC) { 8483 pr_warn("specified path %s is not on BPF FS\n", path); 8484 err = -EINVAL; 8485 } 8486 8487 return err; 8488} 8489 8490int bpf_program__pin(struct bpf_program *prog, const char *path) 8491{ 8492 char *cp, errmsg[STRERR_BUFSIZE]; 8493 int err; 8494 8495 if (prog->fd < 0) { 8496 pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); 8497 return libbpf_err(-EINVAL); 8498 } 8499 8500 err = make_parent_dir(path); 8501 if (err) 8502 return libbpf_err(err); 8503 8504 err = check_path(path); 8505 if (err) 8506 return libbpf_err(err); 8507 8508 if (bpf_obj_pin(prog->fd, path)) { 8509 err = -errno; 8510 cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); 8511 pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp); 8512 return libbpf_err(err); 8513 } 8514 8515 pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); 8516 return 0; 8517} 8518 8519int bpf_program__unpin(struct bpf_program *prog, const char *path) 8520{ 8521 int err; 8522 8523 if (prog->fd < 0) { 8524 pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); 8525 return libbpf_err(-EINVAL); 8526 } 8527 8528 err = check_path(path); 8529 if (err) 8530 return libbpf_err(err); 8531 8532 err = unlink(path); 8533 if (err) 8534 return libbpf_err(-errno); 8535 8536 pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); 8537 return 0; 8538} 8539 8540int bpf_map__pin(struct bpf_map *map, const char *path) 8541{ 8542 char *cp, errmsg[STRERR_BUFSIZE]; 8543 int err; 8544 8545 if (map == NULL) { 8546 pr_warn("invalid map pointer\n"); 8547 return libbpf_err(-EINVAL); 8548 } 8549 8550 if (map->pin_path) { 8551 if (path && strcmp(path, map->pin_path)) { 8552 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 8553 bpf_map__name(map), map->pin_path, path); 8554 return libbpf_err(-EINVAL); 8555 } else if (map->pinned) { 8556 pr_debug("map '%s' already pinned at '%s'; not re-pinning\n", 8557 bpf_map__name(map), map->pin_path); 8558 return 0; 8559 } 8560 } else { 8561 if (!path) { 8562 pr_warn("missing a path to pin map '%s' at\n", 8563 bpf_map__name(map)); 8564 return libbpf_err(-EINVAL); 8565 } else if (map->pinned) { 8566 pr_warn("map '%s' already pinned\n", bpf_map__name(map)); 8567 return libbpf_err(-EEXIST); 8568 } 8569 8570 map->pin_path = strdup(path); 8571 if (!map->pin_path) { 8572 err = -errno; 8573 goto out_err; 8574 } 8575 } 8576 8577 err = make_parent_dir(map->pin_path); 8578 if (err) 8579 return libbpf_err(err); 8580 8581 err = check_path(map->pin_path); 8582 if (err) 8583 return libbpf_err(err); 8584 8585 if (bpf_obj_pin(map->fd, map->pin_path)) { 8586 err = -errno; 8587 goto out_err; 8588 } 8589 8590 map->pinned = true; 8591 pr_debug("pinned map '%s'\n", map->pin_path); 8592 8593 return 0; 8594 8595out_err: 8596 cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg)); 8597 pr_warn("failed to pin map: %s\n", cp); 8598 return libbpf_err(err); 8599} 8600 8601int bpf_map__unpin(struct bpf_map *map, const char *path) 8602{ 8603 int err; 8604 8605 if (map == NULL) { 8606 pr_warn("invalid map pointer\n"); 8607 return libbpf_err(-EINVAL); 8608 } 8609 8610 if (map->pin_path) { 8611 if (path && strcmp(path, map->pin_path)) { 8612 pr_warn("map '%s' already has pin path '%s' different from '%s'\n", 8613 bpf_map__name(map), map->pin_path, path); 8614 return libbpf_err(-EINVAL); 8615 } 8616 path = map->pin_path; 8617 } else if (!path) { 8618 pr_warn("no path to unpin map '%s' from\n", 8619 bpf_map__name(map)); 8620 return libbpf_err(-EINVAL); 8621 } 8622 8623 err = check_path(path); 8624 if (err) 8625 return libbpf_err(err); 8626 8627 err = unlink(path); 8628 if (err != 0) 8629 return libbpf_err(-errno); 8630 8631 map->pinned = false; 8632 pr_debug("unpinned map '%s' from '%s'\n", bpf_map__name(map), path); 8633 8634 return 0; 8635} 8636 8637int bpf_map__set_pin_path(struct bpf_map *map, const char *path) 8638{ 8639 char *new = NULL; 8640 8641 if (path) { 8642 new = strdup(path); 8643 if (!new) 8644 return libbpf_err(-errno); 8645 } 8646 8647 free(map->pin_path); 8648 map->pin_path = new; 8649 return 0; 8650} 8651 8652__alias(bpf_map__pin_path) 8653const char *bpf_map__get_pin_path(const struct bpf_map *map); 8654 8655const char *bpf_map__pin_path(const struct bpf_map *map) 8656{ 8657 return map->pin_path; 8658} 8659 8660bool bpf_map__is_pinned(const struct bpf_map *map) 8661{ 8662 return map->pinned; 8663} 8664 8665static void sanitize_pin_path(char *s) 8666{ 8667 /* bpffs disallows periods in path names */ 8668 while (*s) { 8669 if (*s == '.') 8670 *s = '_'; 8671 s++; 8672 } 8673} 8674 8675int bpf_object__pin_maps(struct bpf_object *obj, const char *path) 8676{ 8677 struct bpf_map *map; 8678 int err; 8679 8680 if (!obj) 8681 return libbpf_err(-ENOENT); 8682 8683 if (!obj->loaded) { 8684 pr_warn("object not yet loaded; load it first\n"); 8685 return libbpf_err(-ENOENT); 8686 } 8687 8688 bpf_object__for_each_map(map, obj) { 8689 char *pin_path = NULL; 8690 char buf[PATH_MAX]; 8691 8692 if (!map->autocreate) 8693 continue; 8694 8695 if (path) { 8696 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 8697 if (err) 8698 goto err_unpin_maps; 8699 sanitize_pin_path(buf); 8700 pin_path = buf; 8701 } else if (!map->pin_path) { 8702 continue; 8703 } 8704 8705 err = bpf_map__pin(map, pin_path); 8706 if (err) 8707 goto err_unpin_maps; 8708 } 8709 8710 return 0; 8711 8712err_unpin_maps: 8713 while ((map = bpf_object__prev_map(obj, map))) { 8714 if (!map->pin_path) 8715 continue; 8716 8717 bpf_map__unpin(map, NULL); 8718 } 8719 8720 return libbpf_err(err); 8721} 8722 8723int bpf_object__unpin_maps(struct bpf_object *obj, const char *path) 8724{ 8725 struct bpf_map *map; 8726 int err; 8727 8728 if (!obj) 8729 return libbpf_err(-ENOENT); 8730 8731 bpf_object__for_each_map(map, obj) { 8732 char *pin_path = NULL; 8733 char buf[PATH_MAX]; 8734 8735 if (path) { 8736 err = pathname_concat(buf, sizeof(buf), path, bpf_map__name(map)); 8737 if (err) 8738 return libbpf_err(err); 8739 sanitize_pin_path(buf); 8740 pin_path = buf; 8741 } else if (!map->pin_path) { 8742 continue; 8743 } 8744 8745 err = bpf_map__unpin(map, pin_path); 8746 if (err) 8747 return libbpf_err(err); 8748 } 8749 8750 return 0; 8751} 8752 8753int bpf_object__pin_programs(struct bpf_object *obj, const char *path) 8754{ 8755 struct bpf_program *prog; 8756 char buf[PATH_MAX]; 8757 int err; 8758 8759 if (!obj) 8760 return libbpf_err(-ENOENT); 8761 8762 if (!obj->loaded) { 8763 pr_warn("object not yet loaded; load it first\n"); 8764 return libbpf_err(-ENOENT); 8765 } 8766 8767 bpf_object__for_each_program(prog, obj) { 8768 err = pathname_concat(buf, sizeof(buf), path, prog->name); 8769 if (err) 8770 goto err_unpin_programs; 8771 8772 err = bpf_program__pin(prog, buf); 8773 if (err) 8774 goto err_unpin_programs; 8775 } 8776 8777 return 0; 8778 8779err_unpin_programs: 8780 while ((prog = bpf_object__prev_program(obj, prog))) { 8781 if (pathname_concat(buf, sizeof(buf), path, prog->name)) 8782 continue; 8783 8784 bpf_program__unpin(prog, buf); 8785 } 8786 8787 return libbpf_err(err); 8788} 8789 8790int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) 8791{ 8792 struct bpf_program *prog; 8793 int err; 8794 8795 if (!obj) 8796 return libbpf_err(-ENOENT); 8797 8798 bpf_object__for_each_program(prog, obj) { 8799 char buf[PATH_MAX]; 8800 8801 err = pathname_concat(buf, sizeof(buf), path, prog->name); 8802 if (err) 8803 return libbpf_err(err); 8804 8805 err = bpf_program__unpin(prog, buf); 8806 if (err) 8807 return libbpf_err(err); 8808 } 8809 8810 return 0; 8811} 8812 8813int bpf_object__pin(struct bpf_object *obj, const char *path) 8814{ 8815 int err; 8816 8817 err = bpf_object__pin_maps(obj, path); 8818 if (err) 8819 return libbpf_err(err); 8820 8821 err = bpf_object__pin_programs(obj, path); 8822 if (err) { 8823 bpf_object__unpin_maps(obj, path); 8824 return libbpf_err(err); 8825 } 8826 8827 return 0; 8828} 8829 8830int bpf_object__unpin(struct bpf_object *obj, const char *path) 8831{ 8832 int err; 8833 8834 err = bpf_object__unpin_programs(obj, path); 8835 if (err) 8836 return libbpf_err(err); 8837 8838 err = bpf_object__unpin_maps(obj, path); 8839 if (err) 8840 return libbpf_err(err); 8841 8842 return 0; 8843} 8844 8845static void bpf_map__destroy(struct bpf_map *map) 8846{ 8847 if (map->inner_map) { 8848 bpf_map__destroy(map->inner_map); 8849 zfree(&map->inner_map); 8850 } 8851 8852 zfree(&map->init_slots); 8853 map->init_slots_sz = 0; 8854 8855 if (map->mmaped) { 8856 size_t mmap_sz; 8857 8858 mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 8859 munmap(map->mmaped, mmap_sz); 8860 map->mmaped = NULL; 8861 } 8862 8863 if (map->st_ops) { 8864 zfree(&map->st_ops->data); 8865 zfree(&map->st_ops->progs); 8866 zfree(&map->st_ops->kern_func_off); 8867 zfree(&map->st_ops); 8868 } 8869 8870 zfree(&map->name); 8871 zfree(&map->real_name); 8872 zfree(&map->pin_path); 8873 8874 if (map->fd >= 0) 8875 zclose(map->fd); 8876} 8877 8878void bpf_object__close(struct bpf_object *obj) 8879{ 8880 size_t i; 8881 8882 if (IS_ERR_OR_NULL(obj)) 8883 return; 8884#ifdef HAVE_LIBELF 8885 usdt_manager_free(obj->usdt_man); 8886 obj->usdt_man = NULL; 8887#endif //HAVE_LIBELF 8888 bpf_gen__free(obj->gen_loader); 8889 bpf_object__elf_finish(obj); 8890 bpf_object_unload(obj); 8891 btf__free(obj->btf); 8892 btf__free(obj->btf_vmlinux); 8893 btf_ext__free(obj->btf_ext); 8894 8895 for (i = 0; i < obj->nr_maps; i++) 8896 bpf_map__destroy(&obj->maps[i]); 8897 8898 zfree(&obj->btf_custom_path); 8899 zfree(&obj->kconfig); 8900 8901 for (i = 0; i < obj->nr_extern; i++) 8902 zfree(&obj->externs[i].essent_name); 8903 8904 zfree(&obj->externs); 8905 obj->nr_extern = 0; 8906 8907 zfree(&obj->maps); 8908 obj->nr_maps = 0; 8909 8910 if (obj->programs && obj->nr_programs) { 8911 for (i = 0; i < obj->nr_programs; i++) 8912 bpf_program__exit(&obj->programs[i]); 8913 } 8914 zfree(&obj->programs); 8915 8916 free(obj); 8917} 8918 8919const char *bpf_object__name(const struct bpf_object *obj) 8920{ 8921 return obj ? obj->name : libbpf_err_ptr(-EINVAL); 8922} 8923 8924unsigned int bpf_object__kversion(const struct bpf_object *obj) 8925{ 8926 return obj ? obj->kern_version : 0; 8927} 8928 8929struct btf *bpf_object__btf(const struct bpf_object *obj) 8930{ 8931 return obj ? obj->btf : NULL; 8932} 8933 8934int bpf_object__btf_fd(const struct bpf_object *obj) 8935{ 8936 return obj->btf ? btf__fd(obj->btf) : -1; 8937} 8938 8939int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) 8940{ 8941 if (obj->loaded) 8942 return libbpf_err(-EINVAL); 8943 8944 obj->kern_version = kern_version; 8945 8946 return 0; 8947} 8948 8949int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) 8950{ 8951 struct bpf_gen *gen; 8952 8953 if (!opts) 8954 return -EFAULT; 8955 if (!OPTS_VALID(opts, gen_loader_opts)) 8956 return -EINVAL; 8957 gen = calloc(sizeof(*gen), 1); 8958 if (!gen) 8959 return -ENOMEM; 8960 gen->opts = opts; 8961 obj->gen_loader = gen; 8962 return 0; 8963} 8964 8965static struct bpf_program * 8966__bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, 8967 bool forward) 8968{ 8969 size_t nr_programs = obj->nr_programs; 8970 ssize_t idx; 8971 8972 if (!nr_programs) 8973 return NULL; 8974 8975 if (!p) 8976 /* Iter from the beginning */ 8977 return forward ? &obj->programs[0] : 8978 &obj->programs[nr_programs - 1]; 8979 8980 if (p->obj != obj) { 8981 pr_warn("error: program handler doesn't match object\n"); 8982 return errno = EINVAL, NULL; 8983 } 8984 8985 idx = (p - obj->programs) + (forward ? 1 : -1); 8986 if (idx >= obj->nr_programs || idx < 0) 8987 return NULL; 8988 return &obj->programs[idx]; 8989} 8990 8991struct bpf_program * 8992bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) 8993{ 8994 struct bpf_program *prog = prev; 8995 8996 do { 8997 prog = __bpf_program__iter(prog, obj, true); 8998 } while (prog && prog_is_subprog(obj, prog)); 8999 9000 return prog; 9001} 9002 9003struct bpf_program * 9004bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next) 9005{ 9006 struct bpf_program *prog = next; 9007 9008 do { 9009 prog = __bpf_program__iter(prog, obj, false); 9010 } while (prog && prog_is_subprog(obj, prog)); 9011 9012 return prog; 9013} 9014 9015void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) 9016{ 9017 prog->prog_ifindex = ifindex; 9018} 9019 9020const char *bpf_program__name(const struct bpf_program *prog) 9021{ 9022 return prog->name; 9023} 9024 9025const char *bpf_program__section_name(const struct bpf_program *prog) 9026{ 9027 return prog->sec_name; 9028} 9029 9030bool bpf_program__autoload(const struct bpf_program *prog) 9031{ 9032 return prog->autoload; 9033} 9034 9035int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) 9036{ 9037 if (prog->obj->loaded) 9038 return libbpf_err(-EINVAL); 9039 9040 prog->autoload = autoload; 9041 return 0; 9042} 9043 9044bool bpf_program__autoattach(const struct bpf_program *prog) 9045{ 9046 return prog->autoattach; 9047} 9048 9049void bpf_program__set_autoattach(struct bpf_program *prog, bool autoattach) 9050{ 9051 prog->autoattach = autoattach; 9052} 9053 9054const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog) 9055{ 9056 return prog->insns; 9057} 9058 9059size_t bpf_program__insn_cnt(const struct bpf_program *prog) 9060{ 9061 return prog->insns_cnt; 9062} 9063 9064int bpf_program__set_insns(struct bpf_program *prog, 9065 struct bpf_insn *new_insns, size_t new_insn_cnt) 9066{ 9067 struct bpf_insn *insns; 9068 9069 if (prog->obj->loaded) 9070 return -EBUSY; 9071 9072 insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); 9073 /* NULL is a valid return from reallocarray if the new count is zero */ 9074 if (!insns && new_insn_cnt) { 9075 pr_warn("prog '%s': failed to realloc prog code\n", prog->name); 9076 return -ENOMEM; 9077 } 9078 memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns)); 9079 9080 prog->insns = insns; 9081 prog->insns_cnt = new_insn_cnt; 9082 return 0; 9083} 9084 9085int bpf_program__fd(const struct bpf_program *prog) 9086{ 9087 if (!prog) 9088 return libbpf_err(-EINVAL); 9089 9090 if (prog->fd < 0) 9091 return libbpf_err(-ENOENT); 9092 9093 return prog->fd; 9094} 9095 9096__alias(bpf_program__type) 9097enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); 9098 9099enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) 9100{ 9101 return prog->type; 9102} 9103 9104static size_t custom_sec_def_cnt; 9105static struct bpf_sec_def *custom_sec_defs; 9106static struct bpf_sec_def custom_fallback_def; 9107static bool has_custom_fallback_def; 9108static int last_custom_sec_def_handler_id; 9109 9110int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) 9111{ 9112 if (prog->obj->loaded) 9113 return libbpf_err(-EBUSY); 9114 9115 /* if type is not changed, do nothing */ 9116 if (prog->type == type) 9117 return 0; 9118 9119 prog->type = type; 9120 9121 /* If a program type was changed, we need to reset associated SEC() 9122 * handler, as it will be invalid now. The only exception is a generic 9123 * fallback handler, which by definition is program type-agnostic and 9124 * is a catch-all custom handler, optionally set by the application, 9125 * so should be able to handle any type of BPF program. 9126 */ 9127 if (prog->sec_def != &custom_fallback_def) 9128 prog->sec_def = NULL; 9129 return 0; 9130} 9131 9132__alias(bpf_program__expected_attach_type) 9133enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); 9134 9135enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog) 9136{ 9137 return prog->expected_attach_type; 9138} 9139 9140int bpf_program__set_expected_attach_type(struct bpf_program *prog, 9141 enum bpf_attach_type type) 9142{ 9143 if (prog->obj->loaded) 9144 return libbpf_err(-EBUSY); 9145 9146 prog->expected_attach_type = type; 9147 return 0; 9148} 9149 9150__u32 bpf_program__flags(const struct bpf_program *prog) 9151{ 9152 return prog->prog_flags; 9153} 9154 9155int bpf_program__set_flags(struct bpf_program *prog, __u32 flags) 9156{ 9157 if (prog->obj->loaded) 9158 return libbpf_err(-EBUSY); 9159 9160 prog->prog_flags = flags; 9161 return 0; 9162} 9163 9164__u32 bpf_program__log_level(const struct bpf_program *prog) 9165{ 9166 return prog->log_level; 9167} 9168 9169int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_level) 9170{ 9171 if (prog->obj->loaded) 9172 return libbpf_err(-EBUSY); 9173 9174 prog->log_level = log_level; 9175 return 0; 9176} 9177 9178const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size) 9179{ 9180 *log_size = prog->log_size; 9181 return prog->log_buf; 9182} 9183 9184int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size) 9185{ 9186 if (log_size && !log_buf) 9187 return -EINVAL; 9188 if (prog->log_size > UINT_MAX) 9189 return -EINVAL; 9190 if (prog->obj->loaded) 9191 return -EBUSY; 9192 9193 prog->log_buf = log_buf; 9194 prog->log_size = log_size; 9195 return 0; 9196} 9197 9198#define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \ 9199 .sec = (char *)sec_pfx, \ 9200 .prog_type = BPF_PROG_TYPE_##ptype, \ 9201 .expected_attach_type = atype, \ 9202 .cookie = (long)(flags), \ 9203 .prog_prepare_load_fn = libbpf_prepare_prog_load, \ 9204 __VA_ARGS__ \ 9205} 9206 9207static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9208static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9209static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9210#ifdef HAVE_LIBELF 9211static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9212#endif //HAVE_LIBELF 9213static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9214static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9215static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9216static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9217static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9218static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9219static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link); 9220 9221static const struct bpf_sec_def section_defs[] = { 9222 SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE), 9223 SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE), 9224 SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE), 9225 SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), 9226 SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), 9227 SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), 9228 SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), 9229 SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), 9230 SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), 9231 SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), 9232 SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), 9233 SEC_DEF("uprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), 9234 SEC_DEF("uretprobe.multi+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_NONE, attach_uprobe_multi), 9235 SEC_DEF("uprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), 9236 SEC_DEF("uretprobe.multi.s+", KPROBE, BPF_TRACE_UPROBE_MULTI, SEC_SLEEPABLE, attach_uprobe_multi), 9237 SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), 9238 SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), 9239#ifdef HAVE_LIBELF 9240 SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), 9241 SEC_DEF("usdt.s+", KPROBE, 0, SEC_USDT | SEC_SLEEPABLE, attach_usdt), 9242#endif //HAVE_LIBELF 9243 SEC_DEF("tc/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), /* alias for tcx */ 9244 SEC_DEF("tc/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), /* alias for tcx */ 9245 SEC_DEF("tcx/ingress", SCHED_CLS, BPF_TCX_INGRESS, SEC_NONE), 9246 SEC_DEF("tcx/egress", SCHED_CLS, BPF_TCX_EGRESS, SEC_NONE), 9247 SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 9248 SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 9249 SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), /* deprecated / legacy, use tcx */ 9250 SEC_DEF("netkit/primary", SCHED_CLS, BPF_NETKIT_PRIMARY, SEC_NONE), 9251 SEC_DEF("netkit/peer", SCHED_CLS, BPF_NETKIT_PEER, SEC_NONE), 9252 SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp), 9253 SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp), 9254 SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), 9255 SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), 9256 SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), 9257 SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), 9258 SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace), 9259 SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace), 9260 SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace), 9261 SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace), 9262 SEC_DEF("fentry.s+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 9263 SEC_DEF("fmod_ret.s+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 9264 SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), 9265 SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace), 9266 SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm), 9267 SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), 9268 SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF), 9269 SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter), 9270 SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter), 9271 SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE), 9272 SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS), 9273 SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), 9274 SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS), 9275 SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), 9276 SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS), 9277 SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT), 9278 SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE), 9279 SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE), 9280 SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE), 9281 SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE), 9282 SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE), 9283 SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT), 9284 SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT), 9285 SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT), 9286 SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE), 9287 SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT), 9288 SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT), 9289 SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT), 9290 SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT), 9291 SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT), 9292 SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE), 9293 SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE), 9294 SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE), 9295 SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT), 9296 SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE), 9297 SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE), 9298 SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE), 9299 SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE), 9300 SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE), 9301 SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE), 9302 SEC_DEF("cgroup/connect_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_CONNECT, SEC_ATTACHABLE), 9303 SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE), 9304 SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE), 9305 SEC_DEF("cgroup/sendmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_SENDMSG, SEC_ATTACHABLE), 9306 SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE), 9307 SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE), 9308 SEC_DEF("cgroup/recvmsg_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_RECVMSG, SEC_ATTACHABLE), 9309 SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE), 9310 SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE), 9311 SEC_DEF("cgroup/getpeername_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETPEERNAME, SEC_ATTACHABLE), 9312 SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE), 9313 SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE), 9314 SEC_DEF("cgroup/getsockname_unix", CGROUP_SOCK_ADDR, BPF_CGROUP_UNIX_GETSOCKNAME, SEC_ATTACHABLE), 9315 SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE), 9316 SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE), 9317 SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE), 9318 SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT), 9319 SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE), 9320 SEC_DEF("struct_ops.s+", STRUCT_OPS, 0, SEC_SLEEPABLE), 9321 SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE), 9322 SEC_DEF("netfilter", NETFILTER, BPF_NETFILTER, SEC_NONE), 9323}; 9324 9325int libbpf_register_prog_handler(const char *sec, 9326 enum bpf_prog_type prog_type, 9327 enum bpf_attach_type exp_attach_type, 9328 const struct libbpf_prog_handler_opts *opts) 9329{ 9330 struct bpf_sec_def *sec_def; 9331 9332 if (!OPTS_VALID(opts, libbpf_prog_handler_opts)) 9333 return libbpf_err(-EINVAL); 9334 9335 if (last_custom_sec_def_handler_id == INT_MAX) /* prevent overflow */ 9336 return libbpf_err(-E2BIG); 9337 9338 if (sec) { 9339 sec_def = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt + 1, 9340 sizeof(*sec_def)); 9341 if (!sec_def) 9342 return libbpf_err(-ENOMEM); 9343 9344 custom_sec_defs = sec_def; 9345 sec_def = &custom_sec_defs[custom_sec_def_cnt]; 9346 } else { 9347 if (has_custom_fallback_def) 9348 return libbpf_err(-EBUSY); 9349 9350 sec_def = &custom_fallback_def; 9351 } 9352 9353 sec_def->sec = sec ? strdup(sec) : NULL; 9354 if (sec && !sec_def->sec) 9355 return libbpf_err(-ENOMEM); 9356 9357 sec_def->prog_type = prog_type; 9358 sec_def->expected_attach_type = exp_attach_type; 9359 sec_def->cookie = OPTS_GET(opts, cookie, 0); 9360 9361 sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL); 9362 sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL); 9363 sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL); 9364 9365 sec_def->handler_id = ++last_custom_sec_def_handler_id; 9366 9367 if (sec) 9368 custom_sec_def_cnt++; 9369 else 9370 has_custom_fallback_def = true; 9371 9372 return sec_def->handler_id; 9373} 9374 9375int libbpf_unregister_prog_handler(int handler_id) 9376{ 9377 struct bpf_sec_def *sec_defs; 9378 int i; 9379 9380 if (handler_id <= 0) 9381 return libbpf_err(-EINVAL); 9382 9383 if (has_custom_fallback_def && custom_fallback_def.handler_id == handler_id) { 9384 memset(&custom_fallback_def, 0, sizeof(custom_fallback_def)); 9385 has_custom_fallback_def = false; 9386 return 0; 9387 } 9388 9389 for (i = 0; i < custom_sec_def_cnt; i++) { 9390 if (custom_sec_defs[i].handler_id == handler_id) 9391 break; 9392 } 9393 9394 if (i == custom_sec_def_cnt) 9395 return libbpf_err(-ENOENT); 9396 9397 free(custom_sec_defs[i].sec); 9398 for (i = i + 1; i < custom_sec_def_cnt; i++) 9399 custom_sec_defs[i - 1] = custom_sec_defs[i]; 9400 custom_sec_def_cnt--; 9401 9402 /* try to shrink the array, but it's ok if we couldn't */ 9403 sec_defs = libbpf_reallocarray(custom_sec_defs, custom_sec_def_cnt, sizeof(*sec_defs)); 9404 /* if new count is zero, reallocarray can return a valid NULL result; 9405 * in this case the previous pointer will be freed, so we *have to* 9406 * reassign old pointer to the new value (even if it's NULL) 9407 */ 9408 if (sec_defs || custom_sec_def_cnt == 0) 9409 custom_sec_defs = sec_defs; 9410 9411 return 0; 9412} 9413 9414static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name) 9415{ 9416 size_t len = strlen(sec_def->sec); 9417 9418 /* "type/" always has to have proper SEC("type/extras") form */ 9419 if (sec_def->sec[len - 1] == '/') { 9420 if (str_has_pfx(sec_name, sec_def->sec)) 9421 return true; 9422 return false; 9423 } 9424 9425 /* "type+" means it can be either exact SEC("type") or 9426 * well-formed SEC("type/extras") with proper '/' separator 9427 */ 9428 if (sec_def->sec[len - 1] == '+') { 9429 len--; 9430 /* not even a prefix */ 9431 if (strncmp(sec_name, sec_def->sec, len) != 0) 9432 return false; 9433 /* exact match or has '/' separator */ 9434 if (sec_name[len] == '\0' || sec_name[len] == '/') 9435 return true; 9436 return false; 9437 } 9438 9439 return strcmp(sec_name, sec_def->sec) == 0; 9440} 9441 9442static const struct bpf_sec_def *find_sec_def(const char *sec_name) 9443{ 9444 const struct bpf_sec_def *sec_def; 9445 int i, n; 9446 9447 n = custom_sec_def_cnt; 9448 for (i = 0; i < n; i++) { 9449 sec_def = &custom_sec_defs[i]; 9450 if (sec_def_matches(sec_def, sec_name)) 9451 return sec_def; 9452 } 9453 9454 n = ARRAY_SIZE(section_defs); 9455 for (i = 0; i < n; i++) { 9456 sec_def = §ion_defs[i]; 9457 if (sec_def_matches(sec_def, sec_name)) 9458 return sec_def; 9459 } 9460 9461 if (has_custom_fallback_def) 9462 return &custom_fallback_def; 9463 9464 return NULL; 9465} 9466 9467#define MAX_TYPE_NAME_SIZE 32 9468 9469static char *libbpf_get_type_names(bool attach_type) 9470{ 9471 int i, len = ARRAY_SIZE(section_defs) * MAX_TYPE_NAME_SIZE; 9472 char *buf; 9473 9474 buf = malloc(len); 9475 if (!buf) 9476 return NULL; 9477 9478 buf[0] = '\0'; 9479 /* Forge string buf with all available names */ 9480 for (i = 0; i < ARRAY_SIZE(section_defs); i++) { 9481 const struct bpf_sec_def *sec_def = §ion_defs[i]; 9482 9483 if (attach_type) { 9484 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) 9485 continue; 9486 9487 if (!(sec_def->cookie & SEC_ATTACHABLE)) 9488 continue; 9489 } 9490 9491 if (strlen(buf) + strlen(section_defs[i].sec) + 2 > len) { 9492 free(buf); 9493 return NULL; 9494 } 9495 strcat(buf, " "); 9496 strcat(buf, section_defs[i].sec); 9497 } 9498 9499 return buf; 9500} 9501 9502int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, 9503 enum bpf_attach_type *expected_attach_type) 9504{ 9505 const struct bpf_sec_def *sec_def; 9506 char *type_names; 9507 9508 if (!name) 9509 return libbpf_err(-EINVAL); 9510 9511 sec_def = find_sec_def(name); 9512 if (sec_def) { 9513 *prog_type = sec_def->prog_type; 9514 *expected_attach_type = sec_def->expected_attach_type; 9515 return 0; 9516 } 9517 9518 pr_debug("failed to guess program type from ELF section '%s'\n", name); 9519 type_names = libbpf_get_type_names(false); 9520 if (type_names != NULL) { 9521 pr_debug("supported section(type) names are:%s\n", type_names); 9522 free(type_names); 9523 } 9524 9525 return libbpf_err(-ESRCH); 9526} 9527 9528const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t) 9529{ 9530 if (t < 0 || t >= ARRAY_SIZE(attach_type_name)) 9531 return NULL; 9532 9533 return attach_type_name[t]; 9534} 9535 9536const char *libbpf_bpf_link_type_str(enum bpf_link_type t) 9537{ 9538 if (t < 0 || t >= ARRAY_SIZE(link_type_name)) 9539 return NULL; 9540 9541 return link_type_name[t]; 9542} 9543 9544const char *libbpf_bpf_map_type_str(enum bpf_map_type t) 9545{ 9546 if (t < 0 || t >= ARRAY_SIZE(map_type_name)) 9547 return NULL; 9548 9549 return map_type_name[t]; 9550} 9551 9552const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t) 9553{ 9554 if (t < 0 || t >= ARRAY_SIZE(prog_type_name)) 9555 return NULL; 9556 9557 return prog_type_name[t]; 9558} 9559 9560static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, 9561 int sec_idx, 9562 size_t offset) 9563{ 9564 struct bpf_map *map; 9565 size_t i; 9566 9567 for (i = 0; i < obj->nr_maps; i++) { 9568 map = &obj->maps[i]; 9569 if (!bpf_map__is_struct_ops(map)) 9570 continue; 9571 if (map->sec_idx == sec_idx && 9572 map->sec_offset <= offset && 9573 offset - map->sec_offset < map->def.value_size) 9574 return map; 9575 } 9576 9577 return NULL; 9578} 9579 9580/* Collect the reloc from ELF and populate the st_ops->progs[] */ 9581static int bpf_object__collect_st_ops_relos(struct bpf_object *obj, 9582 Elf64_Shdr *shdr, Elf_Data *data) 9583{ 9584 const struct btf_member *member; 9585 struct bpf_struct_ops *st_ops; 9586 struct bpf_program *prog; 9587 unsigned int shdr_idx; 9588 const struct btf *btf; 9589 struct bpf_map *map; 9590 unsigned int moff, insn_idx; 9591 const char *name; 9592 __u32 member_idx; 9593 Elf64_Sym *sym; 9594 Elf64_Rel *rel; 9595 int i, nrels; 9596 9597 btf = obj->btf; 9598 nrels = shdr->sh_size / shdr->sh_entsize; 9599 for (i = 0; i < nrels; i++) { 9600 rel = elf_rel_by_idx(data, i); 9601 if (!rel) { 9602 pr_warn("struct_ops reloc: failed to get %d reloc\n", i); 9603 return -LIBBPF_ERRNO__FORMAT; 9604 } 9605 9606 sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); 9607 if (!sym) { 9608 pr_warn("struct_ops reloc: symbol %zx not found\n", 9609 (size_t)ELF64_R_SYM(rel->r_info)); 9610 return -LIBBPF_ERRNO__FORMAT; 9611 } 9612 9613 name = elf_sym_str(obj, sym->st_name) ?: "<?>"; 9614 map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset); 9615 if (!map) { 9616 pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n", 9617 (size_t)rel->r_offset); 9618 return -EINVAL; 9619 } 9620 9621 moff = rel->r_offset - map->sec_offset; 9622 shdr_idx = sym->st_shndx; 9623 st_ops = map->st_ops; 9624 pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %zu name %d (\'%s\')\n", 9625 map->name, 9626 (long long)(rel->r_info >> 32), 9627 (long long)sym->st_value, 9628 shdr_idx, (size_t)rel->r_offset, 9629 map->sec_offset, sym->st_name, name); 9630 9631 if (shdr_idx >= SHN_LORESERVE) { 9632 pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n", 9633 map->name, (size_t)rel->r_offset, shdr_idx); 9634 return -LIBBPF_ERRNO__RELOC; 9635 } 9636 if (sym->st_value % BPF_INSN_SZ) { 9637 pr_warn("struct_ops reloc %s: invalid target program offset %llu\n", 9638 map->name, (unsigned long long)sym->st_value); 9639 return -LIBBPF_ERRNO__FORMAT; 9640 } 9641 insn_idx = sym->st_value / BPF_INSN_SZ; 9642 9643 member = find_member_by_offset(st_ops->type, moff * 8); 9644 if (!member) { 9645 pr_warn("struct_ops reloc %s: cannot find member at moff %u\n", 9646 map->name, moff); 9647 return -EINVAL; 9648 } 9649 member_idx = member - btf_members(st_ops->type); 9650 name = btf__name_by_offset(btf, member->name_off); 9651 9652 if (!resolve_func_ptr(btf, member->type, NULL)) { 9653 pr_warn("struct_ops reloc %s: cannot relocate non func ptr %s\n", 9654 map->name, name); 9655 return -EINVAL; 9656 } 9657 9658 prog = find_prog_by_sec_insn(obj, shdr_idx, insn_idx); 9659 if (!prog) { 9660 pr_warn("struct_ops reloc %s: cannot find prog at shdr_idx %u to relocate func ptr %s\n", 9661 map->name, shdr_idx, name); 9662 return -EINVAL; 9663 } 9664 9665 /* prevent the use of BPF prog with invalid type */ 9666 if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) { 9667 pr_warn("struct_ops reloc %s: prog %s is not struct_ops BPF program\n", 9668 map->name, prog->name); 9669 return -EINVAL; 9670 } 9671 9672 /* if we haven't yet processed this BPF program, record proper 9673 * attach_btf_id and member_idx 9674 */ 9675 if (!prog->attach_btf_id) { 9676 prog->attach_btf_id = st_ops->type_id; 9677 prog->expected_attach_type = member_idx; 9678 } 9679 9680 /* struct_ops BPF prog can be re-used between multiple 9681 * .struct_ops & .struct_ops.link as long as it's the 9682 * same struct_ops struct definition and the same 9683 * function pointer field 9684 */ 9685 if (prog->attach_btf_id != st_ops->type_id || 9686 prog->expected_attach_type != member_idx) { 9687 pr_warn("struct_ops reloc %s: cannot use prog %s in sec %s with type %u attach_btf_id %u expected_attach_type %u for func ptr %s\n", 9688 map->name, prog->name, prog->sec_name, prog->type, 9689 prog->attach_btf_id, prog->expected_attach_type, name); 9690 return -EINVAL; 9691 } 9692 9693 st_ops->progs[member_idx] = prog; 9694 } 9695 9696 return 0; 9697} 9698 9699#define BTF_TRACE_PREFIX "btf_trace_" 9700#define BTF_LSM_PREFIX "bpf_lsm_" 9701#define BTF_ITER_PREFIX "bpf_iter_" 9702#define BTF_MAX_NAME_SIZE 128 9703 9704void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, 9705 const char **prefix, int *kind) 9706{ 9707 switch (attach_type) { 9708 case BPF_TRACE_RAW_TP: 9709 *prefix = BTF_TRACE_PREFIX; 9710 *kind = BTF_KIND_TYPEDEF; 9711 break; 9712 case BPF_LSM_MAC: 9713 case BPF_LSM_CGROUP: 9714 *prefix = BTF_LSM_PREFIX; 9715 *kind = BTF_KIND_FUNC; 9716 break; 9717 case BPF_TRACE_ITER: 9718 *prefix = BTF_ITER_PREFIX; 9719 *kind = BTF_KIND_FUNC; 9720 break; 9721 default: 9722 *prefix = ""; 9723 *kind = BTF_KIND_FUNC; 9724 } 9725} 9726 9727static int find_btf_by_prefix_kind(const struct btf *btf, const char *prefix, 9728 const char *name, __u32 kind) 9729{ 9730 char btf_type_name[BTF_MAX_NAME_SIZE]; 9731 int ret; 9732 9733 ret = snprintf(btf_type_name, sizeof(btf_type_name), 9734 "%s%s", prefix, name); 9735 /* snprintf returns the number of characters written excluding the 9736 * terminating null. So, if >= BTF_MAX_NAME_SIZE are written, it 9737 * indicates truncation. 9738 */ 9739 if (ret < 0 || ret >= sizeof(btf_type_name)) 9740 return -ENAMETOOLONG; 9741 return btf__find_by_name_kind(btf, btf_type_name, kind); 9742} 9743 9744static inline int find_attach_btf_id(struct btf *btf, const char *name, 9745 enum bpf_attach_type attach_type) 9746{ 9747 const char *prefix; 9748 int kind; 9749 9750 btf_get_kernel_prefix_kind(attach_type, &prefix, &kind); 9751 return find_btf_by_prefix_kind(btf, prefix, name, kind); 9752} 9753 9754int libbpf_find_vmlinux_btf_id(const char *name, 9755 enum bpf_attach_type attach_type) 9756{ 9757 struct btf *btf; 9758 int err; 9759 9760 btf = btf__load_vmlinux_btf(); 9761 err = libbpf_get_error(btf); 9762 if (err) { 9763 pr_warn("vmlinux BTF is not found\n"); 9764 return libbpf_err(err); 9765 } 9766 9767 err = find_attach_btf_id(btf, name, attach_type); 9768 if (err <= 0) 9769 pr_warn("%s is not found in vmlinux BTF\n", name); 9770 9771 btf__free(btf); 9772 return libbpf_err(err); 9773} 9774 9775static int libbpf_find_prog_btf_id(const char *name, __u32 attach_prog_fd) 9776{ 9777 struct bpf_prog_info info; 9778 __u32 info_len = sizeof(info); 9779 struct btf *btf; 9780 int err; 9781 9782 memset(&info, 0, info_len); 9783 err = bpf_prog_get_info_by_fd(attach_prog_fd, &info, &info_len); 9784 if (err) { 9785 pr_warn("failed bpf_prog_get_info_by_fd for FD %d: %d\n", 9786 attach_prog_fd, err); 9787 return err; 9788 } 9789 9790 err = -EINVAL; 9791 if (!info.btf_id) { 9792 pr_warn("The target program doesn't have BTF\n"); 9793 goto out; 9794 } 9795 btf = btf__load_from_kernel_by_id(info.btf_id); 9796 err = libbpf_get_error(btf); 9797 if (err) { 9798 pr_warn("Failed to get BTF %d of the program: %d\n", info.btf_id, err); 9799 goto out; 9800 } 9801 err = btf__find_by_name_kind(btf, name, BTF_KIND_FUNC); 9802 btf__free(btf); 9803 if (err <= 0) { 9804 pr_warn("%s is not found in prog's BTF\n", name); 9805 goto out; 9806 } 9807out: 9808 return err; 9809} 9810 9811static int find_kernel_btf_id(struct bpf_object *obj, const char *attach_name, 9812 enum bpf_attach_type attach_type, 9813 int *btf_obj_fd, int *btf_type_id) 9814{ 9815 int ret, i; 9816 9817 ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type); 9818 if (ret > 0) { 9819 *btf_obj_fd = 0; /* vmlinux BTF */ 9820 *btf_type_id = ret; 9821 return 0; 9822 } 9823 if (ret != -ENOENT) 9824 return ret; 9825 9826 ret = load_module_btfs(obj); 9827 if (ret) 9828 return ret; 9829 9830 for (i = 0; i < obj->btf_module_cnt; i++) { 9831 const struct module_btf *mod = &obj->btf_modules[i]; 9832 9833 ret = find_attach_btf_id(mod->btf, attach_name, attach_type); 9834 if (ret > 0) { 9835 *btf_obj_fd = mod->fd; 9836 *btf_type_id = ret; 9837 return 0; 9838 } 9839 if (ret == -ENOENT) 9840 continue; 9841 9842 return ret; 9843 } 9844 9845 return -ESRCH; 9846} 9847 9848static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attach_name, 9849 int *btf_obj_fd, int *btf_type_id) 9850{ 9851 enum bpf_attach_type attach_type = prog->expected_attach_type; 9852 __u32 attach_prog_fd = prog->attach_prog_fd; 9853 int err = 0; 9854 9855 /* BPF program's BTF ID */ 9856 if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) { 9857 if (!attach_prog_fd) { 9858 pr_warn("prog '%s': attach program FD is not set\n", prog->name); 9859 return -EINVAL; 9860 } 9861 err = libbpf_find_prog_btf_id(attach_name, attach_prog_fd); 9862 if (err < 0) { 9863 pr_warn("prog '%s': failed to find BPF program (FD %d) BTF ID for '%s': %d\n", 9864 prog->name, attach_prog_fd, attach_name, err); 9865 return err; 9866 } 9867 *btf_obj_fd = 0; 9868 *btf_type_id = err; 9869 return 0; 9870 } 9871 9872 /* kernel/module BTF ID */ 9873 if (prog->obj->gen_loader) { 9874 bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type); 9875 *btf_obj_fd = 0; 9876 *btf_type_id = 1; 9877 } else { 9878 err = find_kernel_btf_id(prog->obj, attach_name, attach_type, btf_obj_fd, btf_type_id); 9879 } 9880 if (err) { 9881 pr_warn("prog '%s': failed to find kernel BTF type ID of '%s': %d\n", 9882 prog->name, attach_name, err); 9883 return err; 9884 } 9885 return 0; 9886} 9887 9888int libbpf_attach_type_by_name(const char *name, 9889 enum bpf_attach_type *attach_type) 9890{ 9891 char *type_names; 9892 const struct bpf_sec_def *sec_def; 9893 9894 if (!name) 9895 return libbpf_err(-EINVAL); 9896 9897 sec_def = find_sec_def(name); 9898 if (!sec_def) { 9899 pr_debug("failed to guess attach type based on ELF section name '%s'\n", name); 9900 type_names = libbpf_get_type_names(true); 9901 if (type_names != NULL) { 9902 pr_debug("attachable section(type) names are:%s\n", type_names); 9903 free(type_names); 9904 } 9905 9906 return libbpf_err(-EINVAL); 9907 } 9908 9909 if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load) 9910 return libbpf_err(-EINVAL); 9911 if (!(sec_def->cookie & SEC_ATTACHABLE)) 9912 return libbpf_err(-EINVAL); 9913 9914 *attach_type = sec_def->expected_attach_type; 9915 return 0; 9916} 9917 9918int bpf_map__fd(const struct bpf_map *map) 9919{ 9920 return map ? map->fd : libbpf_err(-EINVAL); 9921} 9922 9923static bool map_uses_real_name(const struct bpf_map *map) 9924{ 9925 /* Since libbpf started to support custom .data.* and .rodata.* maps, 9926 * their user-visible name differs from kernel-visible name. Users see 9927 * such map's corresponding ELF section name as a map name. 9928 * This check distinguishes .data/.rodata from .data.* and .rodata.* 9929 * maps to know which name has to be returned to the user. 9930 */ 9931 if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0) 9932 return true; 9933 if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0) 9934 return true; 9935 return false; 9936} 9937 9938const char *bpf_map__name(const struct bpf_map *map) 9939{ 9940 if (!map) 9941 return NULL; 9942 9943 if (map_uses_real_name(map)) 9944 return map->real_name; 9945 9946 return map->name; 9947} 9948 9949enum bpf_map_type bpf_map__type(const struct bpf_map *map) 9950{ 9951 return map->def.type; 9952} 9953 9954int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type) 9955{ 9956 if (map->fd >= 0) 9957 return libbpf_err(-EBUSY); 9958 map->def.type = type; 9959 return 0; 9960} 9961 9962__u32 bpf_map__map_flags(const struct bpf_map *map) 9963{ 9964 return map->def.map_flags; 9965} 9966 9967int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags) 9968{ 9969 if (map->fd >= 0) 9970 return libbpf_err(-EBUSY); 9971 map->def.map_flags = flags; 9972 return 0; 9973} 9974 9975__u64 bpf_map__map_extra(const struct bpf_map *map) 9976{ 9977 return map->map_extra; 9978} 9979 9980int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra) 9981{ 9982 if (map->fd >= 0) 9983 return libbpf_err(-EBUSY); 9984 map->map_extra = map_extra; 9985 return 0; 9986} 9987 9988__u32 bpf_map__numa_node(const struct bpf_map *map) 9989{ 9990 return map->numa_node; 9991} 9992 9993int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node) 9994{ 9995 if (map->fd >= 0) 9996 return libbpf_err(-EBUSY); 9997 map->numa_node = numa_node; 9998 return 0; 9999} 10000 10001__u32 bpf_map__key_size(const struct bpf_map *map) 10002{ 10003 return map->def.key_size; 10004} 10005 10006int bpf_map__set_key_size(struct bpf_map *map, __u32 size) 10007{ 10008 if (map->fd >= 0) 10009 return libbpf_err(-EBUSY); 10010 map->def.key_size = size; 10011 return 0; 10012} 10013 10014__u32 bpf_map__value_size(const struct bpf_map *map) 10015{ 10016 return map->def.value_size; 10017} 10018 10019static int map_btf_datasec_resize(struct bpf_map *map, __u32 size) 10020{ 10021 struct btf *btf; 10022 struct btf_type *datasec_type, *var_type; 10023 struct btf_var_secinfo *var; 10024 const struct btf_type *array_type; 10025 const struct btf_array *array; 10026 int vlen, element_sz, new_array_id; 10027 __u32 nr_elements; 10028 10029 /* check btf existence */ 10030 btf = bpf_object__btf(map->obj); 10031 if (!btf) 10032 return -ENOENT; 10033 10034 /* verify map is datasec */ 10035 datasec_type = btf_type_by_id(btf, bpf_map__btf_value_type_id(map)); 10036 if (!btf_is_datasec(datasec_type)) { 10037 pr_warn("map '%s': cannot be resized, map value type is not a datasec\n", 10038 bpf_map__name(map)); 10039 return -EINVAL; 10040 } 10041 10042 /* verify datasec has at least one var */ 10043 vlen = btf_vlen(datasec_type); 10044 if (vlen == 0) { 10045 pr_warn("map '%s': cannot be resized, map value datasec is empty\n", 10046 bpf_map__name(map)); 10047 return -EINVAL; 10048 } 10049 10050 /* verify last var in the datasec is an array */ 10051 var = &btf_var_secinfos(datasec_type)[vlen - 1]; 10052 var_type = btf_type_by_id(btf, var->type); 10053 array_type = skip_mods_and_typedefs(btf, var_type->type, NULL); 10054 if (!btf_is_array(array_type)) { 10055 pr_warn("map '%s': cannot be resized, last var must be an array\n", 10056 bpf_map__name(map)); 10057 return -EINVAL; 10058 } 10059 10060 /* verify request size aligns with array */ 10061 array = btf_array(array_type); 10062 element_sz = btf__resolve_size(btf, array->type); 10063 if (element_sz <= 0 || (size - var->offset) % element_sz != 0) { 10064 pr_warn("map '%s': cannot be resized, element size (%d) doesn't align with new total size (%u)\n", 10065 bpf_map__name(map), element_sz, size); 10066 return -EINVAL; 10067 } 10068 10069 /* create a new array based on the existing array, but with new length */ 10070 nr_elements = (size - var->offset) / element_sz; 10071 new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements); 10072 if (new_array_id < 0) 10073 return new_array_id; 10074 10075 /* adding a new btf type invalidates existing pointers to btf objects, 10076 * so refresh pointers before proceeding 10077 */ 10078 datasec_type = btf_type_by_id(btf, map->btf_value_type_id); 10079 var = &btf_var_secinfos(datasec_type)[vlen - 1]; 10080 var_type = btf_type_by_id(btf, var->type); 10081 10082 /* finally update btf info */ 10083 datasec_type->size = size; 10084 var->size = size - var->offset; 10085 var_type->type = new_array_id; 10086 10087 return 0; 10088} 10089 10090int bpf_map__set_value_size(struct bpf_map *map, __u32 size) 10091{ 10092 if (map->fd >= 0) 10093 return libbpf_err(-EBUSY); 10094 10095 if (map->mmaped) { 10096 int err; 10097 size_t mmap_old_sz, mmap_new_sz; 10098 10099 mmap_old_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 10100 mmap_new_sz = bpf_map_mmap_sz(size, map->def.max_entries); 10101 err = bpf_map_mmap_resize(map, mmap_old_sz, mmap_new_sz); 10102 if (err) { 10103 pr_warn("map '%s': failed to resize memory-mapped region: %d\n", 10104 bpf_map__name(map), err); 10105 return err; 10106 } 10107 err = map_btf_datasec_resize(map, size); 10108 if (err && err != -ENOENT) { 10109 pr_warn("map '%s': failed to adjust resized BTF, clearing BTF key/value info: %d\n", 10110 bpf_map__name(map), err); 10111 map->btf_value_type_id = 0; 10112 map->btf_key_type_id = 0; 10113 } 10114 } 10115 10116 map->def.value_size = size; 10117 return 0; 10118} 10119 10120__u32 bpf_map__btf_key_type_id(const struct bpf_map *map) 10121{ 10122 return map ? map->btf_key_type_id : 0; 10123} 10124 10125__u32 bpf_map__btf_value_type_id(const struct bpf_map *map) 10126{ 10127 return map ? map->btf_value_type_id : 0; 10128} 10129 10130int bpf_map__set_initial_value(struct bpf_map *map, 10131 const void *data, size_t size) 10132{ 10133 if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG || 10134 size != map->def.value_size || map->fd >= 0) 10135 return libbpf_err(-EINVAL); 10136 10137 memcpy(map->mmaped, data, size); 10138 return 0; 10139} 10140 10141void *bpf_map__initial_value(struct bpf_map *map, size_t *psize) 10142{ 10143 if (!map->mmaped) 10144 return NULL; 10145 *psize = map->def.value_size; 10146 return map->mmaped; 10147} 10148 10149bool bpf_map__is_internal(const struct bpf_map *map) 10150{ 10151 return map->libbpf_type != LIBBPF_MAP_UNSPEC; 10152} 10153 10154__u32 bpf_map__ifindex(const struct bpf_map *map) 10155{ 10156 return map->map_ifindex; 10157} 10158 10159int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex) 10160{ 10161 if (map->fd >= 0) 10162 return libbpf_err(-EBUSY); 10163 map->map_ifindex = ifindex; 10164 return 0; 10165} 10166 10167int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) 10168{ 10169 if (!bpf_map_type__is_map_in_map(map->def.type)) { 10170 pr_warn("error: unsupported map type\n"); 10171 return libbpf_err(-EINVAL); 10172 } 10173 if (map->inner_map_fd != -1) { 10174 pr_warn("error: inner_map_fd already specified\n"); 10175 return libbpf_err(-EINVAL); 10176 } 10177 if (map->inner_map) { 10178 bpf_map__destroy(map->inner_map); 10179 zfree(&map->inner_map); 10180 } 10181 map->inner_map_fd = fd; 10182 return 0; 10183} 10184 10185static struct bpf_map * 10186__bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) 10187{ 10188 ssize_t idx; 10189 struct bpf_map *s, *e; 10190 10191 if (!obj || !obj->maps) 10192 return errno = EINVAL, NULL; 10193 10194 s = obj->maps; 10195 e = obj->maps + obj->nr_maps; 10196 10197 if ((m < s) || (m >= e)) { 10198 pr_warn("error in %s: map handler doesn't belong to object\n", 10199 __func__); 10200 return errno = EINVAL, NULL; 10201 } 10202 10203 idx = (m - obj->maps) + i; 10204 if (idx >= obj->nr_maps || idx < 0) 10205 return NULL; 10206 return &obj->maps[idx]; 10207} 10208 10209struct bpf_map * 10210bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) 10211{ 10212 if (prev == NULL) 10213 return obj->maps; 10214 10215 return __bpf_map__iter(prev, obj, 1); 10216} 10217 10218struct bpf_map * 10219bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next) 10220{ 10221 if (next == NULL) { 10222 if (!obj->nr_maps) 10223 return NULL; 10224 return obj->maps + obj->nr_maps - 1; 10225 } 10226 10227 return __bpf_map__iter(next, obj, -1); 10228} 10229 10230struct bpf_map * 10231bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name) 10232{ 10233 struct bpf_map *pos; 10234 10235 bpf_object__for_each_map(pos, obj) { 10236 /* if it's a special internal map name (which always starts 10237 * with dot) then check if that special name matches the 10238 * real map name (ELF section name) 10239 */ 10240 if (name[0] == '.') { 10241 if (pos->real_name && strcmp(pos->real_name, name) == 0) 10242 return pos; 10243 continue; 10244 } 10245 /* otherwise map name has to be an exact match */ 10246 if (map_uses_real_name(pos)) { 10247 if (strcmp(pos->real_name, name) == 0) 10248 return pos; 10249 continue; 10250 } 10251 if (strcmp(pos->name, name) == 0) 10252 return pos; 10253 } 10254 return errno = ENOENT, NULL; 10255} 10256 10257int 10258bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) 10259{ 10260 return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); 10261} 10262 10263static int validate_map_op(const struct bpf_map *map, size_t key_sz, 10264 size_t value_sz, bool check_value_sz) 10265{ 10266 if (map->fd <= 0) 10267 return -ENOENT; 10268 10269 if (map->def.key_size != key_sz) { 10270 pr_warn("map '%s': unexpected key size %zu provided, expected %u\n", 10271 map->name, key_sz, map->def.key_size); 10272 return -EINVAL; 10273 } 10274 10275 if (!check_value_sz) 10276 return 0; 10277 10278 switch (map->def.type) { 10279 case BPF_MAP_TYPE_PERCPU_ARRAY: 10280 case BPF_MAP_TYPE_PERCPU_HASH: 10281 case BPF_MAP_TYPE_LRU_PERCPU_HASH: 10282 case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: { 10283 int num_cpu = libbpf_num_possible_cpus(); 10284 size_t elem_sz = roundup(map->def.value_size, 8); 10285 10286 if (value_sz != num_cpu * elem_sz) { 10287 pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", 10288 map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); 10289 return -EINVAL; 10290 } 10291 break; 10292 } 10293 default: 10294 if (map->def.value_size != value_sz) { 10295 pr_warn("map '%s': unexpected value size %zu provided, expected %u\n", 10296 map->name, value_sz, map->def.value_size); 10297 return -EINVAL; 10298 } 10299 break; 10300 } 10301 return 0; 10302} 10303 10304int bpf_map__lookup_elem(const struct bpf_map *map, 10305 const void *key, size_t key_sz, 10306 void *value, size_t value_sz, __u64 flags) 10307{ 10308 int err; 10309 10310 err = validate_map_op(map, key_sz, value_sz, true); 10311 if (err) 10312 return libbpf_err(err); 10313 10314 return bpf_map_lookup_elem_flags(map->fd, key, value, flags); 10315} 10316 10317int bpf_map__update_elem(const struct bpf_map *map, 10318 const void *key, size_t key_sz, 10319 const void *value, size_t value_sz, __u64 flags) 10320{ 10321 int err; 10322 10323 err = validate_map_op(map, key_sz, value_sz, true); 10324 if (err) 10325 return libbpf_err(err); 10326 10327 return bpf_map_update_elem(map->fd, key, value, flags); 10328} 10329 10330int bpf_map__delete_elem(const struct bpf_map *map, 10331 const void *key, size_t key_sz, __u64 flags) 10332{ 10333 int err; 10334 10335 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); 10336 if (err) 10337 return libbpf_err(err); 10338 10339 return bpf_map_delete_elem_flags(map->fd, key, flags); 10340} 10341 10342int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, 10343 const void *key, size_t key_sz, 10344 void *value, size_t value_sz, __u64 flags) 10345{ 10346 int err; 10347 10348 err = validate_map_op(map, key_sz, value_sz, true); 10349 if (err) 10350 return libbpf_err(err); 10351 10352 return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags); 10353} 10354 10355int bpf_map__get_next_key(const struct bpf_map *map, 10356 const void *cur_key, void *next_key, size_t key_sz) 10357{ 10358 int err; 10359 10360 err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); 10361 if (err) 10362 return libbpf_err(err); 10363 10364 return bpf_map_get_next_key(map->fd, cur_key, next_key); 10365} 10366 10367long libbpf_get_error(const void *ptr) 10368{ 10369 if (!IS_ERR_OR_NULL(ptr)) 10370 return 0; 10371 10372 if (IS_ERR(ptr)) 10373 errno = -PTR_ERR(ptr); 10374 10375 /* If ptr == NULL, then errno should be already set by the failing 10376 * API, because libbpf never returns NULL on success and it now always 10377 * sets errno on error. So no extra errno handling for ptr == NULL 10378 * case. 10379 */ 10380 return -errno; 10381} 10382 10383/* Replace link's underlying BPF program with the new one */ 10384int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) 10385{ 10386 int ret; 10387 10388 ret = bpf_link_update(bpf_link__fd(link), bpf_program__fd(prog), NULL); 10389 return libbpf_err_errno(ret); 10390} 10391 10392/* Release "ownership" of underlying BPF resource (typically, BPF program 10393 * attached to some BPF hook, e.g., tracepoint, kprobe, etc). Disconnected 10394 * link, when destructed through bpf_link__destroy() call won't attempt to 10395 * detach/unregisted that BPF resource. This is useful in situations where, 10396 * say, attached BPF program has to outlive userspace program that attached it 10397 * in the system. Depending on type of BPF program, though, there might be 10398 * additional steps (like pinning BPF program in BPF FS) necessary to ensure 10399 * exit of userspace program doesn't trigger automatic detachment and clean up 10400 * inside the kernel. 10401 */ 10402void bpf_link__disconnect(struct bpf_link *link) 10403{ 10404 link->disconnected = true; 10405} 10406 10407int bpf_link__destroy(struct bpf_link *link) 10408{ 10409 int err = 0; 10410 10411 if (IS_ERR_OR_NULL(link)) 10412 return 0; 10413 10414 if (!link->disconnected && link->detach) 10415 err = link->detach(link); 10416 if (link->pin_path) 10417 free(link->pin_path); 10418 if (link->dealloc) 10419 link->dealloc(link); 10420 else 10421 free(link); 10422 10423 return libbpf_err(err); 10424} 10425 10426int bpf_link__fd(const struct bpf_link *link) 10427{ 10428 return link->fd; 10429} 10430 10431const char *bpf_link__pin_path(const struct bpf_link *link) 10432{ 10433 return link->pin_path; 10434} 10435 10436static int bpf_link__detach_fd(struct bpf_link *link) 10437{ 10438 return libbpf_err_errno(close(link->fd)); 10439} 10440 10441struct bpf_link *bpf_link__open(const char *path) 10442{ 10443 struct bpf_link *link; 10444 int fd; 10445 10446 fd = bpf_obj_get(path); 10447 if (fd < 0) { 10448 fd = -errno; 10449 pr_warn("failed to open link at %s: %d\n", path, fd); 10450 return libbpf_err_ptr(fd); 10451 } 10452 10453 link = calloc(1, sizeof(*link)); 10454 if (!link) { 10455 close(fd); 10456 return libbpf_err_ptr(-ENOMEM); 10457 } 10458 link->detach = &bpf_link__detach_fd; 10459 link->fd = fd; 10460 10461 link->pin_path = strdup(path); 10462 if (!link->pin_path) { 10463 bpf_link__destroy(link); 10464 return libbpf_err_ptr(-ENOMEM); 10465 } 10466 10467 return link; 10468} 10469 10470int bpf_link__detach(struct bpf_link *link) 10471{ 10472 return bpf_link_detach(link->fd) ? -errno : 0; 10473} 10474 10475int bpf_link__pin(struct bpf_link *link, const char *path) 10476{ 10477 int err; 10478 10479 if (link->pin_path) 10480 return libbpf_err(-EBUSY); 10481 err = make_parent_dir(path); 10482 if (err) 10483 return libbpf_err(err); 10484 err = check_path(path); 10485 if (err) 10486 return libbpf_err(err); 10487 10488 link->pin_path = strdup(path); 10489 if (!link->pin_path) 10490 return libbpf_err(-ENOMEM); 10491 10492 if (bpf_obj_pin(link->fd, link->pin_path)) { 10493 err = -errno; 10494 zfree(&link->pin_path); 10495 return libbpf_err(err); 10496 } 10497 10498 pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path); 10499 return 0; 10500} 10501 10502int bpf_link__unpin(struct bpf_link *link) 10503{ 10504 int err; 10505 10506 if (!link->pin_path) 10507 return libbpf_err(-EINVAL); 10508 10509 err = unlink(link->pin_path); 10510 if (err != 0) 10511 return -errno; 10512 10513 pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path); 10514 zfree(&link->pin_path); 10515 return 0; 10516} 10517 10518struct bpf_link_perf { 10519 struct bpf_link link; 10520 int perf_event_fd; 10521 /* legacy kprobe support: keep track of probe identifier and type */ 10522 char *legacy_probe_name; 10523 bool legacy_is_kprobe; 10524 bool legacy_is_retprobe; 10525}; 10526 10527static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe); 10528static int remove_uprobe_event_legacy(const char *probe_name, bool retprobe); 10529 10530static int bpf_link_perf_detach(struct bpf_link *link) 10531{ 10532 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 10533 int err = 0; 10534 10535 if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0) 10536 err = -errno; 10537 10538 if (perf_link->perf_event_fd != link->fd) 10539 close(perf_link->perf_event_fd); 10540 close(link->fd); 10541 10542 /* legacy uprobe/kprobe needs to be removed after perf event fd closure */ 10543 if (perf_link->legacy_probe_name) { 10544 if (perf_link->legacy_is_kprobe) { 10545 err = remove_kprobe_event_legacy(perf_link->legacy_probe_name, 10546 perf_link->legacy_is_retprobe); 10547 } else { 10548 err = remove_uprobe_event_legacy(perf_link->legacy_probe_name, 10549 perf_link->legacy_is_retprobe); 10550 } 10551 } 10552 10553 return err; 10554} 10555 10556static void bpf_link_perf_dealloc(struct bpf_link *link) 10557{ 10558 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 10559 10560 free(perf_link->legacy_probe_name); 10561 free(perf_link); 10562} 10563 10564struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *prog, int pfd, 10565 const struct bpf_perf_event_opts *opts) 10566{ 10567 char errmsg[STRERR_BUFSIZE]; 10568 struct bpf_link_perf *link; 10569 int prog_fd, link_fd = -1, err; 10570 bool force_ioctl_attach; 10571 10572 if (!OPTS_VALID(opts, bpf_perf_event_opts)) 10573 return libbpf_err_ptr(-EINVAL); 10574 10575 if (pfd < 0) { 10576 pr_warn("prog '%s': invalid perf event FD %d\n", 10577 prog->name, pfd); 10578 return libbpf_err_ptr(-EINVAL); 10579 } 10580 prog_fd = bpf_program__fd(prog); 10581 if (prog_fd < 0) { 10582 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 10583 prog->name); 10584 return libbpf_err_ptr(-EINVAL); 10585 } 10586 10587 link = calloc(1, sizeof(*link)); 10588 if (!link) 10589 return libbpf_err_ptr(-ENOMEM); 10590 link->link.detach = &bpf_link_perf_detach; 10591 link->link.dealloc = &bpf_link_perf_dealloc; 10592 link->perf_event_fd = pfd; 10593 10594 force_ioctl_attach = OPTS_GET(opts, force_ioctl_attach, false); 10595 if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) { 10596 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_opts, 10597 .perf_event.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0)); 10598 10599 link_fd = bpf_link_create(prog_fd, pfd, BPF_PERF_EVENT, &link_opts); 10600 if (link_fd < 0) { 10601 err = -errno; 10602 pr_warn("prog '%s': failed to create BPF link for perf_event FD %d: %d (%s)\n", 10603 prog->name, pfd, 10604 err, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10605 goto err_out; 10606 } 10607 link->link.fd = link_fd; 10608 } else { 10609 if (OPTS_GET(opts, bpf_cookie, 0)) { 10610 pr_warn("prog '%s': user context value is not supported\n", prog->name); 10611 err = -EOPNOTSUPP; 10612 goto err_out; 10613 } 10614 10615 if (ioctl(pfd, PERF_EVENT_IOC_SET_BPF, prog_fd) < 0) { 10616 err = -errno; 10617 pr_warn("prog '%s': failed to attach to perf_event FD %d: %s\n", 10618 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10619 if (err == -EPROTO) 10620 pr_warn("prog '%s': try add PERF_SAMPLE_CALLCHAIN to or remove exclude_callchain_[kernel|user] from pfd %d\n", 10621 prog->name, pfd); 10622 goto err_out; 10623 } 10624 link->link.fd = pfd; 10625 } 10626 if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 10627 err = -errno; 10628 pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", 10629 prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10630 goto err_out; 10631 } 10632 10633 return &link->link; 10634err_out: 10635 if (link_fd >= 0) 10636 close(link_fd); 10637 free(link); 10638 return libbpf_err_ptr(err); 10639} 10640 10641struct bpf_link *bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd) 10642{ 10643 return bpf_program__attach_perf_event_opts(prog, pfd, NULL); 10644} 10645 10646/* 10647 * this function is expected to parse integer in the range of [0, 2^31-1] from 10648 * given file using scanf format string fmt. If actual parsed value is 10649 * negative, the result might be indistinguishable from error 10650 */ 10651static int parse_uint_from_file(const char *file, const char *fmt) 10652{ 10653 char buf[STRERR_BUFSIZE]; 10654 int err, ret; 10655 FILE *f; 10656 10657 f = fopen(file, "re"); 10658 if (!f) { 10659 err = -errno; 10660 pr_debug("failed to open '%s': %s\n", file, 10661 libbpf_strerror_r(err, buf, sizeof(buf))); 10662 return err; 10663 } 10664 err = fscanf(f, fmt, &ret); 10665 if (err != 1) { 10666 err = err == EOF ? -EIO : -errno; 10667 pr_debug("failed to parse '%s': %s\n", file, 10668 libbpf_strerror_r(err, buf, sizeof(buf))); 10669 fclose(f); 10670 return err; 10671 } 10672 fclose(f); 10673 return ret; 10674} 10675 10676static int determine_kprobe_perf_type(void) 10677{ 10678 const char *file = "/sys/bus/event_source/devices/kprobe/type"; 10679 10680 return parse_uint_from_file(file, "%d\n"); 10681} 10682 10683static int determine_uprobe_perf_type(void) 10684{ 10685 const char *file = "/sys/bus/event_source/devices/uprobe/type"; 10686 10687 return parse_uint_from_file(file, "%d\n"); 10688} 10689 10690static int determine_kprobe_retprobe_bit(void) 10691{ 10692 const char *file = "/sys/bus/event_source/devices/kprobe/format/retprobe"; 10693 10694 return parse_uint_from_file(file, "config:%d\n"); 10695} 10696 10697static int determine_uprobe_retprobe_bit(void) 10698{ 10699 const char *file = "/sys/bus/event_source/devices/uprobe/format/retprobe"; 10700 10701 return parse_uint_from_file(file, "config:%d\n"); 10702} 10703 10704#define PERF_UPROBE_REF_CTR_OFFSET_BITS 32 10705#define PERF_UPROBE_REF_CTR_OFFSET_SHIFT 32 10706 10707static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, 10708 uint64_t offset, int pid, size_t ref_ctr_off) 10709{ 10710 const size_t attr_sz = sizeof(struct perf_event_attr); 10711 struct perf_event_attr attr; 10712 char errmsg[STRERR_BUFSIZE]; 10713 int type, pfd; 10714 10715 if ((__u64)ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) 10716 return -EINVAL; 10717 10718 memset(&attr, 0, attr_sz); 10719 10720 type = uprobe ? determine_uprobe_perf_type() 10721 : determine_kprobe_perf_type(); 10722 if (type < 0) { 10723 pr_warn("failed to determine %s perf type: %s\n", 10724 uprobe ? "uprobe" : "kprobe", 10725 libbpf_strerror_r(type, errmsg, sizeof(errmsg))); 10726 return type; 10727 } 10728 if (retprobe) { 10729 int bit = uprobe ? determine_uprobe_retprobe_bit() 10730 : determine_kprobe_retprobe_bit(); 10731 10732 if (bit < 0) { 10733 pr_warn("failed to determine %s retprobe bit: %s\n", 10734 uprobe ? "uprobe" : "kprobe", 10735 libbpf_strerror_r(bit, errmsg, sizeof(errmsg))); 10736 return bit; 10737 } 10738 attr.config |= 1 << bit; 10739 } 10740 attr.size = attr_sz; 10741 attr.type = type; 10742 attr.config |= (__u64)ref_ctr_off << PERF_UPROBE_REF_CTR_OFFSET_SHIFT; 10743 attr.config1 = ptr_to_u64(name); /* kprobe_func or uprobe_path */ 10744 attr.config2 = offset; /* kprobe_addr or probe_offset */ 10745 10746 /* pid filter is meaningful only for uprobes */ 10747 pfd = syscall(__NR_perf_event_open, &attr, 10748 pid < 0 ? -1 : pid /* pid */, 10749 pid == -1 ? 0 : -1 /* cpu */, 10750 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 10751 return pfd >= 0 ? pfd : -errno; 10752} 10753 10754static int append_to_file(const char *file, const char *fmt, ...) 10755{ 10756 int fd, n, err = 0; 10757 va_list ap; 10758 char buf[1024]; 10759 10760 va_start(ap, fmt); 10761 n = vsnprintf(buf, sizeof(buf), fmt, ap); 10762 va_end(ap); 10763 10764 if (n < 0 || n >= sizeof(buf)) 10765 return -EINVAL; 10766 10767 fd = open(file, O_WRONLY | O_APPEND | O_CLOEXEC, 0); 10768 if (fd < 0) 10769 return -errno; 10770 10771 if (write(fd, buf, n) < 0) 10772 err = -errno; 10773 10774 close(fd); 10775 return err; 10776} 10777 10778#define DEBUGFS "/sys/kernel/debug/tracing" 10779#define TRACEFS "/sys/kernel/tracing" 10780 10781static bool use_debugfs(void) 10782{ 10783 static int has_debugfs = -1; 10784 10785 if (has_debugfs < 0) 10786 has_debugfs = faccessat(AT_FDCWD, DEBUGFS, F_OK, AT_EACCESS) == 0; 10787 10788 return has_debugfs == 1; 10789} 10790 10791static const char *tracefs_path(void) 10792{ 10793 return use_debugfs() ? DEBUGFS : TRACEFS; 10794} 10795 10796static const char *tracefs_kprobe_events(void) 10797{ 10798 return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events"; 10799} 10800 10801static const char *tracefs_uprobe_events(void) 10802{ 10803 return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events"; 10804} 10805 10806static const char *tracefs_available_filter_functions(void) 10807{ 10808 return use_debugfs() ? DEBUGFS"/available_filter_functions" 10809 : TRACEFS"/available_filter_functions"; 10810} 10811 10812static const char *tracefs_available_filter_functions_addrs(void) 10813{ 10814 return use_debugfs() ? DEBUGFS"/available_filter_functions_addrs" 10815 : TRACEFS"/available_filter_functions_addrs"; 10816} 10817 10818static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, 10819 const char *kfunc_name, size_t offset) 10820{ 10821 static int index = 0; 10822 int i; 10823 10824 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx_%d", getpid(), kfunc_name, offset, 10825 __sync_fetch_and_add(&index, 1)); 10826 10827 /* sanitize binary_path in the probe name */ 10828 for (i = 0; buf[i]; i++) { 10829 if (!isalnum(buf[i])) 10830 buf[i] = '_'; 10831 } 10832} 10833 10834static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, 10835 const char *kfunc_name, size_t offset) 10836{ 10837 return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx", 10838 retprobe ? 'r' : 'p', 10839 retprobe ? "kretprobes" : "kprobes", 10840 probe_name, kfunc_name, offset); 10841} 10842 10843static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe) 10844{ 10845 return append_to_file(tracefs_kprobe_events(), "-:%s/%s", 10846 retprobe ? "kretprobes" : "kprobes", probe_name); 10847} 10848 10849static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe) 10850{ 10851 char file[256]; 10852 10853 snprintf(file, sizeof(file), "%s/events/%s/%s/id", 10854 tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name); 10855 10856 return parse_uint_from_file(file, "%d\n"); 10857} 10858 10859static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, 10860 const char *kfunc_name, size_t offset, int pid) 10861{ 10862 const size_t attr_sz = sizeof(struct perf_event_attr); 10863 struct perf_event_attr attr; 10864 char errmsg[STRERR_BUFSIZE]; 10865 int type, pfd, err; 10866 10867 err = add_kprobe_event_legacy(probe_name, retprobe, kfunc_name, offset); 10868 if (err < 0) { 10869 pr_warn("failed to add legacy kprobe event for '%s+0x%zx': %s\n", 10870 kfunc_name, offset, 10871 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10872 return err; 10873 } 10874 type = determine_kprobe_perf_type_legacy(probe_name, retprobe); 10875 if (type < 0) { 10876 err = type; 10877 pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n", 10878 kfunc_name, offset, 10879 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10880 goto err_clean_legacy; 10881 } 10882 10883 memset(&attr, 0, attr_sz); 10884 attr.size = attr_sz; 10885 attr.config = type; 10886 attr.type = PERF_TYPE_TRACEPOINT; 10887 10888 pfd = syscall(__NR_perf_event_open, &attr, 10889 pid < 0 ? -1 : pid, /* pid */ 10890 pid == -1 ? 0 : -1, /* cpu */ 10891 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 10892 if (pfd < 0) { 10893 err = -errno; 10894 pr_warn("legacy kprobe perf_event_open() failed: %s\n", 10895 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 10896 goto err_clean_legacy; 10897 } 10898 return pfd; 10899 10900err_clean_legacy: 10901 /* Clear the newly added legacy kprobe_event */ 10902 remove_kprobe_event_legacy(probe_name, retprobe); 10903 return err; 10904} 10905 10906static const char *arch_specific_syscall_pfx(void) 10907{ 10908#if defined(__x86_64__) 10909 return "x64"; 10910#elif defined(__i386__) 10911 return "ia32"; 10912#elif defined(__s390x__) 10913 return "s390x"; 10914#elif defined(__s390__) 10915 return "s390"; 10916#elif defined(__arm__) 10917 return "arm"; 10918#elif defined(__aarch64__) 10919 return "arm64"; 10920#elif defined(__mips__) 10921 return "mips"; 10922#elif defined(__riscv) 10923 return "riscv"; 10924#elif defined(__powerpc__) 10925 return "powerpc"; 10926#elif defined(__powerpc64__) 10927 return "powerpc64"; 10928#else 10929 return NULL; 10930#endif 10931} 10932 10933static int probe_kern_syscall_wrapper(void) 10934{ 10935 char syscall_name[64]; 10936 const char *ksys_pfx; 10937 10938 ksys_pfx = arch_specific_syscall_pfx(); 10939 if (!ksys_pfx) 10940 return 0; 10941 10942 snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx); 10943 10944 if (determine_kprobe_perf_type() >= 0) { 10945 int pfd; 10946 10947 pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0); 10948 if (pfd >= 0) 10949 close(pfd); 10950 10951 return pfd >= 0 ? 1 : 0; 10952 } else { /* legacy mode */ 10953 char probe_name[128]; 10954 10955 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0); 10956 if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0) 10957 return 0; 10958 10959 (void)remove_kprobe_event_legacy(probe_name, false); 10960 return 1; 10961 } 10962} 10963 10964struct bpf_link * 10965bpf_program__attach_kprobe_opts(const struct bpf_program *prog, 10966 const char *func_name, 10967 const struct bpf_kprobe_opts *opts) 10968{ 10969 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 10970 enum probe_attach_mode attach_mode; 10971 char errmsg[STRERR_BUFSIZE]; 10972 char *legacy_probe = NULL; 10973 struct bpf_link *link; 10974 size_t offset; 10975 bool retprobe, legacy; 10976 int pfd, err; 10977 10978 if (!OPTS_VALID(opts, bpf_kprobe_opts)) 10979 return libbpf_err_ptr(-EINVAL); 10980 10981 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); 10982 retprobe = OPTS_GET(opts, retprobe, false); 10983 offset = OPTS_GET(opts, offset, 0); 10984 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 10985 10986 legacy = determine_kprobe_perf_type() < 0; 10987 switch (attach_mode) { 10988 case PROBE_ATTACH_MODE_LEGACY: 10989 legacy = true; 10990 pe_opts.force_ioctl_attach = true; 10991 break; 10992 case PROBE_ATTACH_MODE_PERF: 10993 if (legacy) 10994 return libbpf_err_ptr(-ENOTSUP); 10995 pe_opts.force_ioctl_attach = true; 10996 break; 10997 case PROBE_ATTACH_MODE_LINK: 10998 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) 10999 return libbpf_err_ptr(-ENOTSUP); 11000 break; 11001 case PROBE_ATTACH_MODE_DEFAULT: 11002 break; 11003 default: 11004 return libbpf_err_ptr(-EINVAL); 11005 } 11006 11007 if (!legacy) { 11008 pfd = perf_event_open_probe(false /* uprobe */, retprobe, 11009 func_name, offset, 11010 -1 /* pid */, 0 /* ref_ctr_off */); 11011 } else { 11012 char probe_name[256]; 11013 11014 gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), 11015 func_name, offset); 11016 11017 legacy_probe = strdup(probe_name); 11018 if (!legacy_probe) 11019 return libbpf_err_ptr(-ENOMEM); 11020 11021 pfd = perf_event_kprobe_open_legacy(legacy_probe, retprobe, func_name, 11022 offset, -1 /* pid */); 11023 } 11024 if (pfd < 0) { 11025 err = -errno; 11026 pr_warn("prog '%s': failed to create %s '%s+0x%zx' perf event: %s\n", 11027 prog->name, retprobe ? "kretprobe" : "kprobe", 11028 func_name, offset, 11029 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11030 goto err_out; 11031 } 11032 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 11033 err = libbpf_get_error(link); 11034 if (err) { 11035 close(pfd); 11036 pr_warn("prog '%s': failed to attach to %s '%s+0x%zx': %s\n", 11037 prog->name, retprobe ? "kretprobe" : "kprobe", 11038 func_name, offset, 11039 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11040 goto err_clean_legacy; 11041 } 11042 if (legacy) { 11043 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 11044 11045 perf_link->legacy_probe_name = legacy_probe; 11046 perf_link->legacy_is_kprobe = true; 11047 perf_link->legacy_is_retprobe = retprobe; 11048 } 11049 11050 return link; 11051 11052err_clean_legacy: 11053 if (legacy) 11054 remove_kprobe_event_legacy(legacy_probe, retprobe); 11055err_out: 11056 free(legacy_probe); 11057 return libbpf_err_ptr(err); 11058} 11059 11060struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, 11061 bool retprobe, 11062 const char *func_name) 11063{ 11064 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts, 11065 .retprobe = retprobe, 11066 ); 11067 11068 return bpf_program__attach_kprobe_opts(prog, func_name, &opts); 11069} 11070 11071struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog, 11072 const char *syscall_name, 11073 const struct bpf_ksyscall_opts *opts) 11074{ 11075 LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); 11076 char func_name[128]; 11077 11078 if (!OPTS_VALID(opts, bpf_ksyscall_opts)) 11079 return libbpf_err_ptr(-EINVAL); 11080 11081 if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { 11082 /* arch_specific_syscall_pfx() should never return NULL here 11083 * because it is guarded by kernel_supports(). However, since 11084 * compiler does not know that we have an explicit conditional 11085 * as well. 11086 */ 11087 snprintf(func_name, sizeof(func_name), "__%s_sys_%s", 11088 arch_specific_syscall_pfx() ? : "", syscall_name); 11089 } else { 11090 snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name); 11091 } 11092 11093 kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false); 11094 kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 11095 11096 return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts); 11097} 11098 11099/* Adapted from perf/util/string.c */ 11100bool glob_match(const char *str, const char *pat) 11101{ 11102 while (*str && *pat && *pat != '*') { 11103 if (*pat == '?') { /* Matches any single character */ 11104 str++; 11105 pat++; 11106 continue; 11107 } 11108 if (*str != *pat) 11109 return false; 11110 str++; 11111 pat++; 11112 } 11113 /* Check wild card */ 11114 if (*pat == '*') { 11115 while (*pat == '*') 11116 pat++; 11117 if (!*pat) /* Tail wild card matches all */ 11118 return true; 11119 while (*str) 11120 if (glob_match(str++, pat)) 11121 return true; 11122 } 11123 return !*str && !*pat; 11124} 11125 11126struct kprobe_multi_resolve { 11127 const char *pattern; 11128 unsigned long *addrs; 11129 size_t cap; 11130 size_t cnt; 11131}; 11132 11133struct avail_kallsyms_data { 11134 char **syms; 11135 size_t cnt; 11136 struct kprobe_multi_resolve *res; 11137}; 11138 11139static int avail_func_cmp(const void *a, const void *b) 11140{ 11141 return strcmp(*(const char **)a, *(const char **)b); 11142} 11143 11144static int avail_kallsyms_cb(unsigned long long sym_addr, char sym_type, 11145 const char *sym_name, void *ctx) 11146{ 11147 struct avail_kallsyms_data *data = ctx; 11148 struct kprobe_multi_resolve *res = data->res; 11149 int err; 11150 11151 if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp)) 11152 return 0; 11153 11154 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1); 11155 if (err) 11156 return err; 11157 11158 res->addrs[res->cnt++] = (unsigned long)sym_addr; 11159 return 0; 11160} 11161 11162static int libbpf_available_kallsyms_parse(struct kprobe_multi_resolve *res) 11163{ 11164 const char *available_functions_file = tracefs_available_filter_functions(); 11165 struct avail_kallsyms_data data; 11166 char sym_name[500]; 11167 FILE *f; 11168 int err = 0, ret, i; 11169 char **syms = NULL; 11170 size_t cap = 0, cnt = 0; 11171 11172 f = fopen(available_functions_file, "re"); 11173 if (!f) { 11174 err = -errno; 11175 pr_warn("failed to open %s: %d\n", available_functions_file, err); 11176 return err; 11177 } 11178 11179 while (true) { 11180 char *name; 11181 11182 ret = fscanf(f, "%499s%*[^\n]\n", sym_name); 11183 if (ret == EOF && feof(f)) 11184 break; 11185 11186 if (ret != 1) { 11187 pr_warn("failed to parse available_filter_functions entry: %d\n", ret); 11188 err = -EINVAL; 11189 goto cleanup; 11190 } 11191 11192 if (!glob_match(sym_name, res->pattern)) 11193 continue; 11194 11195 err = libbpf_ensure_mem((void **)&syms, &cap, sizeof(*syms), cnt + 1); 11196 if (err) 11197 goto cleanup; 11198 11199 name = strdup(sym_name); 11200 if (!name) { 11201 err = -errno; 11202 goto cleanup; 11203 } 11204 11205 syms[cnt++] = name; 11206 } 11207 11208 /* no entries found, bail out */ 11209 if (cnt == 0) { 11210 err = -ENOENT; 11211 goto cleanup; 11212 } 11213 11214 /* sort available functions */ 11215 qsort(syms, cnt, sizeof(*syms), avail_func_cmp); 11216 11217 data.syms = syms; 11218 data.res = res; 11219 data.cnt = cnt; 11220 libbpf_kallsyms_parse(avail_kallsyms_cb, &data); 11221 11222 if (res->cnt == 0) 11223 err = -ENOENT; 11224 11225cleanup: 11226 for (i = 0; i < cnt; i++) 11227 free((char *)syms[i]); 11228 free(syms); 11229 11230 fclose(f); 11231 return err; 11232} 11233 11234static bool has_available_filter_functions_addrs(void) 11235{ 11236 return access(tracefs_available_filter_functions_addrs(), R_OK) != -1; 11237} 11238 11239static int libbpf_available_kprobes_parse(struct kprobe_multi_resolve *res) 11240{ 11241 const char *available_path = tracefs_available_filter_functions_addrs(); 11242 char sym_name[500]; 11243 FILE *f; 11244 int ret, err = 0; 11245 unsigned long long sym_addr; 11246 11247 f = fopen(available_path, "re"); 11248 if (!f) { 11249 err = -errno; 11250 pr_warn("failed to open %s: %d\n", available_path, err); 11251 return err; 11252 } 11253 11254 while (true) { 11255 ret = fscanf(f, "%llx %499s%*[^\n]\n", &sym_addr, sym_name); 11256 if (ret == EOF && feof(f)) 11257 break; 11258 11259 if (ret != 2) { 11260 pr_warn("failed to parse available_filter_functions_addrs entry: %d\n", 11261 ret); 11262 err = -EINVAL; 11263 goto cleanup; 11264 } 11265 11266 if (!glob_match(sym_name, res->pattern)) 11267 continue; 11268 11269 err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, 11270 sizeof(*res->addrs), res->cnt + 1); 11271 if (err) 11272 goto cleanup; 11273 11274 res->addrs[res->cnt++] = (unsigned long)sym_addr; 11275 } 11276 11277 if (res->cnt == 0) 11278 err = -ENOENT; 11279 11280cleanup: 11281 fclose(f); 11282 return err; 11283} 11284 11285struct bpf_link * 11286bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, 11287 const char *pattern, 11288 const struct bpf_kprobe_multi_opts *opts) 11289{ 11290 LIBBPF_OPTS(bpf_link_create_opts, lopts); 11291 struct kprobe_multi_resolve res = { 11292 .pattern = pattern, 11293 }; 11294 struct bpf_link *link = NULL; 11295 char errmsg[STRERR_BUFSIZE]; 11296 const unsigned long *addrs; 11297 int err, link_fd, prog_fd; 11298 const __u64 *cookies; 11299 const char **syms; 11300 bool retprobe; 11301 size_t cnt; 11302 11303 if (!OPTS_VALID(opts, bpf_kprobe_multi_opts)) 11304 return libbpf_err_ptr(-EINVAL); 11305 11306 syms = OPTS_GET(opts, syms, false); 11307 addrs = OPTS_GET(opts, addrs, false); 11308 cnt = OPTS_GET(opts, cnt, false); 11309 cookies = OPTS_GET(opts, cookies, false); 11310 11311 if (!pattern && !addrs && !syms) 11312 return libbpf_err_ptr(-EINVAL); 11313 if (pattern && (addrs || syms || cookies || cnt)) 11314 return libbpf_err_ptr(-EINVAL); 11315 if (!pattern && !cnt) 11316 return libbpf_err_ptr(-EINVAL); 11317 if (addrs && syms) 11318 return libbpf_err_ptr(-EINVAL); 11319 11320 if (pattern) { 11321 if (has_available_filter_functions_addrs()) 11322 err = libbpf_available_kprobes_parse(&res); 11323 else 11324 err = libbpf_available_kallsyms_parse(&res); 11325 if (err) 11326 goto error; 11327 addrs = res.addrs; 11328 cnt = res.cnt; 11329 } 11330 11331 retprobe = OPTS_GET(opts, retprobe, false); 11332 11333 lopts.kprobe_multi.syms = syms; 11334 lopts.kprobe_multi.addrs = addrs; 11335 lopts.kprobe_multi.cookies = cookies; 11336 lopts.kprobe_multi.cnt = cnt; 11337 lopts.kprobe_multi.flags = retprobe ? BPF_F_KPROBE_MULTI_RETURN : 0; 11338 11339 link = calloc(1, sizeof(*link)); 11340 if (!link) { 11341 err = -ENOMEM; 11342 goto error; 11343 } 11344 link->detach = &bpf_link__detach_fd; 11345 11346 prog_fd = bpf_program__fd(prog); 11347 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, &lopts); 11348 if (link_fd < 0) { 11349 err = -errno; 11350 pr_warn("prog '%s': failed to attach: %s\n", 11351 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11352 goto error; 11353 } 11354 link->fd = link_fd; 11355 free(res.addrs); 11356 return link; 11357 11358error: 11359 free(link); 11360 free(res.addrs); 11361 return libbpf_err_ptr(err); 11362} 11363 11364static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11365{ 11366 DECLARE_LIBBPF_OPTS(bpf_kprobe_opts, opts); 11367 unsigned long offset = 0; 11368 const char *func_name; 11369 char *func; 11370 int n; 11371 11372 *link = NULL; 11373 11374 /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */ 11375 if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) 11376 return 0; 11377 11378 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); 11379 if (opts.retprobe) 11380 func_name = prog->sec_name + sizeof("kretprobe/") - 1; 11381 else 11382 func_name = prog->sec_name + sizeof("kprobe/") - 1; 11383 11384 n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset); 11385 if (n < 1) { 11386 pr_warn("kprobe name is invalid: %s\n", func_name); 11387 return -EINVAL; 11388 } 11389 if (opts.retprobe && offset != 0) { 11390 free(func); 11391 pr_warn("kretprobes do not support offset specification\n"); 11392 return -EINVAL; 11393 } 11394 11395 opts.offset = offset; 11396 *link = bpf_program__attach_kprobe_opts(prog, func, &opts); 11397 free(func); 11398 return libbpf_get_error(*link); 11399} 11400 11401static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11402{ 11403 LIBBPF_OPTS(bpf_ksyscall_opts, opts); 11404 const char *syscall_name; 11405 11406 *link = NULL; 11407 11408 /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */ 11409 if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) 11410 return 0; 11411 11412 opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); 11413 if (opts.retprobe) 11414 syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; 11415 else 11416 syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; 11417 11418 *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts); 11419 return *link ? 0 : -errno; 11420} 11421 11422static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11423{ 11424 LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); 11425 const char *spec; 11426 char *pattern; 11427 int n; 11428 11429 *link = NULL; 11430 11431 /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */ 11432 if (strcmp(prog->sec_name, "kprobe.multi") == 0 || 11433 strcmp(prog->sec_name, "kretprobe.multi") == 0) 11434 return 0; 11435 11436 opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); 11437 if (opts.retprobe) 11438 spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; 11439 else 11440 spec = prog->sec_name + sizeof("kprobe.multi/") - 1; 11441 11442 n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern); 11443 if (n < 1) { 11444 pr_warn("kprobe multi pattern is invalid: %s\n", pattern); 11445 return -EINVAL; 11446 } 11447 11448 *link = bpf_program__attach_kprobe_multi_opts(prog, pattern, &opts); 11449 free(pattern); 11450 return libbpf_get_error(*link); 11451} 11452 11453static int attach_uprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11454{ 11455 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL; 11456 LIBBPF_OPTS(bpf_uprobe_multi_opts, opts); 11457 int n, ret = -EINVAL; 11458 11459 *link = NULL; 11460 11461 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", 11462 &probe_type, &binary_path, &func_name); 11463 switch (n) { 11464 case 1: 11465 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ 11466 ret = 0; 11467 break; 11468 case 3: 11469 opts.retprobe = strcmp(probe_type, "uretprobe.multi") == 0; 11470 *link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts); 11471 ret = libbpf_get_error(*link); 11472 break; 11473 default: 11474 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, 11475 prog->sec_name); 11476 break; 11477 } 11478 free(probe_type); 11479 free(binary_path); 11480 free(func_name); 11481 return ret; 11482} 11483 11484static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz, 11485 const char *binary_path, uint64_t offset) 11486{ 11487 int i; 11488 11489 snprintf(buf, buf_sz, "libbpf_%u_%s_0x%zx", getpid(), binary_path, (size_t)offset); 11490 11491 /* sanitize binary_path in the probe name */ 11492 for (i = 0; buf[i]; i++) { 11493 if (!isalnum(buf[i])) 11494 buf[i] = '_'; 11495 } 11496} 11497 11498static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, 11499 const char *binary_path, size_t offset) 11500{ 11501 return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx", 11502 retprobe ? 'r' : 'p', 11503 retprobe ? "uretprobes" : "uprobes", 11504 probe_name, binary_path, offset); 11505} 11506 11507static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe) 11508{ 11509 return append_to_file(tracefs_uprobe_events(), "-:%s/%s", 11510 retprobe ? "uretprobes" : "uprobes", probe_name); 11511} 11512 11513static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe) 11514{ 11515 char file[512]; 11516 11517 snprintf(file, sizeof(file), "%s/events/%s/%s/id", 11518 tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name); 11519 11520 return parse_uint_from_file(file, "%d\n"); 11521} 11522 11523static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, 11524 const char *binary_path, size_t offset, int pid) 11525{ 11526 const size_t attr_sz = sizeof(struct perf_event_attr); 11527 struct perf_event_attr attr; 11528 int type, pfd, err; 11529 11530 err = add_uprobe_event_legacy(probe_name, retprobe, binary_path, offset); 11531 if (err < 0) { 11532 pr_warn("failed to add legacy uprobe event for %s:0x%zx: %d\n", 11533 binary_path, (size_t)offset, err); 11534 return err; 11535 } 11536 type = determine_uprobe_perf_type_legacy(probe_name, retprobe); 11537 if (type < 0) { 11538 err = type; 11539 pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n", 11540 binary_path, offset, err); 11541 goto err_clean_legacy; 11542 } 11543 11544 memset(&attr, 0, attr_sz); 11545 attr.size = attr_sz; 11546 attr.config = type; 11547 attr.type = PERF_TYPE_TRACEPOINT; 11548 11549 pfd = syscall(__NR_perf_event_open, &attr, 11550 pid < 0 ? -1 : pid, /* pid */ 11551 pid == -1 ? 0 : -1, /* cpu */ 11552 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 11553 if (pfd < 0) { 11554 err = -errno; 11555 pr_warn("legacy uprobe perf_event_open() failed: %d\n", err); 11556 goto err_clean_legacy; 11557 } 11558 return pfd; 11559 11560err_clean_legacy: 11561 /* Clear the newly added legacy uprobe_event */ 11562 remove_uprobe_event_legacy(probe_name, retprobe); 11563 return err; 11564} 11565 11566/* Find offset of function name in object specified by path. "name" matches 11567 * symbol name or name@@LIB for library functions. 11568 */ 11569static long elf_find_func_offset_from_archive(const char *archive_path, const char *file_name, 11570 const char *func_name) 11571{ 11572 struct zip_archive *archive; 11573 struct zip_entry entry; 11574 long ret; 11575#ifdef HAVE_LIBELF 11576 Elf *elf; 11577#elif defined HAVE_ELFIO 11578 pelfio_t elf; 11579#endif 11580 11581 archive = zip_archive_open(archive_path); 11582 if (IS_ERR(archive)) { 11583 ret = PTR_ERR(archive); 11584 pr_warn("zip: failed to open %s: %ld\n", archive_path, ret); 11585 return ret; 11586 } 11587 11588 ret = zip_archive_find_entry(archive, file_name, &entry); 11589 if (ret) { 11590 pr_warn("zip: could not find archive member %s in %s: %ld\n", file_name, 11591 archive_path, ret); 11592 goto out; 11593 } 11594 pr_debug("zip: found entry for %s in %s at 0x%lx\n", file_name, archive_path, 11595 (unsigned long)entry.data_offset); 11596 11597 if (entry.compression) { 11598 pr_warn("zip: entry %s of %s is compressed and cannot be handled\n", file_name, 11599 archive_path); 11600 ret = -LIBBPF_ERRNO__FORMAT; 11601 goto out; 11602 } 11603#ifdef HAVE_LIBELF 11604 elf = elf_memory((void *)entry.data, entry.data_length); 11605#elif defined HAVE_ELFIO 11606 char memfd_path[PATH_MAX] = {0}; 11607 elf = elfio_new(); 11608 int fdm = syscall(__NR_memfd_create, "bpfelf", MFD_CLOEXEC); 11609 ftruncate(fdm, entry.data_length); 11610 write(fdm, (char *)entry.data, entry.data_length); 11611 snprintf(memfd_path, PATH_MAX, "/proc/self/fd/%d", fdm); 11612 elfio_load(elf, memfd_path); 11613#endif 11614 if (!elf) { 11615 pr_warn("elf: could not read elf file %s from %s: %s\n", file_name, archive_path, 11616 elf_errmsg(-1)); 11617 ret = -LIBBPF_ERRNO__LIBELF; 11618 goto out; 11619 } 11620 11621 ret = elf_find_func_offset(elf, file_name, func_name); 11622 if (ret > 0) { 11623 pr_debug("elf: symbol address match for %s of %s in %s: 0x%x + 0x%lx = 0x%lx\n", 11624 func_name, file_name, archive_path, entry.data_offset, ret, 11625 ret + entry.data_offset); 11626 ret += entry.data_offset; 11627 } 11628#ifdef HAVA_LIBELF 11629 elf_end(elf); 11630#elif HAVA_ELFIO 11631 elfio_delete(elf); 11632#endif 11633out: 11634 zip_archive_close(archive); 11635 return ret; 11636} 11637 11638static const char *arch_specific_lib_paths(void) 11639{ 11640 /* 11641 * Based on https://packages.debian.org/sid/libc6. 11642 * 11643 * Assume that the traced program is built for the same architecture 11644 * as libbpf, which should cover the vast majority of cases. 11645 */ 11646#if defined(__x86_64__) 11647 return "/lib/x86_64-linux-gnu"; 11648#elif defined(__i386__) 11649 return "/lib/i386-linux-gnu"; 11650#elif defined(__s390x__) 11651 return "/lib/s390x-linux-gnu"; 11652#elif defined(__s390__) 11653 return "/lib/s390-linux-gnu"; 11654#elif defined(__arm__) && defined(__SOFTFP__) 11655 return "/lib/arm-linux-gnueabi"; 11656#elif defined(__arm__) && !defined(__SOFTFP__) 11657 return "/lib/arm-linux-gnueabihf"; 11658#elif defined(__aarch64__) 11659 return "/lib/aarch64-linux-gnu"; 11660#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64 11661 return "/lib/mips64el-linux-gnuabi64"; 11662#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32 11663 return "/lib/mipsel-linux-gnu"; 11664#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 11665 return "/lib/powerpc64le-linux-gnu"; 11666#elif defined(__sparc__) && defined(__arch64__) 11667 return "/lib/sparc64-linux-gnu"; 11668#elif defined(__riscv) && __riscv_xlen == 64 11669 return "/lib/riscv64-linux-gnu"; 11670#else 11671 return NULL; 11672#endif 11673} 11674 11675/* Get full path to program/shared library. */ 11676static int resolve_full_path(const char *file, char *result, size_t result_sz) 11677{ 11678 const char *search_paths[3] = {}; 11679 int i, perm; 11680 11681 if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { 11682 search_paths[0] = getenv("LD_LIBRARY_PATH"); 11683 search_paths[1] = "/usr/lib64:/usr/lib"; 11684 search_paths[2] = arch_specific_lib_paths(); 11685 perm = R_OK; 11686 } else { 11687 search_paths[0] = getenv("PATH"); 11688 search_paths[1] = "/usr/bin:/usr/sbin"; 11689 perm = R_OK | X_OK; 11690 } 11691 11692 for (i = 0; i < ARRAY_SIZE(search_paths); i++) { 11693 const char *s; 11694 11695 if (!search_paths[i]) 11696 continue; 11697 for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) { 11698 char *next_path; 11699 int seg_len; 11700 11701 if (s[0] == ':') 11702 s++; 11703 next_path = strchr(s, ':'); 11704 seg_len = next_path ? next_path - s : strlen(s); 11705 if (!seg_len) 11706 continue; 11707 snprintf(result, result_sz, "%.*s/%s", seg_len, s, file); 11708 /* ensure it has required permissions */ 11709 if (faccessat(AT_FDCWD, result, perm, AT_EACCESS) < 0) 11710 continue; 11711 pr_debug("resolved '%s' to '%s'\n", file, result); 11712 return 0; 11713 } 11714 } 11715 return -ENOENT; 11716} 11717 11718struct bpf_link * 11719bpf_program__attach_uprobe_multi(const struct bpf_program *prog, 11720 pid_t pid, 11721 const char *path, 11722 const char *func_pattern, 11723 const struct bpf_uprobe_multi_opts *opts) 11724{ 11725 const unsigned long *ref_ctr_offsets = NULL, *offsets = NULL; 11726 LIBBPF_OPTS(bpf_link_create_opts, lopts); 11727 unsigned long *resolved_offsets = NULL; 11728 int err = 0, link_fd, prog_fd; 11729 struct bpf_link *link = NULL; 11730 char errmsg[STRERR_BUFSIZE]; 11731 char full_path[PATH_MAX]; 11732 const __u64 *cookies; 11733 const char **syms; 11734 size_t cnt; 11735 11736 if (!OPTS_VALID(opts, bpf_uprobe_multi_opts)) 11737 return libbpf_err_ptr(-EINVAL); 11738 11739 syms = OPTS_GET(opts, syms, NULL); 11740 offsets = OPTS_GET(opts, offsets, NULL); 11741 ref_ctr_offsets = OPTS_GET(opts, ref_ctr_offsets, NULL); 11742 cookies = OPTS_GET(opts, cookies, NULL); 11743 cnt = OPTS_GET(opts, cnt, 0); 11744 11745 /* 11746 * User can specify 2 mutually exclusive set of inputs: 11747 * 11748 * 1) use only path/func_pattern/pid arguments 11749 * 11750 * 2) use path/pid with allowed combinations of: 11751 * syms/offsets/ref_ctr_offsets/cookies/cnt 11752 * 11753 * - syms and offsets are mutually exclusive 11754 * - ref_ctr_offsets and cookies are optional 11755 * 11756 * Any other usage results in error. 11757 */ 11758 11759 if (!path) 11760 return libbpf_err_ptr(-EINVAL); 11761 if (!func_pattern && cnt == 0) 11762 return libbpf_err_ptr(-EINVAL); 11763 11764 if (func_pattern) { 11765 if (syms || offsets || ref_ctr_offsets || cookies || cnt) 11766 return libbpf_err_ptr(-EINVAL); 11767 } else { 11768 if (!!syms == !!offsets) 11769 return libbpf_err_ptr(-EINVAL); 11770 } 11771 11772 if (func_pattern) { 11773 if (!strchr(path, '/')) { 11774 err = resolve_full_path(path, full_path, sizeof(full_path)); 11775 if (err) { 11776 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", 11777 prog->name, path, err); 11778 return libbpf_err_ptr(err); 11779 } 11780 path = full_path; 11781 } 11782 11783 err = elf_resolve_pattern_offsets(path, func_pattern, 11784 &resolved_offsets, &cnt); 11785 if (err < 0) 11786 return libbpf_err_ptr(err); 11787 offsets = resolved_offsets; 11788 } else if (syms) { 11789 err = elf_resolve_syms_offsets(path, cnt, syms, &resolved_offsets); 11790 if (err < 0) 11791 return libbpf_err_ptr(err); 11792 offsets = resolved_offsets; 11793 } 11794 11795 lopts.uprobe_multi.path = path; 11796 lopts.uprobe_multi.offsets = offsets; 11797 lopts.uprobe_multi.ref_ctr_offsets = ref_ctr_offsets; 11798 lopts.uprobe_multi.cookies = cookies; 11799 lopts.uprobe_multi.cnt = cnt; 11800 lopts.uprobe_multi.flags = OPTS_GET(opts, retprobe, false) ? BPF_F_UPROBE_MULTI_RETURN : 0; 11801 11802 if (pid == 0) 11803 pid = getpid(); 11804 if (pid > 0) 11805 lopts.uprobe_multi.pid = pid; 11806 11807 link = calloc(1, sizeof(*link)); 11808 if (!link) { 11809 err = -ENOMEM; 11810 goto error; 11811 } 11812 link->detach = &bpf_link__detach_fd; 11813 11814 prog_fd = bpf_program__fd(prog); 11815 link_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_UPROBE_MULTI, &lopts); 11816 if (link_fd < 0) { 11817 err = -errno; 11818 pr_warn("prog '%s': failed to attach multi-uprobe: %s\n", 11819 prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11820 goto error; 11821 } 11822 link->fd = link_fd; 11823 free(resolved_offsets); 11824 return link; 11825 11826error: 11827 free(resolved_offsets); 11828 free(link); 11829 return libbpf_err_ptr(err); 11830} 11831 11832LIBBPF_API struct bpf_link * 11833bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, 11834 const char *binary_path, size_t func_offset, 11835 const struct bpf_uprobe_opts *opts) 11836{ 11837 const char *archive_path = NULL, *archive_sep = NULL; 11838 char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL; 11839 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 11840 enum probe_attach_mode attach_mode; 11841 char full_path[PATH_MAX]; 11842 struct bpf_link *link; 11843 size_t ref_ctr_off; 11844 int pfd, err; 11845 bool retprobe, legacy; 11846 const char *func_name; 11847 11848 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 11849 return libbpf_err_ptr(-EINVAL); 11850 11851 attach_mode = OPTS_GET(opts, attach_mode, PROBE_ATTACH_MODE_DEFAULT); 11852 retprobe = OPTS_GET(opts, retprobe, false); 11853 ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); 11854 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 11855 11856 if (!binary_path) 11857 return libbpf_err_ptr(-EINVAL); 11858 11859 /* Check if "binary_path" refers to an archive. */ 11860 archive_sep = strstr(binary_path, "!/"); 11861 if (archive_sep) { 11862 full_path[0] = '\0'; 11863 libbpf_strlcpy(full_path, binary_path, 11864 min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1))); 11865 archive_path = full_path; 11866 binary_path = archive_sep + 2; 11867 } else if (!strchr(binary_path, '/')) { 11868 err = resolve_full_path(binary_path, full_path, sizeof(full_path)); 11869 if (err) { 11870 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", 11871 prog->name, binary_path, err); 11872 return libbpf_err_ptr(err); 11873 } 11874 binary_path = full_path; 11875 } 11876 func_name = OPTS_GET(opts, func_name, NULL); 11877 if (func_name) { 11878 long sym_off; 11879 11880 if (archive_path) { 11881 sym_off = elf_find_func_offset_from_archive(archive_path, binary_path, 11882 func_name); 11883 binary_path = archive_path; 11884 } else { 11885 sym_off = elf_find_func_offset_from_file(binary_path, func_name); 11886 } 11887 if (sym_off < 0) 11888 return libbpf_err_ptr(sym_off); 11889 func_offset += sym_off; 11890 } 11891 11892 legacy = determine_uprobe_perf_type() < 0; 11893 switch (attach_mode) { 11894 case PROBE_ATTACH_MODE_LEGACY: 11895 legacy = true; 11896 pe_opts.force_ioctl_attach = true; 11897 break; 11898 case PROBE_ATTACH_MODE_PERF: 11899 if (legacy) 11900 return libbpf_err_ptr(-ENOTSUP); 11901 pe_opts.force_ioctl_attach = true; 11902 break; 11903 case PROBE_ATTACH_MODE_LINK: 11904 if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK)) 11905 return libbpf_err_ptr(-ENOTSUP); 11906 break; 11907 case PROBE_ATTACH_MODE_DEFAULT: 11908 break; 11909 default: 11910 return libbpf_err_ptr(-EINVAL); 11911 } 11912 11913 if (!legacy) { 11914 pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, 11915 func_offset, pid, ref_ctr_off); 11916 } else { 11917 char probe_name[PATH_MAX + 64]; 11918 11919 if (ref_ctr_off) 11920 return libbpf_err_ptr(-EINVAL); 11921 11922 gen_uprobe_legacy_event_name(probe_name, sizeof(probe_name), 11923 binary_path, func_offset); 11924 11925 legacy_probe = strdup(probe_name); 11926 if (!legacy_probe) 11927 return libbpf_err_ptr(-ENOMEM); 11928 11929 pfd = perf_event_uprobe_open_legacy(legacy_probe, retprobe, 11930 binary_path, func_offset, pid); 11931 } 11932 if (pfd < 0) { 11933 err = -errno; 11934 pr_warn("prog '%s': failed to create %s '%s:0x%zx' perf event: %s\n", 11935 prog->name, retprobe ? "uretprobe" : "uprobe", 11936 binary_path, func_offset, 11937 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11938 goto err_out; 11939 } 11940 11941 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 11942 err = libbpf_get_error(link); 11943 if (err) { 11944 close(pfd); 11945 pr_warn("prog '%s': failed to attach to %s '%s:0x%zx': %s\n", 11946 prog->name, retprobe ? "uretprobe" : "uprobe", 11947 binary_path, func_offset, 11948 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 11949 goto err_clean_legacy; 11950 } 11951 if (legacy) { 11952 struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); 11953 11954 perf_link->legacy_probe_name = legacy_probe; 11955 perf_link->legacy_is_kprobe = false; 11956 perf_link->legacy_is_retprobe = retprobe; 11957 } 11958 return link; 11959 11960err_clean_legacy: 11961 if (legacy) 11962 remove_uprobe_event_legacy(legacy_probe, retprobe); 11963err_out: 11964 free(legacy_probe); 11965 return libbpf_err_ptr(err); 11966} 11967 11968/* Format of u[ret]probe section definition supporting auto-attach: 11969 * u[ret]probe/binary:function[+offset] 11970 * 11971 * binary can be an absolute/relative path or a filename; the latter is resolved to a 11972 * full binary path via bpf_program__attach_uprobe_opts. 11973 * 11974 * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be 11975 * specified (and auto-attach is not possible) or the above format is specified for 11976 * auto-attach. 11977 */ 11978static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) 11979{ 11980 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts); 11981 char *probe_type = NULL, *binary_path = NULL, *func_name = NULL, *func_off; 11982 int n, c, ret = -EINVAL; 11983 long offset = 0; 11984 11985 *link = NULL; 11986 11987 n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]", 11988 &probe_type, &binary_path, &func_name); 11989 switch (n) { 11990 case 1: 11991 /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ 11992 ret = 0; 11993 break; 11994 case 2: 11995 pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n", 11996 prog->name, prog->sec_name); 11997 break; 11998 case 3: 11999 /* check if user specifies `+offset`, if yes, this should be 12000 * the last part of the string, make sure sscanf read to EOL 12001 */ 12002 func_off = strrchr(func_name, '+'); 12003 if (func_off) { 12004 n = sscanf(func_off, "+%li%n", &offset, &c); 12005 if (n == 1 && *(func_off + c) == '\0') 12006 func_off[0] = '\0'; 12007 else 12008 offset = 0; 12009 } 12010 opts.retprobe = strcmp(probe_type, "uretprobe") == 0 || 12011 strcmp(probe_type, "uretprobe.s") == 0; 12012 if (opts.retprobe && offset != 0) { 12013 pr_warn("prog '%s': uretprobes do not support offset specification\n", 12014 prog->name); 12015 break; 12016 } 12017 opts.func_name = func_name; 12018 *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); 12019 ret = libbpf_get_error(*link); 12020 break; 12021 default: 12022 pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, 12023 prog->sec_name); 12024 break; 12025 } 12026 free(probe_type); 12027 free(binary_path); 12028 free(func_name); 12029 12030 return ret; 12031} 12032 12033struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, 12034 bool retprobe, pid_t pid, 12035 const char *binary_path, 12036 size_t func_offset) 12037{ 12038 DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts, .retprobe = retprobe); 12039 12040 return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); 12041} 12042 12043#ifdef HAVE_LIBELF 12044struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, 12045 pid_t pid, const char *binary_path, 12046 const char *usdt_provider, const char *usdt_name, 12047 const struct bpf_usdt_opts *opts) 12048{ 12049 char resolved_path[512]; 12050 struct bpf_object *obj = prog->obj; 12051 struct bpf_link *link; 12052 __u64 usdt_cookie; 12053 int err; 12054 12055 if (!OPTS_VALID(opts, bpf_uprobe_opts)) 12056 return libbpf_err_ptr(-EINVAL); 12057 12058 if (bpf_program__fd(prog) < 0) { 12059 pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", 12060 prog->name); 12061 return libbpf_err_ptr(-EINVAL); 12062 } 12063 12064 if (!binary_path) 12065 return libbpf_err_ptr(-EINVAL); 12066 12067 if (!strchr(binary_path, '/')) { 12068 err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path)); 12069 if (err) { 12070 pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", 12071 prog->name, binary_path, err); 12072 return libbpf_err_ptr(err); 12073 } 12074 binary_path = resolved_path; 12075 } 12076 12077 /* USDT manager is instantiated lazily on first USDT attach. It will 12078 * be destroyed together with BPF object in bpf_object__close(). 12079 */ 12080 if (IS_ERR(obj->usdt_man)) 12081 return libbpf_ptr(obj->usdt_man); 12082 if (!obj->usdt_man) { 12083 obj->usdt_man = usdt_manager_new(obj); 12084 if (IS_ERR(obj->usdt_man)) 12085 return libbpf_ptr(obj->usdt_man); 12086 } 12087 12088 usdt_cookie = OPTS_GET(opts, usdt_cookie, 0); 12089 link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, 12090 usdt_provider, usdt_name, usdt_cookie); 12091 err = libbpf_get_error(link); 12092 if (err) 12093 return libbpf_err_ptr(err); 12094 return link; 12095} 12096#endif //HAVE_LIBELF 12097 12098#ifdef HAVE_LIBELF 12099static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12100{ 12101 char *path = NULL, *provider = NULL, *name = NULL; 12102 const char *sec_name; 12103 int n, err; 12104 12105 sec_name = bpf_program__section_name(prog); 12106 if (strcmp(sec_name, "usdt") == 0) { 12107 /* no auto-attach for just SEC("usdt") */ 12108 *link = NULL; 12109 return 0; 12110 } 12111 12112 n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name); 12113 if (n != 3) { 12114 pr_warn("invalid section '%s', expected SEC(\"usdt/<path>:<provider>:<name>\")\n", 12115 sec_name); 12116 err = -EINVAL; 12117 } else { 12118 *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, 12119 provider, name, NULL); 12120 err = libbpf_get_error(*link); 12121 } 12122 free(path); 12123 free(provider); 12124 free(name); 12125 return err; 12126} 12127#endif //HAVE_LIBELF 12128 12129static int determine_tracepoint_id(const char *tp_category, 12130 const char *tp_name) 12131{ 12132 char file[PATH_MAX]; 12133 int ret; 12134 12135 ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id", 12136 tracefs_path(), tp_category, tp_name); 12137 if (ret < 0) 12138 return -errno; 12139 if (ret >= sizeof(file)) { 12140 pr_debug("tracepoint %s/%s path is too long\n", 12141 tp_category, tp_name); 12142 return -E2BIG; 12143 } 12144 return parse_uint_from_file(file, "%d\n"); 12145} 12146 12147static int perf_event_open_tracepoint(const char *tp_category, 12148 const char *tp_name) 12149{ 12150 const size_t attr_sz = sizeof(struct perf_event_attr); 12151 struct perf_event_attr attr; 12152 char errmsg[STRERR_BUFSIZE]; 12153 int tp_id, pfd, err; 12154 12155 tp_id = determine_tracepoint_id(tp_category, tp_name); 12156 if (tp_id < 0) { 12157 pr_warn("failed to determine tracepoint '%s/%s' perf event ID: %s\n", 12158 tp_category, tp_name, 12159 libbpf_strerror_r(tp_id, errmsg, sizeof(errmsg))); 12160 return tp_id; 12161 } 12162 12163 memset(&attr, 0, attr_sz); 12164 attr.type = PERF_TYPE_TRACEPOINT; 12165 attr.size = attr_sz; 12166 attr.config = tp_id; 12167 12168 pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */, 12169 -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); 12170 if (pfd < 0) { 12171 err = -errno; 12172 pr_warn("tracepoint '%s/%s' perf_event_open() failed: %s\n", 12173 tp_category, tp_name, 12174 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 12175 return err; 12176 } 12177 return pfd; 12178} 12179 12180struct bpf_link *bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, 12181 const char *tp_category, 12182 const char *tp_name, 12183 const struct bpf_tracepoint_opts *opts) 12184{ 12185 DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); 12186 char errmsg[STRERR_BUFSIZE]; 12187 struct bpf_link *link; 12188 int pfd, err; 12189 12190 if (!OPTS_VALID(opts, bpf_tracepoint_opts)) 12191 return libbpf_err_ptr(-EINVAL); 12192 12193 pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); 12194 12195 pfd = perf_event_open_tracepoint(tp_category, tp_name); 12196 if (pfd < 0) { 12197 pr_warn("prog '%s': failed to create tracepoint '%s/%s' perf event: %s\n", 12198 prog->name, tp_category, tp_name, 12199 libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 12200 return libbpf_err_ptr(pfd); 12201 } 12202 link = bpf_program__attach_perf_event_opts(prog, pfd, &pe_opts); 12203 err = libbpf_get_error(link); 12204 if (err) { 12205 close(pfd); 12206 pr_warn("prog '%s': failed to attach to tracepoint '%s/%s': %s\n", 12207 prog->name, tp_category, tp_name, 12208 libbpf_strerror_r(err, errmsg, sizeof(errmsg))); 12209 return libbpf_err_ptr(err); 12210 } 12211 return link; 12212} 12213 12214struct bpf_link *bpf_program__attach_tracepoint(const struct bpf_program *prog, 12215 const char *tp_category, 12216 const char *tp_name) 12217{ 12218 return bpf_program__attach_tracepoint_opts(prog, tp_category, tp_name, NULL); 12219} 12220 12221static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12222{ 12223 char *sec_name, *tp_cat, *tp_name; 12224 12225 *link = NULL; 12226 12227 /* no auto-attach for SEC("tp") or SEC("tracepoint") */ 12228 if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) 12229 return 0; 12230 12231 sec_name = strdup(prog->sec_name); 12232 if (!sec_name) 12233 return -ENOMEM; 12234 12235 /* extract "tp/<category>/<name>" or "tracepoint/<category>/<name>" */ 12236 if (str_has_pfx(prog->sec_name, "tp/")) 12237 tp_cat = sec_name + sizeof("tp/") - 1; 12238 else 12239 tp_cat = sec_name + sizeof("tracepoint/") - 1; 12240 tp_name = strchr(tp_cat, '/'); 12241 if (!tp_name) { 12242 free(sec_name); 12243 return -EINVAL; 12244 } 12245 *tp_name = '\0'; 12246 tp_name++; 12247 12248 *link = bpf_program__attach_tracepoint(prog, tp_cat, tp_name); 12249 free(sec_name); 12250 return libbpf_get_error(*link); 12251} 12252 12253struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, 12254 const char *tp_name) 12255{ 12256 char errmsg[STRERR_BUFSIZE]; 12257 struct bpf_link *link; 12258 int prog_fd, pfd; 12259 12260 prog_fd = bpf_program__fd(prog); 12261 if (prog_fd < 0) { 12262 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12263 return libbpf_err_ptr(-EINVAL); 12264 } 12265 12266 link = calloc(1, sizeof(*link)); 12267 if (!link) 12268 return libbpf_err_ptr(-ENOMEM); 12269 link->detach = &bpf_link__detach_fd; 12270 12271 pfd = bpf_raw_tracepoint_open(tp_name, prog_fd); 12272 if (pfd < 0) { 12273 pfd = -errno; 12274 free(link); 12275 pr_warn("prog '%s': failed to attach to raw tracepoint '%s': %s\n", 12276 prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 12277 return libbpf_err_ptr(pfd); 12278 } 12279 link->fd = pfd; 12280 return link; 12281} 12282 12283static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12284{ 12285 static const char *const prefixes[] = { 12286 "raw_tp", 12287 "raw_tracepoint", 12288 "raw_tp.w", 12289 "raw_tracepoint.w", 12290 }; 12291 size_t i; 12292 const char *tp_name = NULL; 12293 12294 *link = NULL; 12295 12296 for (i = 0; i < ARRAY_SIZE(prefixes); i++) { 12297 size_t pfx_len; 12298 12299 if (!str_has_pfx(prog->sec_name, prefixes[i])) 12300 continue; 12301 12302 pfx_len = strlen(prefixes[i]); 12303 /* no auto-attach case of, e.g., SEC("raw_tp") */ 12304 if (prog->sec_name[pfx_len] == '\0') 12305 return 0; 12306 12307 if (prog->sec_name[pfx_len] != '/') 12308 continue; 12309 12310 tp_name = prog->sec_name + pfx_len + 1; 12311 break; 12312 } 12313 12314 if (!tp_name) { 12315 pr_warn("prog '%s': invalid section name '%s'\n", 12316 prog->name, prog->sec_name); 12317 return -EINVAL; 12318 } 12319 12320 *link = bpf_program__attach_raw_tracepoint(prog, tp_name); 12321 return libbpf_get_error(*link); 12322} 12323 12324/* Common logic for all BPF program types that attach to a btf_id */ 12325static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog, 12326 const struct bpf_trace_opts *opts) 12327{ 12328 LIBBPF_OPTS(bpf_link_create_opts, link_opts); 12329 char errmsg[STRERR_BUFSIZE]; 12330 struct bpf_link *link; 12331 int prog_fd, pfd; 12332 12333 if (!OPTS_VALID(opts, bpf_trace_opts)) 12334 return libbpf_err_ptr(-EINVAL); 12335 12336 prog_fd = bpf_program__fd(prog); 12337 if (prog_fd < 0) { 12338 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12339 return libbpf_err_ptr(-EINVAL); 12340 } 12341 12342 link = calloc(1, sizeof(*link)); 12343 if (!link) 12344 return libbpf_err_ptr(-ENOMEM); 12345 link->detach = &bpf_link__detach_fd; 12346 12347 /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */ 12348 link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0); 12349 pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts); 12350 if (pfd < 0) { 12351 pfd = -errno; 12352 free(link); 12353 pr_warn("prog '%s': failed to attach: %s\n", 12354 prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg))); 12355 return libbpf_err_ptr(pfd); 12356 } 12357 link->fd = pfd; 12358 return link; 12359} 12360 12361struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) 12362{ 12363 return bpf_program__attach_btf_id(prog, NULL); 12364} 12365 12366struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog, 12367 const struct bpf_trace_opts *opts) 12368{ 12369 return bpf_program__attach_btf_id(prog, opts); 12370} 12371 12372struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog) 12373{ 12374 return bpf_program__attach_btf_id(prog, NULL); 12375} 12376 12377static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12378{ 12379 *link = bpf_program__attach_trace(prog); 12380 return libbpf_get_error(*link); 12381} 12382 12383static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12384{ 12385 *link = bpf_program__attach_lsm(prog); 12386 return libbpf_get_error(*link); 12387} 12388 12389static struct bpf_link * 12390bpf_program_attach_fd(const struct bpf_program *prog, 12391 int target_fd, const char *target_name, 12392 const struct bpf_link_create_opts *opts) 12393{ 12394 enum bpf_attach_type attach_type; 12395 char errmsg[STRERR_BUFSIZE]; 12396 struct bpf_link *link; 12397 int prog_fd, link_fd; 12398 12399 prog_fd = bpf_program__fd(prog); 12400 if (prog_fd < 0) { 12401 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12402 return libbpf_err_ptr(-EINVAL); 12403 } 12404 12405 link = calloc(1, sizeof(*link)); 12406 if (!link) 12407 return libbpf_err_ptr(-ENOMEM); 12408 link->detach = &bpf_link__detach_fd; 12409 12410 attach_type = bpf_program__expected_attach_type(prog); 12411 link_fd = bpf_link_create(prog_fd, target_fd, attach_type, opts); 12412 if (link_fd < 0) { 12413 link_fd = -errno; 12414 free(link); 12415 pr_warn("prog '%s': failed to attach to %s: %s\n", 12416 prog->name, target_name, 12417 libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 12418 return libbpf_err_ptr(link_fd); 12419 } 12420 link->fd = link_fd; 12421 return link; 12422} 12423 12424struct bpf_link * 12425bpf_program__attach_cgroup(const struct bpf_program *prog, int cgroup_fd) 12426{ 12427 return bpf_program_attach_fd(prog, cgroup_fd, "cgroup", NULL); 12428} 12429 12430struct bpf_link * 12431bpf_program__attach_netns(const struct bpf_program *prog, int netns_fd) 12432{ 12433 return bpf_program_attach_fd(prog, netns_fd, "netns", NULL); 12434} 12435 12436struct bpf_link *bpf_program__attach_xdp(const struct bpf_program *prog, int ifindex) 12437{ 12438 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 12439 return bpf_program_attach_fd(prog, ifindex, "xdp", NULL); 12440} 12441 12442struct bpf_link * 12443bpf_program__attach_tcx(const struct bpf_program *prog, int ifindex, 12444 const struct bpf_tcx_opts *opts) 12445{ 12446 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 12447 __u32 relative_id; 12448 int relative_fd; 12449 12450 if (!OPTS_VALID(opts, bpf_tcx_opts)) 12451 return libbpf_err_ptr(-EINVAL); 12452 12453 relative_id = OPTS_GET(opts, relative_id, 0); 12454 relative_fd = OPTS_GET(opts, relative_fd, 0); 12455 12456 /* validate we don't have unexpected combinations of non-zero fields */ 12457 if (!ifindex) { 12458 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", 12459 prog->name); 12460 return libbpf_err_ptr(-EINVAL); 12461 } 12462 if (relative_fd && relative_id) { 12463 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", 12464 prog->name); 12465 return libbpf_err_ptr(-EINVAL); 12466 } 12467 12468 link_create_opts.tcx.expected_revision = OPTS_GET(opts, expected_revision, 0); 12469 link_create_opts.tcx.relative_fd = relative_fd; 12470 link_create_opts.tcx.relative_id = relative_id; 12471 link_create_opts.flags = OPTS_GET(opts, flags, 0); 12472 12473 /* target_fd/target_ifindex use the same field in LINK_CREATE */ 12474 return bpf_program_attach_fd(prog, ifindex, "tcx", &link_create_opts); 12475} 12476 12477struct bpf_link * 12478bpf_program__attach_netkit(const struct bpf_program *prog, int ifindex, 12479 const struct bpf_netkit_opts *opts) 12480{ 12481 LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 12482 __u32 relative_id; 12483 int relative_fd; 12484 12485 if (!OPTS_VALID(opts, bpf_netkit_opts)) 12486 return libbpf_err_ptr(-EINVAL); 12487 12488 relative_id = OPTS_GET(opts, relative_id, 0); 12489 relative_fd = OPTS_GET(opts, relative_fd, 0); 12490 12491 /* validate we don't have unexpected combinations of non-zero fields */ 12492 if (!ifindex) { 12493 pr_warn("prog '%s': target netdevice ifindex cannot be zero\n", 12494 prog->name); 12495 return libbpf_err_ptr(-EINVAL); 12496 } 12497 if (relative_fd && relative_id) { 12498 pr_warn("prog '%s': relative_fd and relative_id cannot be set at the same time\n", 12499 prog->name); 12500 return libbpf_err_ptr(-EINVAL); 12501 } 12502 12503 link_create_opts.netkit.expected_revision = OPTS_GET(opts, expected_revision, 0); 12504 link_create_opts.netkit.relative_fd = relative_fd; 12505 link_create_opts.netkit.relative_id = relative_id; 12506 link_create_opts.flags = OPTS_GET(opts, flags, 0); 12507 12508 return bpf_program_attach_fd(prog, ifindex, "netkit", &link_create_opts); 12509} 12510 12511struct bpf_link *bpf_program__attach_freplace(const struct bpf_program *prog, 12512 int target_fd, 12513 const char *attach_func_name) 12514{ 12515 int btf_id; 12516 12517 if (!!target_fd != !!attach_func_name) { 12518 pr_warn("prog '%s': supply none or both of target_fd and attach_func_name\n", 12519 prog->name); 12520 return libbpf_err_ptr(-EINVAL); 12521 } 12522 12523 if (prog->type != BPF_PROG_TYPE_EXT) { 12524 pr_warn("prog '%s': only BPF_PROG_TYPE_EXT can attach as freplace", 12525 prog->name); 12526 return libbpf_err_ptr(-EINVAL); 12527 } 12528 12529 if (target_fd) { 12530 LIBBPF_OPTS(bpf_link_create_opts, target_opts); 12531 12532 btf_id = libbpf_find_prog_btf_id(attach_func_name, target_fd); 12533 if (btf_id < 0) 12534 return libbpf_err_ptr(btf_id); 12535 12536 target_opts.target_btf_id = btf_id; 12537 12538 return bpf_program_attach_fd(prog, target_fd, "freplace", 12539 &target_opts); 12540 } else { 12541 /* no target, so use raw_tracepoint_open for compatibility 12542 * with old kernels 12543 */ 12544 return bpf_program__attach_trace(prog); 12545 } 12546} 12547 12548struct bpf_link * 12549bpf_program__attach_iter(const struct bpf_program *prog, 12550 const struct bpf_iter_attach_opts *opts) 12551{ 12552 DECLARE_LIBBPF_OPTS(bpf_link_create_opts, link_create_opts); 12553 char errmsg[STRERR_BUFSIZE]; 12554 struct bpf_link *link; 12555 int prog_fd, link_fd; 12556 __u32 target_fd = 0; 12557 12558 if (!OPTS_VALID(opts, bpf_iter_attach_opts)) 12559 return libbpf_err_ptr(-EINVAL); 12560 12561 link_create_opts.iter_info = OPTS_GET(opts, link_info, (void *)0); 12562 link_create_opts.iter_info_len = OPTS_GET(opts, link_info_len, 0); 12563 12564 prog_fd = bpf_program__fd(prog); 12565 if (prog_fd < 0) { 12566 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12567 return libbpf_err_ptr(-EINVAL); 12568 } 12569 12570 link = calloc(1, sizeof(*link)); 12571 if (!link) 12572 return libbpf_err_ptr(-ENOMEM); 12573 link->detach = &bpf_link__detach_fd; 12574 12575 link_fd = bpf_link_create(prog_fd, target_fd, BPF_TRACE_ITER, 12576 &link_create_opts); 12577 if (link_fd < 0) { 12578 link_fd = -errno; 12579 free(link); 12580 pr_warn("prog '%s': failed to attach to iterator: %s\n", 12581 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 12582 return libbpf_err_ptr(link_fd); 12583 } 12584 link->fd = link_fd; 12585 return link; 12586} 12587 12588static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link) 12589{ 12590 *link = bpf_program__attach_iter(prog, NULL); 12591 return libbpf_get_error(*link); 12592} 12593 12594struct bpf_link *bpf_program__attach_netfilter(const struct bpf_program *prog, 12595 const struct bpf_netfilter_opts *opts) 12596{ 12597 LIBBPF_OPTS(bpf_link_create_opts, lopts); 12598 struct bpf_link *link; 12599 int prog_fd, link_fd; 12600 12601 if (!OPTS_VALID(opts, bpf_netfilter_opts)) 12602 return libbpf_err_ptr(-EINVAL); 12603 12604 prog_fd = bpf_program__fd(prog); 12605 if (prog_fd < 0) { 12606 pr_warn("prog '%s': can't attach before loaded\n", prog->name); 12607 return libbpf_err_ptr(-EINVAL); 12608 } 12609 12610 link = calloc(1, sizeof(*link)); 12611 if (!link) 12612 return libbpf_err_ptr(-ENOMEM); 12613 12614 link->detach = &bpf_link__detach_fd; 12615 12616 lopts.netfilter.pf = OPTS_GET(opts, pf, 0); 12617 lopts.netfilter.hooknum = OPTS_GET(opts, hooknum, 0); 12618 lopts.netfilter.priority = OPTS_GET(opts, priority, 0); 12619 lopts.netfilter.flags = OPTS_GET(opts, flags, 0); 12620 12621 link_fd = bpf_link_create(prog_fd, 0, BPF_NETFILTER, &lopts); 12622 if (link_fd < 0) { 12623 char errmsg[STRERR_BUFSIZE]; 12624 12625 link_fd = -errno; 12626 free(link); 12627 pr_warn("prog '%s': failed to attach to netfilter: %s\n", 12628 prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg))); 12629 return libbpf_err_ptr(link_fd); 12630 } 12631 link->fd = link_fd; 12632 12633 return link; 12634} 12635 12636struct bpf_link *bpf_program__attach(const struct bpf_program *prog) 12637{ 12638 struct bpf_link *link = NULL; 12639 int err; 12640 12641 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) 12642 return libbpf_err_ptr(-EOPNOTSUPP); 12643 12644 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link); 12645 if (err) 12646 return libbpf_err_ptr(err); 12647 12648 /* When calling bpf_program__attach() explicitly, auto-attach support 12649 * is expected to work, so NULL returned link is considered an error. 12650 * This is different for skeleton's attach, see comment in 12651 * bpf_object__attach_skeleton(). 12652 */ 12653 if (!link) 12654 return libbpf_err_ptr(-EOPNOTSUPP); 12655 12656 return link; 12657} 12658 12659struct bpf_link_struct_ops { 12660 struct bpf_link link; 12661 int map_fd; 12662}; 12663 12664static int bpf_link__detach_struct_ops(struct bpf_link *link) 12665{ 12666 struct bpf_link_struct_ops *st_link; 12667 __u32 zero = 0; 12668 12669 st_link = container_of(link, struct bpf_link_struct_ops, link); 12670 12671 if (st_link->map_fd < 0) 12672 /* w/o a real link */ 12673 return bpf_map_delete_elem(link->fd, &zero); 12674 12675 return close(link->fd); 12676} 12677 12678struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) 12679{ 12680 struct bpf_link_struct_ops *link; 12681 __u32 zero = 0; 12682 int err, fd; 12683 12684 if (!bpf_map__is_struct_ops(map) || map->fd == -1) 12685 return libbpf_err_ptr(-EINVAL); 12686 12687 link = calloc(1, sizeof(*link)); 12688 if (!link) 12689 return libbpf_err_ptr(-EINVAL); 12690 12691 /* kern_vdata should be prepared during the loading phase. */ 12692 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); 12693 /* It can be EBUSY if the map has been used to create or 12694 * update a link before. We don't allow updating the value of 12695 * a struct_ops once it is set. That ensures that the value 12696 * never changed. So, it is safe to skip EBUSY. 12697 */ 12698 if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) { 12699 free(link); 12700 return libbpf_err_ptr(err); 12701 } 12702 12703 link->link.detach = bpf_link__detach_struct_ops; 12704 12705 if (!(map->def.map_flags & BPF_F_LINK)) { 12706 /* w/o a real link */ 12707 link->link.fd = map->fd; 12708 link->map_fd = -1; 12709 return &link->link; 12710 } 12711 12712 fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL); 12713 if (fd < 0) { 12714 free(link); 12715 return libbpf_err_ptr(fd); 12716 } 12717 12718 link->link.fd = fd; 12719 link->map_fd = map->fd; 12720 12721 return &link->link; 12722} 12723 12724/* 12725 * Swap the back struct_ops of a link with a new struct_ops map. 12726 */ 12727int bpf_link__update_map(struct bpf_link *link, const struct bpf_map *map) 12728{ 12729 struct bpf_link_struct_ops *st_ops_link; 12730 __u32 zero = 0; 12731 int err; 12732 12733 if (!bpf_map__is_struct_ops(map) || map->fd < 0) 12734 return -EINVAL; 12735 12736 st_ops_link = container_of(link, struct bpf_link_struct_ops, link); 12737 /* Ensure the type of a link is correct */ 12738 if (st_ops_link->map_fd < 0) 12739 return -EINVAL; 12740 12741 err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0); 12742 /* It can be EBUSY if the map has been used to create or 12743 * update a link before. We don't allow updating the value of 12744 * a struct_ops once it is set. That ensures that the value 12745 * never changed. So, it is safe to skip EBUSY. 12746 */ 12747 if (err && err != -EBUSY) 12748 return err; 12749 12750 err = bpf_link_update(link->fd, map->fd, NULL); 12751 if (err < 0) 12752 return err; 12753 12754 st_ops_link->map_fd = map->fd; 12755 12756 return 0; 12757} 12758 12759typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr, 12760 void *private_data); 12761 12762static enum bpf_perf_event_ret 12763perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, 12764 void **copy_mem, size_t *copy_size, 12765 bpf_perf_event_print_t fn, void *private_data) 12766{ 12767 struct perf_event_mmap_page *header = mmap_mem; 12768 __u64 data_head = ring_buffer_read_head(header); 12769 __u64 data_tail = header->data_tail; 12770 void *base = ((__u8 *)header) + page_size; 12771 int ret = LIBBPF_PERF_EVENT_CONT; 12772 struct perf_event_header *ehdr; 12773 size_t ehdr_size; 12774 12775 while (data_head != data_tail) { 12776 ehdr = base + (data_tail & (mmap_size - 1)); 12777 ehdr_size = ehdr->size; 12778 12779 if (((void *)ehdr) + ehdr_size > base + mmap_size) { 12780 void *copy_start = ehdr; 12781 size_t len_first = base + mmap_size - copy_start; 12782 size_t len_secnd = ehdr_size - len_first; 12783 12784 if (*copy_size < ehdr_size) { 12785 free(*copy_mem); 12786 *copy_mem = malloc(ehdr_size); 12787 if (!*copy_mem) { 12788 *copy_size = 0; 12789 ret = LIBBPF_PERF_EVENT_ERROR; 12790 break; 12791 } 12792 *copy_size = ehdr_size; 12793 } 12794 12795 memcpy(*copy_mem, copy_start, len_first); 12796 memcpy(*copy_mem + len_first, base, len_secnd); 12797 ehdr = *copy_mem; 12798 } 12799 12800 ret = fn(ehdr, private_data); 12801 data_tail += ehdr_size; 12802 if (ret != LIBBPF_PERF_EVENT_CONT) 12803 break; 12804 } 12805 12806 ring_buffer_write_tail(header, data_tail); 12807 return libbpf_err(ret); 12808} 12809 12810struct perf_buffer; 12811 12812struct perf_buffer_params { 12813 struct perf_event_attr *attr; 12814 /* if event_cb is specified, it takes precendence */ 12815 perf_buffer_event_fn event_cb; 12816 /* sample_cb and lost_cb are higher-level common-case callbacks */ 12817 perf_buffer_sample_fn sample_cb; 12818 perf_buffer_lost_fn lost_cb; 12819 void *ctx; 12820 int cpu_cnt; 12821 int *cpus; 12822 int *map_keys; 12823}; 12824 12825struct perf_cpu_buf { 12826 struct perf_buffer *pb; 12827 void *base; /* mmap()'ed memory */ 12828 void *buf; /* for reconstructing segmented data */ 12829 size_t buf_size; 12830 int fd; 12831 int cpu; 12832 int map_key; 12833}; 12834 12835struct perf_buffer { 12836 perf_buffer_event_fn event_cb; 12837 perf_buffer_sample_fn sample_cb; 12838 perf_buffer_lost_fn lost_cb; 12839 void *ctx; /* passed into callbacks */ 12840 12841 size_t page_size; 12842 size_t mmap_size; 12843 struct perf_cpu_buf **cpu_bufs; 12844 struct epoll_event *events; 12845 int cpu_cnt; /* number of allocated CPU buffers */ 12846 int epoll_fd; /* perf event FD */ 12847 int map_fd; /* BPF_MAP_TYPE_PERF_EVENT_ARRAY BPF map FD */ 12848}; 12849 12850static void perf_buffer__free_cpu_buf(struct perf_buffer *pb, 12851 struct perf_cpu_buf *cpu_buf) 12852{ 12853 if (!cpu_buf) 12854 return; 12855 if (cpu_buf->base && 12856 munmap(cpu_buf->base, pb->mmap_size + pb->page_size)) 12857 pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu); 12858 if (cpu_buf->fd >= 0) { 12859 ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0); 12860 close(cpu_buf->fd); 12861 } 12862 free(cpu_buf->buf); 12863 free(cpu_buf); 12864} 12865 12866void perf_buffer__free(struct perf_buffer *pb) 12867{ 12868 int i; 12869 12870 if (IS_ERR_OR_NULL(pb)) 12871 return; 12872 if (pb->cpu_bufs) { 12873 for (i = 0; i < pb->cpu_cnt; i++) { 12874 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 12875 12876 if (!cpu_buf) 12877 continue; 12878 12879 bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key); 12880 perf_buffer__free_cpu_buf(pb, cpu_buf); 12881 } 12882 free(pb->cpu_bufs); 12883 } 12884 if (pb->epoll_fd >= 0) 12885 close(pb->epoll_fd); 12886 free(pb->events); 12887 free(pb); 12888} 12889 12890static struct perf_cpu_buf * 12891perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, 12892 int cpu, int map_key) 12893{ 12894 struct perf_cpu_buf *cpu_buf; 12895 char msg[STRERR_BUFSIZE]; 12896 int err; 12897 12898 cpu_buf = calloc(1, sizeof(*cpu_buf)); 12899 if (!cpu_buf) 12900 return ERR_PTR(-ENOMEM); 12901 12902 cpu_buf->pb = pb; 12903 cpu_buf->cpu = cpu; 12904 cpu_buf->map_key = map_key; 12905 12906 cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu, 12907 -1, PERF_FLAG_FD_CLOEXEC); 12908 if (cpu_buf->fd < 0) { 12909 err = -errno; 12910 pr_warn("failed to open perf buffer event on cpu #%d: %s\n", 12911 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 12912 goto error; 12913 } 12914 12915 cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size, 12916 PROT_READ | PROT_WRITE, MAP_SHARED, 12917 cpu_buf->fd, 0); 12918 if (cpu_buf->base == MAP_FAILED) { 12919 cpu_buf->base = NULL; 12920 err = -errno; 12921 pr_warn("failed to mmap perf buffer on cpu #%d: %s\n", 12922 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 12923 goto error; 12924 } 12925 12926 if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) { 12927 err = -errno; 12928 pr_warn("failed to enable perf buffer event on cpu #%d: %s\n", 12929 cpu, libbpf_strerror_r(err, msg, sizeof(msg))); 12930 goto error; 12931 } 12932 12933 return cpu_buf; 12934 12935error: 12936 perf_buffer__free_cpu_buf(pb, cpu_buf); 12937 return (struct perf_cpu_buf *)ERR_PTR(err); 12938} 12939 12940static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 12941 struct perf_buffer_params *p); 12942 12943struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, 12944 perf_buffer_sample_fn sample_cb, 12945 perf_buffer_lost_fn lost_cb, 12946 void *ctx, 12947 const struct perf_buffer_opts *opts) 12948{ 12949 const size_t attr_sz = sizeof(struct perf_event_attr); 12950 struct perf_buffer_params p = {}; 12951 struct perf_event_attr attr; 12952 __u32 sample_period; 12953 12954 if (!OPTS_VALID(opts, perf_buffer_opts)) 12955 return libbpf_err_ptr(-EINVAL); 12956 12957 sample_period = OPTS_GET(opts, sample_period, 1); 12958 if (!sample_period) 12959 sample_period = 1; 12960 12961 memset(&attr, 0, attr_sz); 12962 attr.size = attr_sz; 12963 attr.config = PERF_COUNT_SW_BPF_OUTPUT; 12964 attr.type = PERF_TYPE_SOFTWARE; 12965 attr.sample_type = PERF_SAMPLE_RAW; 12966 attr.sample_period = sample_period; 12967 attr.wakeup_events = sample_period; 12968 12969 p.attr = &attr; 12970 p.sample_cb = sample_cb; 12971 p.lost_cb = lost_cb; 12972 p.ctx = ctx; 12973 12974 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 12975} 12976 12977struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt, 12978 struct perf_event_attr *attr, 12979 perf_buffer_event_fn event_cb, void *ctx, 12980 const struct perf_buffer_raw_opts *opts) 12981{ 12982 struct perf_buffer_params p = {}; 12983 12984 if (!attr) 12985 return libbpf_err_ptr(-EINVAL); 12986 12987 if (!OPTS_VALID(opts, perf_buffer_raw_opts)) 12988 return libbpf_err_ptr(-EINVAL); 12989 12990 p.attr = attr; 12991 p.event_cb = event_cb; 12992 p.ctx = ctx; 12993 p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0); 12994 p.cpus = OPTS_GET(opts, cpus, NULL); 12995 p.map_keys = OPTS_GET(opts, map_keys, NULL); 12996 12997 return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); 12998} 12999 13000static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, 13001 struct perf_buffer_params *p) 13002{ 13003 const char *online_cpus_file = "/sys/devices/system/cpu/online"; 13004 struct bpf_map_info map; 13005 char msg[STRERR_BUFSIZE]; 13006 struct perf_buffer *pb; 13007 bool *online = NULL; 13008 __u32 map_info_len; 13009 int err, i, j, n; 13010 13011 if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) { 13012 pr_warn("page count should be power of two, but is %zu\n", 13013 page_cnt); 13014 return ERR_PTR(-EINVAL); 13015 } 13016 13017 /* best-effort sanity checks */ 13018 memset(&map, 0, sizeof(map)); 13019 map_info_len = sizeof(map); 13020 err = bpf_map_get_info_by_fd(map_fd, &map, &map_info_len); 13021 if (err) { 13022 err = -errno; 13023 /* if BPF_OBJ_GET_INFO_BY_FD is supported, will return 13024 * -EBADFD, -EFAULT, or -E2BIG on real error 13025 */ 13026 if (err != -EINVAL) { 13027 pr_warn("failed to get map info for map FD %d: %s\n", 13028 map_fd, libbpf_strerror_r(err, msg, sizeof(msg))); 13029 return ERR_PTR(err); 13030 } 13031 pr_debug("failed to get map info for FD %d; API not supported? Ignoring...\n", 13032 map_fd); 13033 } else { 13034 if (map.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) { 13035 pr_warn("map '%s' should be BPF_MAP_TYPE_PERF_EVENT_ARRAY\n", 13036 map.name); 13037 return ERR_PTR(-EINVAL); 13038 } 13039 } 13040 13041 pb = calloc(1, sizeof(*pb)); 13042 if (!pb) 13043 return ERR_PTR(-ENOMEM); 13044 13045 pb->event_cb = p->event_cb; 13046 pb->sample_cb = p->sample_cb; 13047 pb->lost_cb = p->lost_cb; 13048 pb->ctx = p->ctx; 13049 13050 pb->page_size = getpagesize(); 13051 pb->mmap_size = pb->page_size * page_cnt; 13052 pb->map_fd = map_fd; 13053 13054 pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC); 13055 if (pb->epoll_fd < 0) { 13056 err = -errno; 13057 pr_warn("failed to create epoll instance: %s\n", 13058 libbpf_strerror_r(err, msg, sizeof(msg))); 13059 goto error; 13060 } 13061 13062 if (p->cpu_cnt > 0) { 13063 pb->cpu_cnt = p->cpu_cnt; 13064 } else { 13065 pb->cpu_cnt = libbpf_num_possible_cpus(); 13066 if (pb->cpu_cnt < 0) { 13067 err = pb->cpu_cnt; 13068 goto error; 13069 } 13070 if (map.max_entries && map.max_entries < pb->cpu_cnt) 13071 pb->cpu_cnt = map.max_entries; 13072 } 13073 13074 pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events)); 13075 if (!pb->events) { 13076 err = -ENOMEM; 13077 pr_warn("failed to allocate events: out of memory\n"); 13078 goto error; 13079 } 13080 pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs)); 13081 if (!pb->cpu_bufs) { 13082 err = -ENOMEM; 13083 pr_warn("failed to allocate buffers: out of memory\n"); 13084 goto error; 13085 } 13086 13087 err = parse_cpu_mask_file(online_cpus_file, &online, &n); 13088 if (err) { 13089 pr_warn("failed to get online CPU mask: %d\n", err); 13090 goto error; 13091 } 13092 13093 for (i = 0, j = 0; i < pb->cpu_cnt; i++) { 13094 struct perf_cpu_buf *cpu_buf; 13095 int cpu, map_key; 13096 13097 cpu = p->cpu_cnt > 0 ? p->cpus[i] : i; 13098 map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i; 13099 13100 /* in case user didn't explicitly requested particular CPUs to 13101 * be attached to, skip offline/not present CPUs 13102 */ 13103 if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu])) 13104 continue; 13105 13106 cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key); 13107 if (IS_ERR(cpu_buf)) { 13108 err = PTR_ERR(cpu_buf); 13109 goto error; 13110 } 13111 13112 pb->cpu_bufs[j] = cpu_buf; 13113 13114 err = bpf_map_update_elem(pb->map_fd, &map_key, 13115 &cpu_buf->fd, 0); 13116 if (err) { 13117 err = -errno; 13118 pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n", 13119 cpu, map_key, cpu_buf->fd, 13120 libbpf_strerror_r(err, msg, sizeof(msg))); 13121 goto error; 13122 } 13123 13124 pb->events[j].events = EPOLLIN; 13125 pb->events[j].data.ptr = cpu_buf; 13126 if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd, 13127 &pb->events[j]) < 0) { 13128 err = -errno; 13129 pr_warn("failed to epoll_ctl cpu #%d perf FD %d: %s\n", 13130 cpu, cpu_buf->fd, 13131 libbpf_strerror_r(err, msg, sizeof(msg))); 13132 goto error; 13133 } 13134 j++; 13135 } 13136 pb->cpu_cnt = j; 13137 free(online); 13138 13139 return pb; 13140 13141error: 13142 free(online); 13143 if (pb) 13144 perf_buffer__free(pb); 13145 return ERR_PTR(err); 13146} 13147 13148struct perf_sample_raw { 13149 struct perf_event_header header; 13150 uint32_t size; 13151 char data[]; 13152}; 13153 13154struct perf_sample_lost { 13155 struct perf_event_header header; 13156 uint64_t id; 13157 uint64_t lost; 13158 uint64_t sample_id; 13159}; 13160 13161static enum bpf_perf_event_ret 13162perf_buffer__process_record(struct perf_event_header *e, void *ctx) 13163{ 13164 struct perf_cpu_buf *cpu_buf = ctx; 13165 struct perf_buffer *pb = cpu_buf->pb; 13166 void *data = e; 13167 13168 /* user wants full control over parsing perf event */ 13169 if (pb->event_cb) 13170 return pb->event_cb(pb->ctx, cpu_buf->cpu, e); 13171 13172 switch (e->type) { 13173 case PERF_RECORD_SAMPLE: { 13174 struct perf_sample_raw *s = data; 13175 13176 if (pb->sample_cb) 13177 pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size); 13178 break; 13179 } 13180 case PERF_RECORD_LOST: { 13181 struct perf_sample_lost *s = data; 13182 13183 if (pb->lost_cb) 13184 pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost); 13185 break; 13186 } 13187 default: 13188 pr_warn("unknown perf sample type %d\n", e->type); 13189 return LIBBPF_PERF_EVENT_ERROR; 13190 } 13191 return LIBBPF_PERF_EVENT_CONT; 13192} 13193 13194static int perf_buffer__process_records(struct perf_buffer *pb, 13195 struct perf_cpu_buf *cpu_buf) 13196{ 13197 enum bpf_perf_event_ret ret; 13198 13199 ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size, 13200 pb->page_size, &cpu_buf->buf, 13201 &cpu_buf->buf_size, 13202 perf_buffer__process_record, cpu_buf); 13203 if (ret != LIBBPF_PERF_EVENT_CONT) 13204 return ret; 13205 return 0; 13206} 13207 13208int perf_buffer__epoll_fd(const struct perf_buffer *pb) 13209{ 13210 return pb->epoll_fd; 13211} 13212 13213int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms) 13214{ 13215 int i, cnt, err; 13216 13217 cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms); 13218 if (cnt < 0) 13219 return -errno; 13220 13221 for (i = 0; i < cnt; i++) { 13222 struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr; 13223 13224 err = perf_buffer__process_records(pb, cpu_buf); 13225 if (err) { 13226 pr_warn("error while processing records: %d\n", err); 13227 return libbpf_err(err); 13228 } 13229 } 13230 return cnt; 13231} 13232 13233/* Return number of PERF_EVENT_ARRAY map slots set up by this perf_buffer 13234 * manager. 13235 */ 13236size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb) 13237{ 13238 return pb->cpu_cnt; 13239} 13240 13241/* 13242 * Return perf_event FD of a ring buffer in *buf_idx* slot of 13243 * PERF_EVENT_ARRAY BPF map. This FD can be polled for new data using 13244 * select()/poll()/epoll() Linux syscalls. 13245 */ 13246int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) 13247{ 13248 struct perf_cpu_buf *cpu_buf; 13249 13250 if (buf_idx >= pb->cpu_cnt) 13251 return libbpf_err(-EINVAL); 13252 13253 cpu_buf = pb->cpu_bufs[buf_idx]; 13254 if (!cpu_buf) 13255 return libbpf_err(-ENOENT); 13256 13257 return cpu_buf->fd; 13258} 13259 13260int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size) 13261{ 13262 struct perf_cpu_buf *cpu_buf; 13263 13264 if (buf_idx >= pb->cpu_cnt) 13265 return libbpf_err(-EINVAL); 13266 13267 cpu_buf = pb->cpu_bufs[buf_idx]; 13268 if (!cpu_buf) 13269 return libbpf_err(-ENOENT); 13270 13271 *buf = cpu_buf->base; 13272 *buf_size = pb->mmap_size; 13273 return 0; 13274} 13275 13276/* 13277 * Consume data from perf ring buffer corresponding to slot *buf_idx* in 13278 * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to 13279 * consume, do nothing and return success. 13280 * Returns: 13281 * - 0 on success; 13282 * - <0 on failure. 13283 */ 13284int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx) 13285{ 13286 struct perf_cpu_buf *cpu_buf; 13287 13288 if (buf_idx >= pb->cpu_cnt) 13289 return libbpf_err(-EINVAL); 13290 13291 cpu_buf = pb->cpu_bufs[buf_idx]; 13292 if (!cpu_buf) 13293 return libbpf_err(-ENOENT); 13294 13295 return perf_buffer__process_records(pb, cpu_buf); 13296} 13297 13298int perf_buffer__consume(struct perf_buffer *pb) 13299{ 13300 int i, err; 13301 13302 for (i = 0; i < pb->cpu_cnt; i++) { 13303 struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i]; 13304 13305 if (!cpu_buf) 13306 continue; 13307 13308 err = perf_buffer__process_records(pb, cpu_buf); 13309 if (err) { 13310 pr_warn("perf_buffer: failed to process records in buffer #%d: %d\n", i, err); 13311 return libbpf_err(err); 13312 } 13313 } 13314 return 0; 13315} 13316 13317int bpf_program__set_attach_target(struct bpf_program *prog, 13318 int attach_prog_fd, 13319 const char *attach_func_name) 13320{ 13321 int btf_obj_fd = 0, btf_id = 0, err; 13322 13323 if (!prog || attach_prog_fd < 0) 13324 return libbpf_err(-EINVAL); 13325 13326 if (prog->obj->loaded) 13327 return libbpf_err(-EINVAL); 13328 13329 if (attach_prog_fd && !attach_func_name) { 13330 /* remember attach_prog_fd and let bpf_program__load() find 13331 * BTF ID during the program load 13332 */ 13333 prog->attach_prog_fd = attach_prog_fd; 13334 return 0; 13335 } 13336 13337 if (attach_prog_fd) { 13338 btf_id = libbpf_find_prog_btf_id(attach_func_name, 13339 attach_prog_fd); 13340 if (btf_id < 0) 13341 return libbpf_err(btf_id); 13342 } else { 13343 if (!attach_func_name) 13344 return libbpf_err(-EINVAL); 13345 13346 /* load btf_vmlinux, if not yet */ 13347 err = bpf_object__load_vmlinux_btf(prog->obj, true); 13348 if (err) 13349 return libbpf_err(err); 13350 err = find_kernel_btf_id(prog->obj, attach_func_name, 13351 prog->expected_attach_type, 13352 &btf_obj_fd, &btf_id); 13353 if (err) 13354 return libbpf_err(err); 13355 } 13356 13357 prog->attach_btf_id = btf_id; 13358 prog->attach_btf_obj_fd = btf_obj_fd; 13359 prog->attach_prog_fd = attach_prog_fd; 13360 return 0; 13361} 13362 13363int parse_cpu_mask_str(const char *s, bool **mask, int *mask_sz) 13364{ 13365 int err = 0, n, len, start, end = -1; 13366 bool *tmp; 13367 13368 *mask = NULL; 13369 *mask_sz = 0; 13370 13371 /* Each sub string separated by ',' has format \d+-\d+ or \d+ */ 13372 while (*s) { 13373 if (*s == ',' || *s == '\n') { 13374 s++; 13375 continue; 13376 } 13377 n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len); 13378 if (n <= 0 || n > 2) { 13379 pr_warn("Failed to get CPU range %s: %d\n", s, n); 13380 err = -EINVAL; 13381 goto cleanup; 13382 } else if (n == 1) { 13383 end = start; 13384 } 13385 if (start < 0 || start > end) { 13386 pr_warn("Invalid CPU range [%d,%d] in %s\n", 13387 start, end, s); 13388 err = -EINVAL; 13389 goto cleanup; 13390 } 13391 tmp = realloc(*mask, end + 1); 13392 if (!tmp) { 13393 err = -ENOMEM; 13394 goto cleanup; 13395 } 13396 *mask = tmp; 13397 memset(tmp + *mask_sz, 0, start - *mask_sz); 13398 memset(tmp + start, 1, end - start + 1); 13399 *mask_sz = end + 1; 13400 s += len; 13401 } 13402 if (!*mask_sz) { 13403 pr_warn("Empty CPU range\n"); 13404 return -EINVAL; 13405 } 13406 return 0; 13407cleanup: 13408 free(*mask); 13409 *mask = NULL; 13410 return err; 13411} 13412 13413int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz) 13414{ 13415 int fd, err = 0, len; 13416 char buf[128]; 13417 13418 fd = open(fcpu, O_RDONLY | O_CLOEXEC); 13419 if (fd < 0) { 13420 err = -errno; 13421 pr_warn("Failed to open cpu mask file %s: %d\n", fcpu, err); 13422 return err; 13423 } 13424 len = read(fd, buf, sizeof(buf)); 13425 close(fd); 13426 if (len <= 0) { 13427 err = len ? -errno : -EINVAL; 13428 pr_warn("Failed to read cpu mask from %s: %d\n", fcpu, err); 13429 return err; 13430 } 13431 if (len >= sizeof(buf)) { 13432 pr_warn("CPU mask is too big in file %s\n", fcpu); 13433 return -E2BIG; 13434 } 13435 buf[len] = '\0'; 13436 13437 return parse_cpu_mask_str(buf, mask, mask_sz); 13438} 13439 13440int libbpf_num_possible_cpus(void) 13441{ 13442 static const char *fcpu = "/sys/devices/system/cpu/possible"; 13443 static int cpus; 13444 int err, n, i, tmp_cpus; 13445 bool *mask; 13446 13447 tmp_cpus = READ_ONCE(cpus); 13448 if (tmp_cpus > 0) 13449 return tmp_cpus; 13450 13451 err = parse_cpu_mask_file(fcpu, &mask, &n); 13452 if (err) 13453 return libbpf_err(err); 13454 13455 tmp_cpus = 0; 13456 for (i = 0; i < n; i++) { 13457 if (mask[i]) 13458 tmp_cpus++; 13459 } 13460 free(mask); 13461 13462 WRITE_ONCE(cpus, tmp_cpus); 13463 return tmp_cpus; 13464} 13465 13466static int populate_skeleton_maps(const struct bpf_object *obj, 13467 struct bpf_map_skeleton *maps, 13468 size_t map_cnt) 13469{ 13470 int i; 13471 13472 for (i = 0; i < map_cnt; i++) { 13473 struct bpf_map **map = maps[i].map; 13474 const char *name = maps[i].name; 13475 void **mmaped = maps[i].mmaped; 13476 13477 *map = bpf_object__find_map_by_name(obj, name); 13478 if (!*map) { 13479 pr_warn("failed to find skeleton map '%s'\n", name); 13480 return -ESRCH; 13481 } 13482 13483 /* externs shouldn't be pre-setup from user code */ 13484 if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG) 13485 *mmaped = (*map)->mmaped; 13486 } 13487 return 0; 13488} 13489 13490static int populate_skeleton_progs(const struct bpf_object *obj, 13491 struct bpf_prog_skeleton *progs, 13492 size_t prog_cnt) 13493{ 13494 int i; 13495 13496 for (i = 0; i < prog_cnt; i++) { 13497 struct bpf_program **prog = progs[i].prog; 13498 const char *name = progs[i].name; 13499 13500 *prog = bpf_object__find_program_by_name(obj, name); 13501 if (!*prog) { 13502 pr_warn("failed to find skeleton program '%s'\n", name); 13503 return -ESRCH; 13504 } 13505 } 13506 return 0; 13507} 13508 13509int bpf_object__open_skeleton(struct bpf_object_skeleton *s, 13510 const struct bpf_object_open_opts *opts) 13511{ 13512 DECLARE_LIBBPF_OPTS(bpf_object_open_opts, skel_opts, 13513 .object_name = s->name, 13514 ); 13515 struct bpf_object *obj; 13516 int err; 13517 13518 /* Attempt to preserve opts->object_name, unless overriden by user 13519 * explicitly. Overwriting object name for skeletons is discouraged, 13520 * as it breaks global data maps, because they contain object name 13521 * prefix as their own map name prefix. When skeleton is generated, 13522 * bpftool is making an assumption that this name will stay the same. 13523 */ 13524 if (opts) { 13525 memcpy(&skel_opts, opts, sizeof(*opts)); 13526 if (!opts->object_name) 13527 skel_opts.object_name = s->name; 13528 } 13529 13530 obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts); 13531 err = libbpf_get_error(obj); 13532 if (err) { 13533 pr_warn("failed to initialize skeleton BPF object '%s': %d\n", 13534 s->name, err); 13535 return libbpf_err(err); 13536 } 13537 13538 *s->obj = obj; 13539 err = populate_skeleton_maps(obj, s->maps, s->map_cnt); 13540 if (err) { 13541 pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err); 13542 return libbpf_err(err); 13543 } 13544 13545 err = populate_skeleton_progs(obj, s->progs, s->prog_cnt); 13546 if (err) { 13547 pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err); 13548 return libbpf_err(err); 13549 } 13550 13551 return 0; 13552} 13553 13554int bpf_object__open_subskeleton(struct bpf_object_subskeleton *s) 13555{ 13556 int err, len, var_idx, i; 13557 const char *var_name; 13558 const struct bpf_map *map; 13559 struct btf *btf; 13560 __u32 map_type_id; 13561 const struct btf_type *map_type, *var_type; 13562 const struct bpf_var_skeleton *var_skel; 13563 struct btf_var_secinfo *var; 13564 13565 if (!s->obj) 13566 return libbpf_err(-EINVAL); 13567 13568 btf = bpf_object__btf(s->obj); 13569 if (!btf) { 13570 pr_warn("subskeletons require BTF at runtime (object %s)\n", 13571 bpf_object__name(s->obj)); 13572 return libbpf_err(-errno); 13573 } 13574 13575 err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt); 13576 if (err) { 13577 pr_warn("failed to populate subskeleton maps: %d\n", err); 13578 return libbpf_err(err); 13579 } 13580 13581 err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt); 13582 if (err) { 13583 pr_warn("failed to populate subskeleton maps: %d\n", err); 13584 return libbpf_err(err); 13585 } 13586 13587 for (var_idx = 0; var_idx < s->var_cnt; var_idx++) { 13588 var_skel = &s->vars[var_idx]; 13589 map = *var_skel->map; 13590 map_type_id = bpf_map__btf_value_type_id(map); 13591 map_type = btf__type_by_id(btf, map_type_id); 13592 13593 if (!btf_is_datasec(map_type)) { 13594 pr_warn("type for map '%1$s' is not a datasec: %2$s", 13595 bpf_map__name(map), 13596 __btf_kind_str(btf_kind(map_type))); 13597 return libbpf_err(-EINVAL); 13598 } 13599 13600 len = btf_vlen(map_type); 13601 var = btf_var_secinfos(map_type); 13602 for (i = 0; i < len; i++, var++) { 13603 var_type = btf__type_by_id(btf, var->type); 13604 var_name = btf__name_by_offset(btf, var_type->name_off); 13605 if (strcmp(var_name, var_skel->name) == 0) { 13606 *var_skel->addr = map->mmaped + var->offset; 13607 break; 13608 } 13609 } 13610 } 13611 return 0; 13612} 13613 13614void bpf_object__destroy_subskeleton(struct bpf_object_subskeleton *s) 13615{ 13616 if (!s) 13617 return; 13618 free(s->maps); 13619 free(s->progs); 13620 free(s->vars); 13621 free(s); 13622} 13623 13624int bpf_object__load_skeleton(struct bpf_object_skeleton *s) 13625{ 13626 int i, err; 13627 13628 err = bpf_object__load(*s->obj); 13629 if (err) { 13630 pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err); 13631 return libbpf_err(err); 13632 } 13633 13634 for (i = 0; i < s->map_cnt; i++) { 13635 struct bpf_map *map = *s->maps[i].map; 13636 size_t mmap_sz = bpf_map_mmap_sz(map->def.value_size, map->def.max_entries); 13637 int prot, map_fd = bpf_map__fd(map); 13638 void **mmaped = s->maps[i].mmaped; 13639 13640 if (!mmaped) 13641 continue; 13642 13643 if (!(map->def.map_flags & BPF_F_MMAPABLE)) { 13644 *mmaped = NULL; 13645 continue; 13646 } 13647 13648 if (map->def.map_flags & BPF_F_RDONLY_PROG) 13649 prot = PROT_READ; 13650 else 13651 prot = PROT_READ | PROT_WRITE; 13652 13653 /* Remap anonymous mmap()-ed "map initialization image" as 13654 * a BPF map-backed mmap()-ed memory, but preserving the same 13655 * memory address. This will cause kernel to change process' 13656 * page table to point to a different piece of kernel memory, 13657 * but from userspace point of view memory address (and its 13658 * contents, being identical at this point) will stay the 13659 * same. This mapping will be released by bpf_object__close() 13660 * as per normal clean up procedure, so we don't need to worry 13661 * about it from skeleton's clean up perspective. 13662 */ 13663 *mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0); 13664 if (*mmaped == MAP_FAILED) { 13665 err = -errno; 13666 *mmaped = NULL; 13667 pr_warn("failed to re-mmap() map '%s': %d\n", 13668 bpf_map__name(map), err); 13669 return libbpf_err(err); 13670 } 13671 } 13672 13673 return 0; 13674} 13675 13676int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) 13677{ 13678 int i, err; 13679 13680 for (i = 0; i < s->prog_cnt; i++) { 13681 struct bpf_program *prog = *s->progs[i].prog; 13682 struct bpf_link **link = s->progs[i].link; 13683 13684 if (!prog->autoload || !prog->autoattach) 13685 continue; 13686 13687 /* auto-attaching not supported for this program */ 13688 if (!prog->sec_def || !prog->sec_def->prog_attach_fn) 13689 continue; 13690 13691 /* if user already set the link manually, don't attempt auto-attach */ 13692 if (*link) 13693 continue; 13694 13695 err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link); 13696 if (err) { 13697 pr_warn("prog '%s': failed to auto-attach: %d\n", 13698 bpf_program__name(prog), err); 13699 return libbpf_err(err); 13700 } 13701 13702 /* It's possible that for some SEC() definitions auto-attach 13703 * is supported in some cases (e.g., if definition completely 13704 * specifies target information), but is not in other cases. 13705 * SEC("uprobe") is one such case. If user specified target 13706 * binary and function name, such BPF program can be 13707 * auto-attached. But if not, it shouldn't trigger skeleton's 13708 * attach to fail. It should just be skipped. 13709 * attach_fn signals such case with returning 0 (no error) and 13710 * setting link to NULL. 13711 */ 13712 } 13713 13714 return 0; 13715} 13716 13717void bpf_object__detach_skeleton(struct bpf_object_skeleton *s) 13718{ 13719 int i; 13720 13721 for (i = 0; i < s->prog_cnt; i++) { 13722 struct bpf_link **link = s->progs[i].link; 13723 13724 bpf_link__destroy(*link); 13725 *link = NULL; 13726 } 13727} 13728 13729void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s) 13730{ 13731 if (!s) 13732 return; 13733 13734 if (s->progs) 13735 bpf_object__detach_skeleton(s); 13736 if (s->obj) 13737 bpf_object__close(*s->obj); 13738 free(s->maps); 13739 free(s->progs); 13740 free(s); 13741} 13742