1// SPDX-License-Identifier: GPL-2.0 2#include <stdio.h> 3#include <sys/types.h> 4#include <sys/stat.h> 5#include <fcntl.h> 6#include <libelf.h> 7#include <gelf.h> 8#include <errno.h> 9#include <unistd.h> 10#include <string.h> 11#include <stdbool.h> 12#include <stdlib.h> 13#include <linux/bpf.h> 14#include <linux/filter.h> 15#include <linux/perf_event.h> 16#include <linux/netlink.h> 17#include <linux/rtnetlink.h> 18#include <linux/types.h> 19#include <sys/socket.h> 20#include <sys/syscall.h> 21#include <sys/ioctl.h> 22#include <sys/mman.h> 23#include <poll.h> 24#include <ctype.h> 25#include <assert.h> 26#include <bpf/bpf.h> 27#include "bpf_load.h" 28#include "perf-sys.h" 29 30#define DEBUGFS "/sys/kernel/debug/tracing/" 31 32static char license[128]; 33static int kern_version; 34static bool processed_sec[128]; 35char bpf_log_buf[BPF_LOG_BUF_SIZE]; 36int map_fd[MAX_MAPS]; 37int prog_fd[MAX_PROGS]; 38int event_fd[MAX_PROGS]; 39int prog_cnt; 40int prog_array_fd = -1; 41 42struct bpf_map_data map_data[MAX_MAPS]; 43int map_data_count; 44 45static int populate_prog_array(const char *event, int prog_fd) 46{ 47 int ind = atoi(event), err; 48 49 err = bpf_map_update_elem(prog_array_fd, &ind, &prog_fd, BPF_ANY); 50 if (err < 0) { 51 printf("failed to store prog_fd in prog_array\n"); 52 return -1; 53 } 54 return 0; 55} 56 57static int write_kprobe_events(const char *val) 58{ 59 int fd, ret, flags; 60 61 if (val == NULL) 62 return -1; 63 else if (val[0] == '\0') 64 flags = O_WRONLY | O_TRUNC; 65 else 66 flags = O_WRONLY | O_APPEND; 67 68 fd = open(DEBUGFS "kprobe_events", flags); 69 70 ret = write(fd, val, strlen(val)); 71 close(fd); 72 73 return ret; 74} 75 76static int load_and_attach(const char *event, struct bpf_insn *prog, int size) 77{ 78 bool is_socket = strncmp(event, "socket", 6) == 0; 79 bool is_kprobe = strncmp(event, "kprobe/", 7) == 0; 80 bool is_kretprobe = strncmp(event, "kretprobe/", 10) == 0; 81 bool is_tracepoint = strncmp(event, "tracepoint/", 11) == 0; 82 bool is_raw_tracepoint = strncmp(event, "raw_tracepoint/", 15) == 0; 83 bool is_xdp = strncmp(event, "xdp", 3) == 0; 84 bool is_perf_event = strncmp(event, "perf_event", 10) == 0; 85 bool is_cgroup_skb = strncmp(event, "cgroup/skb", 10) == 0; 86 bool is_cgroup_sk = strncmp(event, "cgroup/sock", 11) == 0; 87 bool is_sockops = strncmp(event, "sockops", 7) == 0; 88 bool is_sk_skb = strncmp(event, "sk_skb", 6) == 0; 89 bool is_sk_msg = strncmp(event, "sk_msg", 6) == 0; 90 size_t insns_cnt = size / sizeof(struct bpf_insn); 91 enum bpf_prog_type prog_type; 92 char buf[256]; 93 int fd, efd, err, id; 94 struct perf_event_attr attr = {}; 95 96 attr.type = PERF_TYPE_TRACEPOINT; 97 attr.sample_type = PERF_SAMPLE_RAW; 98 attr.sample_period = 1; 99 attr.wakeup_events = 1; 100 101 if (is_socket) { 102 prog_type = BPF_PROG_TYPE_SOCKET_FILTER; 103 } else if (is_kprobe || is_kretprobe) { 104 prog_type = BPF_PROG_TYPE_KPROBE; 105 } else if (is_tracepoint) { 106 prog_type = BPF_PROG_TYPE_TRACEPOINT; 107 } else if (is_raw_tracepoint) { 108 prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT; 109 } else if (is_xdp) { 110 prog_type = BPF_PROG_TYPE_XDP; 111 } else if (is_perf_event) { 112 prog_type = BPF_PROG_TYPE_PERF_EVENT; 113 } else if (is_cgroup_skb) { 114 prog_type = BPF_PROG_TYPE_CGROUP_SKB; 115 } else if (is_cgroup_sk) { 116 prog_type = BPF_PROG_TYPE_CGROUP_SOCK; 117 } else if (is_sockops) { 118 prog_type = BPF_PROG_TYPE_SOCK_OPS; 119 } else if (is_sk_skb) { 120 prog_type = BPF_PROG_TYPE_SK_SKB; 121 } else if (is_sk_msg) { 122 prog_type = BPF_PROG_TYPE_SK_MSG; 123 } else { 124 printf("Unknown event '%s'\n", event); 125 return -1; 126 } 127 128 if (prog_cnt == MAX_PROGS) 129 return -1; 130 131 fd = bpf_load_program(prog_type, prog, insns_cnt, license, kern_version, 132 bpf_log_buf, BPF_LOG_BUF_SIZE); 133 if (fd < 0) { 134 printf("bpf_load_program() err=%d\n%s", errno, bpf_log_buf); 135 return -1; 136 } 137 138 prog_fd[prog_cnt++] = fd; 139 140 if (is_xdp || is_perf_event || is_cgroup_skb || is_cgroup_sk) 141 return 0; 142 143 if (is_socket || is_sockops || is_sk_skb || is_sk_msg) { 144 if (is_socket) 145 event += 6; 146 else 147 event += 7; 148 if (*event != '/') 149 return 0; 150 event++; 151 if (!isdigit(*event)) { 152 printf("invalid prog number\n"); 153 return -1; 154 } 155 return populate_prog_array(event, fd); 156 } 157 158 if (is_raw_tracepoint) { 159 efd = bpf_raw_tracepoint_open(event + 15, fd); 160 if (efd < 0) { 161 printf("tracepoint %s %s\n", event + 15, strerror(errno)); 162 return -1; 163 } 164 event_fd[prog_cnt - 1] = efd; 165 return 0; 166 } 167 168 if (is_kprobe || is_kretprobe) { 169 bool need_normal_check = true; 170 const char *event_prefix = ""; 171 172 if (is_kprobe) 173 event += 7; 174 else 175 event += 10; 176 177 if (*event == 0) { 178 printf("event name cannot be empty\n"); 179 return -1; 180 } 181 182 if (isdigit(*event)) 183 return populate_prog_array(event, fd); 184 185#ifdef __x86_64__ 186 if (strncmp(event, "sys_", 4) == 0) { 187 snprintf(buf, sizeof(buf), "%c:__x64_%s __x64_%s", 188 is_kprobe ? 'p' : 'r', event, event); 189 err = write_kprobe_events(buf); 190 if (err >= 0) { 191 need_normal_check = false; 192 event_prefix = "__x64_"; 193 } 194 } 195#endif 196 if (need_normal_check) { 197 snprintf(buf, sizeof(buf), "%c:%s %s", 198 is_kprobe ? 'p' : 'r', event, event); 199 err = write_kprobe_events(buf); 200 if (err < 0) { 201 printf("failed to create kprobe '%s' error '%s'\n", 202 event, strerror(errno)); 203 return -1; 204 } 205 } 206 207 strcpy(buf, DEBUGFS); 208 strcat(buf, "events/kprobes/"); 209 strcat(buf, event_prefix); 210 strcat(buf, event); 211 strcat(buf, "/id"); 212 } else if (is_tracepoint) { 213 event += 11; 214 215 if (*event == 0) { 216 printf("event name cannot be empty\n"); 217 return -1; 218 } 219 strcpy(buf, DEBUGFS); 220 strcat(buf, "events/"); 221 strcat(buf, event); 222 strcat(buf, "/id"); 223 } 224 225 efd = open(buf, O_RDONLY, 0); 226 if (efd < 0) { 227 printf("failed to open event %s\n", event); 228 return -1; 229 } 230 231 err = read(efd, buf, sizeof(buf)); 232 if (err < 0 || err >= sizeof(buf)) { 233 printf("read from '%s' failed '%s'\n", event, strerror(errno)); 234 return -1; 235 } 236 237 close(efd); 238 239 buf[err] = 0; 240 id = atoi(buf); 241 attr.config = id; 242 243 efd = sys_perf_event_open(&attr, -1/*pid*/, 0/*cpu*/, -1/*group_fd*/, 0); 244 if (efd < 0) { 245 printf("event %d fd %d err %s\n", id, efd, strerror(errno)); 246 return -1; 247 } 248 event_fd[prog_cnt - 1] = efd; 249 err = ioctl(efd, PERF_EVENT_IOC_ENABLE, 0); 250 if (err < 0) { 251 printf("ioctl PERF_EVENT_IOC_ENABLE failed err %s\n", 252 strerror(errno)); 253 return -1; 254 } 255 err = ioctl(efd, PERF_EVENT_IOC_SET_BPF, fd); 256 if (err < 0) { 257 printf("ioctl PERF_EVENT_IOC_SET_BPF failed err %s\n", 258 strerror(errno)); 259 return -1; 260 } 261 262 return 0; 263} 264 265static int load_maps(struct bpf_map_data *maps, int nr_maps, 266 fixup_map_cb fixup_map) 267{ 268 int i, numa_node; 269 270 for (i = 0; i < nr_maps; i++) { 271 if (fixup_map) { 272 fixup_map(&maps[i], i); 273 /* Allow userspace to assign map FD prior to creation */ 274 if (maps[i].fd != -1) { 275 map_fd[i] = maps[i].fd; 276 continue; 277 } 278 } 279 280 numa_node = maps[i].def.map_flags & BPF_F_NUMA_NODE ? 281 maps[i].def.numa_node : -1; 282 283 if (maps[i].def.type == BPF_MAP_TYPE_ARRAY_OF_MAPS || 284 maps[i].def.type == BPF_MAP_TYPE_HASH_OF_MAPS) { 285 int inner_map_fd = map_fd[maps[i].def.inner_map_idx]; 286 287 map_fd[i] = bpf_create_map_in_map_node(maps[i].def.type, 288 maps[i].name, 289 maps[i].def.key_size, 290 inner_map_fd, 291 maps[i].def.max_entries, 292 maps[i].def.map_flags, 293 numa_node); 294 } else { 295 map_fd[i] = bpf_create_map_node(maps[i].def.type, 296 maps[i].name, 297 maps[i].def.key_size, 298 maps[i].def.value_size, 299 maps[i].def.max_entries, 300 maps[i].def.map_flags, 301 numa_node); 302 } 303 if (map_fd[i] < 0) { 304 printf("failed to create map %d (%s): %d %s\n", 305 i, maps[i].name, errno, strerror(errno)); 306 return 1; 307 } 308 maps[i].fd = map_fd[i]; 309 310 if (maps[i].def.type == BPF_MAP_TYPE_PROG_ARRAY) 311 prog_array_fd = map_fd[i]; 312 } 313 return 0; 314} 315 316static int get_sec(Elf *elf, int i, GElf_Ehdr *ehdr, char **shname, 317 GElf_Shdr *shdr, Elf_Data **data) 318{ 319 Elf_Scn *scn; 320 321 scn = elf_getscn(elf, i); 322 if (!scn) 323 return 1; 324 325 if (gelf_getshdr(scn, shdr) != shdr) 326 return 2; 327 328 *shname = elf_strptr(elf, ehdr->e_shstrndx, shdr->sh_name); 329 if (!*shname || !shdr->sh_size) 330 return 3; 331 332 *data = elf_getdata(scn, 0); 333 if (!*data || elf_getdata(scn, *data) != NULL) 334 return 4; 335 336 return 0; 337} 338 339static int parse_relo_and_apply(Elf_Data *data, Elf_Data *symbols, 340 GElf_Shdr *shdr, struct bpf_insn *insn, 341 struct bpf_map_data *maps, int nr_maps) 342{ 343 int i, nrels; 344 345 nrels = shdr->sh_size / shdr->sh_entsize; 346 347 for (i = 0; i < nrels; i++) { 348 GElf_Sym sym; 349 GElf_Rel rel; 350 unsigned int insn_idx; 351 bool match = false; 352 int j, map_idx; 353 354 gelf_getrel(data, i, &rel); 355 356 insn_idx = rel.r_offset / sizeof(struct bpf_insn); 357 358 gelf_getsym(symbols, GELF_R_SYM(rel.r_info), &sym); 359 360 if (insn[insn_idx].code != (BPF_LD | BPF_IMM | BPF_DW)) { 361 printf("invalid relo for insn[%d].code 0x%x\n", 362 insn_idx, insn[insn_idx].code); 363 return 1; 364 } 365 insn[insn_idx].src_reg = BPF_PSEUDO_MAP_FD; 366 367 /* Match FD relocation against recorded map_data[] offset */ 368 for (map_idx = 0; map_idx < nr_maps; map_idx++) { 369 if (maps[map_idx].elf_offset == sym.st_value) { 370 match = true; 371 break; 372 } 373 } 374 if (match) { 375 insn[insn_idx].imm = maps[map_idx].fd; 376 } else { 377 printf("invalid relo for insn[%d] no map_data match\n", 378 insn_idx); 379 return 1; 380 } 381 } 382 383 return 0; 384} 385 386static int cmp_symbols(const void *l, const void *r) 387{ 388 const GElf_Sym *lsym = (const GElf_Sym *)l; 389 const GElf_Sym *rsym = (const GElf_Sym *)r; 390 391 if (lsym->st_value < rsym->st_value) 392 return -1; 393 else if (lsym->st_value > rsym->st_value) 394 return 1; 395 else 396 return 0; 397} 398 399static int load_elf_maps_section(struct bpf_map_data *maps, int maps_shndx, 400 Elf *elf, Elf_Data *symbols, int strtabidx) 401{ 402 int map_sz_elf, map_sz_copy; 403 bool validate_zero = false; 404 Elf_Data *data_maps; 405 int i, nr_maps; 406 GElf_Sym *sym; 407 Elf_Scn *scn; 408 int copy_sz; 409 410 if (maps_shndx < 0) 411 return -EINVAL; 412 if (!symbols) 413 return -EINVAL; 414 415 /* Get data for maps section via elf index */ 416 scn = elf_getscn(elf, maps_shndx); 417 if (scn) 418 data_maps = elf_getdata(scn, NULL); 419 if (!scn || !data_maps) { 420 printf("Failed to get Elf_Data from maps section %d\n", 421 maps_shndx); 422 return -EINVAL; 423 } 424 425 /* For each map get corrosponding symbol table entry */ 426 sym = calloc(MAX_MAPS+1, sizeof(GElf_Sym)); 427 for (i = 0, nr_maps = 0; i < symbols->d_size / sizeof(GElf_Sym); i++) { 428 assert(nr_maps < MAX_MAPS+1); 429 if (!gelf_getsym(symbols, i, &sym[nr_maps])) 430 continue; 431 if (sym[nr_maps].st_shndx != maps_shndx) 432 continue; 433 /* Only increment iif maps section */ 434 nr_maps++; 435 } 436 437 /* Align to map_fd[] order, via sort on offset in sym.st_value */ 438 qsort(sym, nr_maps, sizeof(GElf_Sym), cmp_symbols); 439 440 /* Keeping compatible with ELF maps section changes 441 * ------------------------------------------------ 442 * The program size of struct bpf_load_map_def is known by loader 443 * code, but struct stored in ELF file can be different. 444 * 445 * Unfortunately sym[i].st_size is zero. To calculate the 446 * struct size stored in the ELF file, assume all struct have 447 * the same size, and simply divide with number of map 448 * symbols. 449 */ 450 map_sz_elf = data_maps->d_size / nr_maps; 451 map_sz_copy = sizeof(struct bpf_load_map_def); 452 if (map_sz_elf < map_sz_copy) { 453 /* 454 * Backward compat, loading older ELF file with 455 * smaller struct, keeping remaining bytes zero. 456 */ 457 map_sz_copy = map_sz_elf; 458 } else if (map_sz_elf > map_sz_copy) { 459 /* 460 * Forward compat, loading newer ELF file with larger 461 * struct with unknown features. Assume zero means 462 * feature not used. Thus, validate rest of struct 463 * data is zero. 464 */ 465 validate_zero = true; 466 } 467 468 /* Memcpy relevant part of ELF maps data to loader maps */ 469 for (i = 0; i < nr_maps; i++) { 470 struct bpf_load_map_def *def; 471 unsigned char *addr, *end; 472 const char *map_name; 473 size_t offset; 474 475 map_name = elf_strptr(elf, strtabidx, sym[i].st_name); 476 maps[i].name = strdup(map_name); 477 if (!maps[i].name) { 478 printf("strdup(%s): %s(%d)\n", map_name, 479 strerror(errno), errno); 480 free(sym); 481 return -errno; 482 } 483 484 /* Symbol value is offset into ELF maps section data area */ 485 offset = sym[i].st_value; 486 def = (struct bpf_load_map_def *)(data_maps->d_buf + offset); 487 maps[i].elf_offset = offset; 488 memset(&maps[i].def, 0, sizeof(struct bpf_load_map_def)); 489 memcpy(&maps[i].def, def, map_sz_copy); 490 491 /* Verify no newer features were requested */ 492 if (validate_zero) { 493 addr = (unsigned char *) def + map_sz_copy; 494 end = (unsigned char *) def + map_sz_elf; 495 for (; addr < end; addr++) { 496 if (*addr != 0) { 497 free(sym); 498 return -EFBIG; 499 } 500 } 501 } 502 } 503 504 free(sym); 505 return nr_maps; 506} 507 508static int do_load_bpf_file(const char *path, fixup_map_cb fixup_map) 509{ 510 int fd, i, ret, maps_shndx = -1, strtabidx = -1; 511 Elf *elf; 512 GElf_Ehdr ehdr; 513 GElf_Shdr shdr, shdr_prog; 514 Elf_Data *data, *data_prog, *data_maps = NULL, *symbols = NULL; 515 char *shname, *shname_prog; 516 int nr_maps = 0; 517 518 /* reset global variables */ 519 kern_version = 0; 520 memset(license, 0, sizeof(license)); 521 memset(processed_sec, 0, sizeof(processed_sec)); 522 523 if (elf_version(EV_CURRENT) == EV_NONE) 524 return 1; 525 526 fd = open(path, O_RDONLY, 0); 527 if (fd < 0) 528 return 1; 529 530 elf = elf_begin(fd, ELF_C_READ, NULL); 531 532 if (!elf) 533 return 1; 534 535 if (gelf_getehdr(elf, &ehdr) != &ehdr) 536 return 1; 537 538 /* clear all kprobes */ 539 i = write_kprobe_events(""); 540 541 /* scan over all elf sections to get license and map info */ 542 for (i = 1; i < ehdr.e_shnum; i++) { 543 544 if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) 545 continue; 546 547 if (0) /* helpful for llvm debugging */ 548 printf("section %d:%s data %p size %zd link %d flags %d\n", 549 i, shname, data->d_buf, data->d_size, 550 shdr.sh_link, (int) shdr.sh_flags); 551 552 if (strcmp(shname, "license") == 0) { 553 processed_sec[i] = true; 554 memcpy(license, data->d_buf, data->d_size); 555 } else if (strcmp(shname, "version") == 0) { 556 processed_sec[i] = true; 557 if (data->d_size != sizeof(int)) { 558 printf("invalid size of version section %zd\n", 559 data->d_size); 560 return 1; 561 } 562 memcpy(&kern_version, data->d_buf, sizeof(int)); 563 } else if (strcmp(shname, "maps") == 0) { 564 int j; 565 566 maps_shndx = i; 567 data_maps = data; 568 for (j = 0; j < MAX_MAPS; j++) 569 map_data[j].fd = -1; 570 } else if (shdr.sh_type == SHT_SYMTAB) { 571 strtabidx = shdr.sh_link; 572 symbols = data; 573 } 574 } 575 576 ret = 1; 577 578 if (!symbols) { 579 printf("missing SHT_SYMTAB section\n"); 580 goto done; 581 } 582 583 if (data_maps) { 584 nr_maps = load_elf_maps_section(map_data, maps_shndx, 585 elf, symbols, strtabidx); 586 if (nr_maps < 0) { 587 printf("Error: Failed loading ELF maps (errno:%d):%s\n", 588 nr_maps, strerror(-nr_maps)); 589 goto done; 590 } 591 if (load_maps(map_data, nr_maps, fixup_map)) 592 goto done; 593 map_data_count = nr_maps; 594 595 processed_sec[maps_shndx] = true; 596 } 597 598 /* process all relo sections, and rewrite bpf insns for maps */ 599 for (i = 1; i < ehdr.e_shnum; i++) { 600 if (processed_sec[i]) 601 continue; 602 603 if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) 604 continue; 605 606 if (shdr.sh_type == SHT_REL) { 607 struct bpf_insn *insns; 608 609 /* locate prog sec that need map fixup (relocations) */ 610 if (get_sec(elf, shdr.sh_info, &ehdr, &shname_prog, 611 &shdr_prog, &data_prog)) 612 continue; 613 614 if (shdr_prog.sh_type != SHT_PROGBITS || 615 !(shdr_prog.sh_flags & SHF_EXECINSTR)) 616 continue; 617 618 insns = (struct bpf_insn *) data_prog->d_buf; 619 processed_sec[i] = true; /* relo section */ 620 621 if (parse_relo_and_apply(data, symbols, &shdr, insns, 622 map_data, nr_maps)) 623 continue; 624 } 625 } 626 627 /* load programs */ 628 for (i = 1; i < ehdr.e_shnum; i++) { 629 630 if (processed_sec[i]) 631 continue; 632 633 if (get_sec(elf, i, &ehdr, &shname, &shdr, &data)) 634 continue; 635 636 if (memcmp(shname, "kprobe/", 7) == 0 || 637 memcmp(shname, "kretprobe/", 10) == 0 || 638 memcmp(shname, "tracepoint/", 11) == 0 || 639 memcmp(shname, "raw_tracepoint/", 15) == 0 || 640 memcmp(shname, "xdp", 3) == 0 || 641 memcmp(shname, "perf_event", 10) == 0 || 642 memcmp(shname, "socket", 6) == 0 || 643 memcmp(shname, "cgroup/", 7) == 0 || 644 memcmp(shname, "sockops", 7) == 0 || 645 memcmp(shname, "sk_skb", 6) == 0 || 646 memcmp(shname, "sk_msg", 6) == 0) { 647 ret = load_and_attach(shname, data->d_buf, 648 data->d_size); 649 if (ret != 0) 650 goto done; 651 } 652 } 653 654done: 655 close(fd); 656 return ret; 657} 658 659int load_bpf_file(char *path) 660{ 661 return do_load_bpf_file(path, NULL); 662} 663 664int load_bpf_file_fixup_map(const char *path, fixup_map_cb fixup_map) 665{ 666 return do_load_bpf_file(path, fixup_map); 667} 668