1// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2/* 3 * BPF static linker 4 * 5 * Copyright (c) 2021 Facebook 6 */ 7#include <stdbool.h> 8#include <stddef.h> 9#include <stdio.h> 10#include <stdlib.h> 11#include <string.h> 12#include <unistd.h> 13#include <errno.h> 14#include <linux/err.h> 15#include <linux/btf.h> 16#include <elf.h> 17#include <libelf.h> 18#include <fcntl.h> 19#include "libbpf.h" 20#include "btf.h" 21#include "libbpf_internal.h" 22#include "strset.h" 23 24#define BTF_EXTERN_SEC ".extern" 25 26struct src_sec { 27 const char *sec_name; 28 /* positional (not necessarily ELF) index in an array of sections */ 29 int id; 30 /* positional (not necessarily ELF) index of a matching section in a final object file */ 31 int dst_id; 32 /* section data offset in a matching output section */ 33 int dst_off; 34 /* whether section is omitted from the final ELF file */ 35 bool skipped; 36 /* whether section is an ephemeral section, not mapped to an ELF section */ 37 bool ephemeral; 38 39 /* ELF info */ 40 size_t sec_idx; 41 Elf_Scn *scn; 42 Elf64_Shdr *shdr; 43 Elf_Data *data; 44 45 /* corresponding BTF DATASEC type ID */ 46 int sec_type_id; 47}; 48 49struct src_obj { 50 const char *filename; 51 int fd; 52 Elf *elf; 53 /* Section header strings section index */ 54 size_t shstrs_sec_idx; 55 /* SYMTAB section index */ 56 size_t symtab_sec_idx; 57 58 struct btf *btf; 59 struct btf_ext *btf_ext; 60 61 /* List of sections (including ephemeral). Slot zero is unused. */ 62 struct src_sec *secs; 63 int sec_cnt; 64 65 /* mapping of symbol indices from src to dst ELF */ 66 int *sym_map; 67 /* mapping from the src BTF type IDs to dst ones */ 68 int *btf_type_map; 69}; 70 71/* single .BTF.ext data section */ 72struct btf_ext_sec_data { 73 size_t rec_cnt; 74 __u32 rec_sz; 75 void *recs; 76}; 77 78struct glob_sym { 79 /* ELF symbol index */ 80 int sym_idx; 81 /* associated section id for .ksyms, .kconfig, etc, but not .extern */ 82 int sec_id; 83 /* extern name offset in STRTAB */ 84 int name_off; 85 /* optional associated BTF type ID */ 86 int btf_id; 87 /* BTF type ID to which VAR/FUNC type is pointing to; used for 88 * rewriting types when extern VAR/FUNC is resolved to a concrete 89 * definition 90 */ 91 int underlying_btf_id; 92 /* sec_var index in the corresponding dst_sec, if exists */ 93 int var_idx; 94 95 /* extern or resolved/global symbol */ 96 bool is_extern; 97 /* weak or strong symbol, never goes back from strong to weak */ 98 bool is_weak; 99}; 100 101struct dst_sec { 102 char *sec_name; 103 /* positional (not necessarily ELF) index in an array of sections */ 104 int id; 105 106 bool ephemeral; 107 108 /* ELF info */ 109 size_t sec_idx; 110 Elf_Scn *scn; 111 Elf64_Shdr *shdr; 112 Elf_Data *data; 113 114 /* final output section size */ 115 int sec_sz; 116 /* final output contents of the section */ 117 void *raw_data; 118 119 /* corresponding STT_SECTION symbol index in SYMTAB */ 120 int sec_sym_idx; 121 122 /* section's DATASEC variable info, emitted on BTF finalization */ 123 bool has_btf; 124 int sec_var_cnt; 125 struct btf_var_secinfo *sec_vars; 126 127 /* section's .BTF.ext data */ 128 struct btf_ext_sec_data func_info; 129 struct btf_ext_sec_data line_info; 130 struct btf_ext_sec_data core_relo_info; 131}; 132 133struct bpf_linker { 134 char *filename; 135 int fd; 136 Elf *elf; 137 Elf64_Ehdr *elf_hdr; 138 139 /* Output sections metadata */ 140 struct dst_sec *secs; 141 int sec_cnt; 142 143 struct strset *strtab_strs; /* STRTAB unique strings */ 144 size_t strtab_sec_idx; /* STRTAB section index */ 145 size_t symtab_sec_idx; /* SYMTAB section index */ 146 147 struct btf *btf; 148 struct btf_ext *btf_ext; 149 150 /* global (including extern) ELF symbols */ 151 int glob_sym_cnt; 152 struct glob_sym *glob_syms; 153}; 154 155#define pr_warn_elf(fmt, ...) \ 156 libbpf_print(LIBBPF_WARN, "libbpf: " fmt ": %s\n", ##__VA_ARGS__, elf_errmsg(-1)) 157 158static int init_output_elf(struct bpf_linker *linker, const char *file); 159 160static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, 161 const struct bpf_linker_file_opts *opts, 162 struct src_obj *obj); 163static int linker_sanity_check_elf(struct src_obj *obj); 164static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec); 165static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec); 166static int linker_sanity_check_btf(struct src_obj *obj); 167static int linker_sanity_check_btf_ext(struct src_obj *obj); 168static int linker_fixup_btf(struct src_obj *obj); 169static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj); 170static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj); 171static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj, 172 Elf64_Sym *sym, const char *sym_name, int src_sym_idx); 173static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj); 174static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj); 175static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj); 176 177static int finalize_btf(struct bpf_linker *linker); 178static int finalize_btf_ext(struct bpf_linker *linker); 179 180void bpf_linker__free(struct bpf_linker *linker) 181{ 182 int i; 183 184 if (!linker) 185 return; 186 187 free(linker->filename); 188 189 if (linker->elf) 190 elf_end(linker->elf); 191 192 if (linker->fd >= 0) 193 close(linker->fd); 194 195 strset__free(linker->strtab_strs); 196 197 btf__free(linker->btf); 198 btf_ext__free(linker->btf_ext); 199 200 for (i = 1; i < linker->sec_cnt; i++) { 201 struct dst_sec *sec = &linker->secs[i]; 202 203 free(sec->sec_name); 204 free(sec->raw_data); 205 free(sec->sec_vars); 206 207 free(sec->func_info.recs); 208 free(sec->line_info.recs); 209 free(sec->core_relo_info.recs); 210 } 211 free(linker->secs); 212 213 free(linker->glob_syms); 214 free(linker); 215} 216 217struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts) 218{ 219 struct bpf_linker *linker; 220 int err; 221 222 if (!OPTS_VALID(opts, bpf_linker_opts)) 223 return errno = EINVAL, NULL; 224 225 if (elf_version(EV_CURRENT) == EV_NONE) { 226 pr_warn_elf("libelf initialization failed"); 227 return errno = EINVAL, NULL; 228 } 229 230 linker = calloc(1, sizeof(*linker)); 231 if (!linker) 232 return errno = ENOMEM, NULL; 233 234 linker->fd = -1; 235 236 err = init_output_elf(linker, filename); 237 if (err) 238 goto err_out; 239 240 return linker; 241 242err_out: 243 bpf_linker__free(linker); 244 return errno = -err, NULL; 245} 246 247static struct dst_sec *add_dst_sec(struct bpf_linker *linker, const char *sec_name) 248{ 249 struct dst_sec *secs = linker->secs, *sec; 250 size_t new_cnt = linker->sec_cnt ? linker->sec_cnt + 1 : 2; 251 252 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 253 if (!secs) 254 return NULL; 255 256 /* zero out newly allocated memory */ 257 memset(secs + linker->sec_cnt, 0, (new_cnt - linker->sec_cnt) * sizeof(*secs)); 258 259 linker->secs = secs; 260 linker->sec_cnt = new_cnt; 261 262 sec = &linker->secs[new_cnt - 1]; 263 sec->id = new_cnt - 1; 264 sec->sec_name = strdup(sec_name); 265 if (!sec->sec_name) 266 return NULL; 267 268 return sec; 269} 270 271static Elf64_Sym *add_new_sym(struct bpf_linker *linker, size_t *sym_idx) 272{ 273 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; 274 Elf64_Sym *syms, *sym; 275 size_t sym_cnt = symtab->sec_sz / sizeof(*sym); 276 277 syms = libbpf_reallocarray(symtab->raw_data, sym_cnt + 1, sizeof(*sym)); 278 if (!syms) 279 return NULL; 280 281 sym = &syms[sym_cnt]; 282 memset(sym, 0, sizeof(*sym)); 283 284 symtab->raw_data = syms; 285 symtab->sec_sz += sizeof(*sym); 286 symtab->shdr->sh_size += sizeof(*sym); 287 symtab->data->d_size += sizeof(*sym); 288 289 if (sym_idx) 290 *sym_idx = sym_cnt; 291 292 return sym; 293} 294 295static int init_output_elf(struct bpf_linker *linker, const char *file) 296{ 297 int err, str_off; 298 Elf64_Sym *init_sym; 299 struct dst_sec *sec; 300 301 linker->filename = strdup(file); 302 if (!linker->filename) 303 return -ENOMEM; 304 305 linker->fd = open(file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); 306 if (linker->fd < 0) { 307 err = -errno; 308 pr_warn("failed to create '%s': %d\n", file, err); 309 return err; 310 } 311 312 linker->elf = elf_begin(linker->fd, ELF_C_WRITE, NULL); 313 if (!linker->elf) { 314 pr_warn_elf("failed to create ELF object"); 315 return -EINVAL; 316 } 317 318 /* ELF header */ 319 linker->elf_hdr = elf64_newehdr(linker->elf); 320 if (!linker->elf_hdr) { 321 pr_warn_elf("failed to create ELF header"); 322 return -EINVAL; 323 } 324 325 linker->elf_hdr->e_machine = EM_BPF; 326 linker->elf_hdr->e_type = ET_REL; 327#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 328 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2LSB; 329#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 330 linker->elf_hdr->e_ident[EI_DATA] = ELFDATA2MSB; 331#else 332#error "Unknown __BYTE_ORDER__" 333#endif 334 335 /* STRTAB */ 336 /* initialize strset with an empty string to conform to ELF */ 337 linker->strtab_strs = strset__new(INT_MAX, "", sizeof("")); 338 if (libbpf_get_error(linker->strtab_strs)) 339 return libbpf_get_error(linker->strtab_strs); 340 341 sec = add_dst_sec(linker, ".strtab"); 342 if (!sec) 343 return -ENOMEM; 344 345 sec->scn = elf_newscn(linker->elf); 346 if (!sec->scn) { 347 pr_warn_elf("failed to create STRTAB section"); 348 return -EINVAL; 349 } 350 351 sec->shdr = elf64_getshdr(sec->scn); 352 if (!sec->shdr) 353 return -EINVAL; 354 355 sec->data = elf_newdata(sec->scn); 356 if (!sec->data) { 357 pr_warn_elf("failed to create STRTAB data"); 358 return -EINVAL; 359 } 360 361 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 362 if (str_off < 0) 363 return str_off; 364 365 sec->sec_idx = elf_ndxscn(sec->scn); 366 linker->elf_hdr->e_shstrndx = sec->sec_idx; 367 linker->strtab_sec_idx = sec->sec_idx; 368 369 sec->shdr->sh_name = str_off; 370 sec->shdr->sh_type = SHT_STRTAB; 371 sec->shdr->sh_flags = SHF_STRINGS; 372 sec->shdr->sh_offset = 0; 373 sec->shdr->sh_link = 0; 374 sec->shdr->sh_info = 0; 375 sec->shdr->sh_addralign = 1; 376 sec->shdr->sh_size = sec->sec_sz = 0; 377 sec->shdr->sh_entsize = 0; 378 379 /* SYMTAB */ 380 sec = add_dst_sec(linker, ".symtab"); 381 if (!sec) 382 return -ENOMEM; 383 384 sec->scn = elf_newscn(linker->elf); 385 if (!sec->scn) { 386 pr_warn_elf("failed to create SYMTAB section"); 387 return -EINVAL; 388 } 389 390 sec->shdr = elf64_getshdr(sec->scn); 391 if (!sec->shdr) 392 return -EINVAL; 393 394 sec->data = elf_newdata(sec->scn); 395 if (!sec->data) { 396 pr_warn_elf("failed to create SYMTAB data"); 397 return -EINVAL; 398 } 399 400 str_off = strset__add_str(linker->strtab_strs, sec->sec_name); 401 if (str_off < 0) 402 return str_off; 403 404 sec->sec_idx = elf_ndxscn(sec->scn); 405 linker->symtab_sec_idx = sec->sec_idx; 406 407 sec->shdr->sh_name = str_off; 408 sec->shdr->sh_type = SHT_SYMTAB; 409 sec->shdr->sh_flags = 0; 410 sec->shdr->sh_offset = 0; 411 sec->shdr->sh_link = linker->strtab_sec_idx; 412 /* sh_info should be one greater than the index of the last local 413 * symbol (i.e., binding is STB_LOCAL). But why and who cares? 414 */ 415 sec->shdr->sh_info = 0; 416 sec->shdr->sh_addralign = 8; 417 sec->shdr->sh_entsize = sizeof(Elf64_Sym); 418 419 /* .BTF */ 420 linker->btf = btf__new_empty(); 421 err = libbpf_get_error(linker->btf); 422 if (err) 423 return err; 424 425 /* add the special all-zero symbol */ 426 init_sym = add_new_sym(linker, NULL); 427 if (!init_sym) 428 return -EINVAL; 429 430 init_sym->st_name = 0; 431 init_sym->st_info = 0; 432 init_sym->st_other = 0; 433 init_sym->st_shndx = SHN_UNDEF; 434 init_sym->st_value = 0; 435 init_sym->st_size = 0; 436 437 return 0; 438} 439 440int bpf_linker__add_file(struct bpf_linker *linker, const char *filename, 441 const struct bpf_linker_file_opts *opts) 442{ 443 struct src_obj obj = {}; 444 int err = 0; 445 446 if (!OPTS_VALID(opts, bpf_linker_file_opts)) 447 return libbpf_err(-EINVAL); 448 449 if (!linker->elf) 450 return libbpf_err(-EINVAL); 451 452 err = err ?: linker_load_obj_file(linker, filename, opts, &obj); 453 err = err ?: linker_append_sec_data(linker, &obj); 454 err = err ?: linker_append_elf_syms(linker, &obj); 455 err = err ?: linker_append_elf_relos(linker, &obj); 456 err = err ?: linker_append_btf(linker, &obj); 457 err = err ?: linker_append_btf_ext(linker, &obj); 458 459 /* free up src_obj resources */ 460 free(obj.btf_type_map); 461 btf__free(obj.btf); 462 btf_ext__free(obj.btf_ext); 463 free(obj.secs); 464 free(obj.sym_map); 465 if (obj.elf) 466 elf_end(obj.elf); 467 if (obj.fd >= 0) 468 close(obj.fd); 469 470 return libbpf_err(err); 471} 472 473static bool is_dwarf_sec_name(const char *name) 474{ 475 /* approximation, but the actual list is too long */ 476 return strncmp(name, ".debug_", sizeof(".debug_") - 1) == 0; 477} 478 479static bool is_ignored_sec(struct src_sec *sec) 480{ 481 Elf64_Shdr *shdr = sec->shdr; 482 const char *name = sec->sec_name; 483 484 /* no special handling of .strtab */ 485 if (shdr->sh_type == SHT_STRTAB) 486 return true; 487 488 /* ignore .llvm_addrsig section as well */ 489 if (shdr->sh_type == SHT_LLVM_ADDRSIG) 490 return true; 491 492 /* no subprograms will lead to an empty .text section, ignore it */ 493 if (shdr->sh_type == SHT_PROGBITS && shdr->sh_size == 0 && 494 strcmp(sec->sec_name, ".text") == 0) 495 return true; 496 497 /* DWARF sections */ 498 if (is_dwarf_sec_name(sec->sec_name)) 499 return true; 500 501 if (strncmp(name, ".rel", sizeof(".rel") - 1) == 0) { 502 name += sizeof(".rel") - 1; 503 /* DWARF section relocations */ 504 if (is_dwarf_sec_name(name)) 505 return true; 506 507 /* .BTF and .BTF.ext don't need relocations */ 508 if (strcmp(name, BTF_ELF_SEC) == 0 || 509 strcmp(name, BTF_EXT_ELF_SEC) == 0) 510 return true; 511 } 512 513 return false; 514} 515 516static struct src_sec *add_src_sec(struct src_obj *obj, const char *sec_name) 517{ 518 struct src_sec *secs = obj->secs, *sec; 519 size_t new_cnt = obj->sec_cnt ? obj->sec_cnt + 1 : 2; 520 521 secs = libbpf_reallocarray(secs, new_cnt, sizeof(*secs)); 522 if (!secs) 523 return NULL; 524 525 /* zero out newly allocated memory */ 526 memset(secs + obj->sec_cnt, 0, (new_cnt - obj->sec_cnt) * sizeof(*secs)); 527 528 obj->secs = secs; 529 obj->sec_cnt = new_cnt; 530 531 sec = &obj->secs[new_cnt - 1]; 532 sec->id = new_cnt - 1; 533 sec->sec_name = sec_name; 534 535 return sec; 536} 537 538static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, 539 const struct bpf_linker_file_opts *opts, 540 struct src_obj *obj) 541{ 542#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 543 const int host_endianness = ELFDATA2LSB; 544#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 545 const int host_endianness = ELFDATA2MSB; 546#else 547#error "Unknown __BYTE_ORDER__" 548#endif 549 int err = 0; 550 Elf_Scn *scn; 551 Elf_Data *data; 552 Elf64_Ehdr *ehdr; 553 Elf64_Shdr *shdr; 554 struct src_sec *sec; 555 556 pr_debug("linker: adding object file '%s'...\n", filename); 557 558 obj->filename = filename; 559 560 obj->fd = open(filename, O_RDONLY | O_CLOEXEC); 561 if (obj->fd < 0) { 562 err = -errno; 563 pr_warn("failed to open file '%s': %d\n", filename, err); 564 return err; 565 } 566 obj->elf = elf_begin(obj->fd, ELF_C_READ_MMAP, NULL); 567 if (!obj->elf) { 568 err = -errno; 569 pr_warn_elf("failed to parse ELF file '%s'", filename); 570 return err; 571 } 572 573 /* Sanity check ELF file high-level properties */ 574 ehdr = elf64_getehdr(obj->elf); 575 if (!ehdr) { 576 err = -errno; 577 pr_warn_elf("failed to get ELF header for %s", filename); 578 return err; 579 } 580 if (ehdr->e_ident[EI_DATA] != host_endianness) { 581 err = -EOPNOTSUPP; 582 pr_warn_elf("unsupported byte order of ELF file %s", filename); 583 return err; 584 } 585 if (ehdr->e_type != ET_REL 586 || ehdr->e_machine != EM_BPF 587 || ehdr->e_ident[EI_CLASS] != ELFCLASS64) { 588 err = -EOPNOTSUPP; 589 pr_warn_elf("unsupported kind of ELF file %s", filename); 590 return err; 591 } 592 593 if (elf_getshdrstrndx(obj->elf, &obj->shstrs_sec_idx)) { 594 err = -errno; 595 pr_warn_elf("failed to get SHSTRTAB section index for %s", filename); 596 return err; 597 } 598 599 scn = NULL; 600 while ((scn = elf_nextscn(obj->elf, scn)) != NULL) { 601 size_t sec_idx = elf_ndxscn(scn); 602 const char *sec_name; 603 604 shdr = elf64_getshdr(scn); 605 if (!shdr) { 606 err = -errno; 607 pr_warn_elf("failed to get section #%zu header for %s", 608 sec_idx, filename); 609 return err; 610 } 611 612 sec_name = elf_strptr(obj->elf, obj->shstrs_sec_idx, shdr->sh_name); 613 if (!sec_name) { 614 err = -errno; 615 pr_warn_elf("failed to get section #%zu name for %s", 616 sec_idx, filename); 617 return err; 618 } 619 620 data = elf_getdata(scn, 0); 621 if (!data) { 622 err = -errno; 623 pr_warn_elf("failed to get section #%zu (%s) data from %s", 624 sec_idx, sec_name, filename); 625 return err; 626 } 627 628 sec = add_src_sec(obj, sec_name); 629 if (!sec) 630 return -ENOMEM; 631 632 sec->scn = scn; 633 sec->shdr = shdr; 634 sec->data = data; 635 sec->sec_idx = elf_ndxscn(scn); 636 637 if (is_ignored_sec(sec)) { 638 sec->skipped = true; 639 continue; 640 } 641 642 switch (shdr->sh_type) { 643 case SHT_SYMTAB: 644 if (obj->symtab_sec_idx) { 645 err = -EOPNOTSUPP; 646 pr_warn("multiple SYMTAB sections found, not supported\n"); 647 return err; 648 } 649 obj->symtab_sec_idx = sec_idx; 650 break; 651 case SHT_STRTAB: 652 /* we'll construct our own string table */ 653 break; 654 case SHT_PROGBITS: 655 if (strcmp(sec_name, BTF_ELF_SEC) == 0) { 656 obj->btf = btf__new(data->d_buf, shdr->sh_size); 657 err = libbpf_get_error(obj->btf); 658 if (err) { 659 pr_warn("failed to parse .BTF from %s: %d\n", filename, err); 660 return err; 661 } 662 sec->skipped = true; 663 continue; 664 } 665 if (strcmp(sec_name, BTF_EXT_ELF_SEC) == 0) { 666 obj->btf_ext = btf_ext__new(data->d_buf, shdr->sh_size); 667 err = libbpf_get_error(obj->btf_ext); 668 if (err) { 669 pr_warn("failed to parse .BTF.ext from '%s': %d\n", filename, err); 670 return err; 671 } 672 sec->skipped = true; 673 continue; 674 } 675 676 /* data & code */ 677 break; 678 case SHT_NOBITS: 679 /* BSS */ 680 break; 681 case SHT_REL: 682 /* relocations */ 683 break; 684 default: 685 pr_warn("unrecognized section #%zu (%s) in %s\n", 686 sec_idx, sec_name, filename); 687 err = -EINVAL; 688 return err; 689 } 690 } 691 692 err = err ?: linker_sanity_check_elf(obj); 693 err = err ?: linker_sanity_check_btf(obj); 694 err = err ?: linker_sanity_check_btf_ext(obj); 695 err = err ?: linker_fixup_btf(obj); 696 697 return err; 698} 699 700static int linker_sanity_check_elf(struct src_obj *obj) 701{ 702 struct src_sec *sec; 703 int i, err; 704 705 if (!obj->symtab_sec_idx) { 706 pr_warn("ELF is missing SYMTAB section in %s\n", obj->filename); 707 return -EINVAL; 708 } 709 if (!obj->shstrs_sec_idx) { 710 pr_warn("ELF is missing section headers STRTAB section in %s\n", obj->filename); 711 return -EINVAL; 712 } 713 714 for (i = 1; i < obj->sec_cnt; i++) { 715 sec = &obj->secs[i]; 716 717 if (sec->sec_name[0] == '\0') { 718 pr_warn("ELF section #%zu has empty name in %s\n", sec->sec_idx, obj->filename); 719 return -EINVAL; 720 } 721 722 if (sec->shdr->sh_addralign && !is_pow_of_2(sec->shdr->sh_addralign)) 723 return -EINVAL; 724 if (sec->shdr->sh_addralign != sec->data->d_align) 725 return -EINVAL; 726 727 if (sec->shdr->sh_size != sec->data->d_size) 728 return -EINVAL; 729 730 switch (sec->shdr->sh_type) { 731 case SHT_SYMTAB: 732 err = linker_sanity_check_elf_symtab(obj, sec); 733 if (err) 734 return err; 735 break; 736 case SHT_STRTAB: 737 break; 738 case SHT_PROGBITS: 739 if (sec->shdr->sh_flags & SHF_EXECINSTR) { 740 if (sec->shdr->sh_size % sizeof(struct bpf_insn) != 0) 741 return -EINVAL; 742 } 743 break; 744 case SHT_NOBITS: 745 break; 746 case SHT_REL: 747 err = linker_sanity_check_elf_relos(obj, sec); 748 if (err) 749 return err; 750 break; 751 case SHT_LLVM_ADDRSIG: 752 break; 753 default: 754 pr_warn("ELF section #%zu (%s) has unrecognized type %zu in %s\n", 755 sec->sec_idx, sec->sec_name, (size_t)sec->shdr->sh_type, obj->filename); 756 return -EINVAL; 757 } 758 } 759 760 return 0; 761} 762 763static int linker_sanity_check_elf_symtab(struct src_obj *obj, struct src_sec *sec) 764{ 765 struct src_sec *link_sec; 766 Elf64_Sym *sym; 767 int i, n; 768 769 if (sec->shdr->sh_entsize != sizeof(Elf64_Sym)) 770 return -EINVAL; 771 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 772 return -EINVAL; 773 774 if (!sec->shdr->sh_link || sec->shdr->sh_link >= obj->sec_cnt) { 775 pr_warn("ELF SYMTAB section #%zu points to missing STRTAB section #%zu in %s\n", 776 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 777 return -EINVAL; 778 } 779 link_sec = &obj->secs[sec->shdr->sh_link]; 780 if (link_sec->shdr->sh_type != SHT_STRTAB) { 781 pr_warn("ELF SYMTAB section #%zu points to invalid STRTAB section #%zu in %s\n", 782 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 783 return -EINVAL; 784 } 785 786 n = sec->shdr->sh_size / sec->shdr->sh_entsize; 787 sym = sec->data->d_buf; 788 for (i = 0; i < n; i++, sym++) { 789 int sym_type = ELF64_ST_TYPE(sym->st_info); 790 int sym_bind = ELF64_ST_BIND(sym->st_info); 791 int sym_vis = ELF64_ST_VISIBILITY(sym->st_other); 792 793 if (i == 0) { 794 if (sym->st_name != 0 || sym->st_info != 0 795 || sym->st_other != 0 || sym->st_shndx != 0 796 || sym->st_value != 0 || sym->st_size != 0) { 797 pr_warn("ELF sym #0 is invalid in %s\n", obj->filename); 798 return -EINVAL; 799 } 800 continue; 801 } 802 if (sym_bind != STB_LOCAL && sym_bind != STB_GLOBAL && sym_bind != STB_WEAK) { 803 pr_warn("ELF sym #%d in section #%zu has unsupported symbol binding %d\n", 804 i, sec->sec_idx, sym_bind); 805 return -EINVAL; 806 } 807 if (sym_vis != STV_DEFAULT && sym_vis != STV_HIDDEN) { 808 pr_warn("ELF sym #%d in section #%zu has unsupported symbol visibility %d\n", 809 i, sec->sec_idx, sym_vis); 810 return -EINVAL; 811 } 812 if (sym->st_shndx == 0) { 813 if (sym_type != STT_NOTYPE || sym_bind == STB_LOCAL 814 || sym->st_value != 0 || sym->st_size != 0) { 815 pr_warn("ELF sym #%d is invalid extern symbol in %s\n", 816 i, obj->filename); 817 818 return -EINVAL; 819 } 820 continue; 821 } 822 if (sym->st_shndx < SHN_LORESERVE && sym->st_shndx >= obj->sec_cnt) { 823 pr_warn("ELF sym #%d in section #%zu points to missing section #%zu in %s\n", 824 i, sec->sec_idx, (size_t)sym->st_shndx, obj->filename); 825 return -EINVAL; 826 } 827 if (sym_type == STT_SECTION) { 828 if (sym->st_value != 0) 829 return -EINVAL; 830 continue; 831 } 832 } 833 834 return 0; 835} 836 837static int linker_sanity_check_elf_relos(struct src_obj *obj, struct src_sec *sec) 838{ 839 struct src_sec *link_sec, *sym_sec; 840 Elf64_Rel *relo; 841 int i, n; 842 843 if (sec->shdr->sh_entsize != sizeof(Elf64_Rel)) 844 return -EINVAL; 845 if (sec->shdr->sh_size % sec->shdr->sh_entsize != 0) 846 return -EINVAL; 847 848 /* SHT_REL's sh_link should point to SYMTAB */ 849 if (sec->shdr->sh_link != obj->symtab_sec_idx) { 850 pr_warn("ELF relo section #%zu points to invalid SYMTAB section #%zu in %s\n", 851 sec->sec_idx, (size_t)sec->shdr->sh_link, obj->filename); 852 return -EINVAL; 853 } 854 855 /* SHT_REL's sh_info points to relocated section */ 856 if (!sec->shdr->sh_info || sec->shdr->sh_info >= obj->sec_cnt) { 857 pr_warn("ELF relo section #%zu points to missing section #%zu in %s\n", 858 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 859 return -EINVAL; 860 } 861 link_sec = &obj->secs[sec->shdr->sh_info]; 862 863 /* .rel<secname> -> <secname> pattern is followed */ 864 if (strncmp(sec->sec_name, ".rel", sizeof(".rel") - 1) != 0 865 || strcmp(sec->sec_name + sizeof(".rel") - 1, link_sec->sec_name) != 0) { 866 pr_warn("ELF relo section #%zu name has invalid name in %s\n", 867 sec->sec_idx, obj->filename); 868 return -EINVAL; 869 } 870 871 /* don't further validate relocations for ignored sections */ 872 if (link_sec->skipped) 873 return 0; 874 875 /* relocatable section is data or instructions */ 876 if (link_sec->shdr->sh_type != SHT_PROGBITS && link_sec->shdr->sh_type != SHT_NOBITS) { 877 pr_warn("ELF relo section #%zu points to invalid section #%zu in %s\n", 878 sec->sec_idx, (size_t)sec->shdr->sh_info, obj->filename); 879 return -EINVAL; 880 } 881 882 /* check sanity of each relocation */ 883 n = sec->shdr->sh_size / sec->shdr->sh_entsize; 884 relo = sec->data->d_buf; 885 sym_sec = &obj->secs[obj->symtab_sec_idx]; 886 for (i = 0; i < n; i++, relo++) { 887 size_t sym_idx = ELF64_R_SYM(relo->r_info); 888 size_t sym_type = ELF64_R_TYPE(relo->r_info); 889 890 if (sym_type != R_BPF_64_64 && sym_type != R_BPF_64_32 && 891 sym_type != R_BPF_64_ABS64 && sym_type != R_BPF_64_ABS32) { 892 pr_warn("ELF relo #%d in section #%zu has unexpected type %zu in %s\n", 893 i, sec->sec_idx, sym_type, obj->filename); 894 return -EINVAL; 895 } 896 897 if (!sym_idx || sym_idx * sizeof(Elf64_Sym) >= sym_sec->shdr->sh_size) { 898 pr_warn("ELF relo #%d in section #%zu points to invalid symbol #%zu in %s\n", 899 i, sec->sec_idx, sym_idx, obj->filename); 900 return -EINVAL; 901 } 902 903 if (link_sec->shdr->sh_flags & SHF_EXECINSTR) { 904 if (relo->r_offset % sizeof(struct bpf_insn) != 0) { 905 pr_warn("ELF relo #%d in section #%zu points to missing symbol #%zu in %s\n", 906 i, sec->sec_idx, sym_idx, obj->filename); 907 return -EINVAL; 908 } 909 } 910 } 911 912 return 0; 913} 914 915static int check_btf_type_id(__u32 *type_id, void *ctx) 916{ 917 struct btf *btf = ctx; 918 919 if (*type_id >= btf__type_cnt(btf)) 920 return -EINVAL; 921 922 return 0; 923} 924 925static int check_btf_str_off(__u32 *str_off, void *ctx) 926{ 927 struct btf *btf = ctx; 928 const char *s; 929 930 s = btf__str_by_offset(btf, *str_off); 931 932 if (!s) 933 return -EINVAL; 934 935 return 0; 936} 937 938static int linker_sanity_check_btf(struct src_obj *obj) 939{ 940 struct btf_type *t; 941 int i, n, err = 0; 942 943 if (!obj->btf) 944 return 0; 945 946 n = btf__type_cnt(obj->btf); 947 for (i = 1; i < n; i++) { 948 t = btf_type_by_id(obj->btf, i); 949 950 err = err ?: btf_type_visit_type_ids(t, check_btf_type_id, obj->btf); 951 err = err ?: btf_type_visit_str_offs(t, check_btf_str_off, obj->btf); 952 if (err) 953 return err; 954 } 955 956 return 0; 957} 958 959static int linker_sanity_check_btf_ext(struct src_obj *obj) 960{ 961 int err = 0; 962 963 if (!obj->btf_ext) 964 return 0; 965 966 /* can't use .BTF.ext without .BTF */ 967 if (!obj->btf) 968 return -EINVAL; 969 970 err = err ?: btf_ext_visit_type_ids(obj->btf_ext, check_btf_type_id, obj->btf); 971 err = err ?: btf_ext_visit_str_offs(obj->btf_ext, check_btf_str_off, obj->btf); 972 if (err) 973 return err; 974 975 return 0; 976} 977 978static int init_sec(struct bpf_linker *linker, struct dst_sec *dst_sec, struct src_sec *src_sec) 979{ 980 Elf_Scn *scn; 981 Elf_Data *data; 982 Elf64_Shdr *shdr; 983 int name_off; 984 985 dst_sec->sec_sz = 0; 986 dst_sec->sec_idx = 0; 987 dst_sec->ephemeral = src_sec->ephemeral; 988 989 /* ephemeral sections are just thin section shells lacking most parts */ 990 if (src_sec->ephemeral) 991 return 0; 992 993 scn = elf_newscn(linker->elf); 994 if (!scn) 995 return -ENOMEM; 996 data = elf_newdata(scn); 997 if (!data) 998 return -ENOMEM; 999 shdr = elf64_getshdr(scn); 1000 if (!shdr) 1001 return -ENOMEM; 1002 1003 dst_sec->scn = scn; 1004 dst_sec->shdr = shdr; 1005 dst_sec->data = data; 1006 dst_sec->sec_idx = elf_ndxscn(scn); 1007 1008 name_off = strset__add_str(linker->strtab_strs, src_sec->sec_name); 1009 if (name_off < 0) 1010 return name_off; 1011 1012 shdr->sh_name = name_off; 1013 shdr->sh_type = src_sec->shdr->sh_type; 1014 shdr->sh_flags = src_sec->shdr->sh_flags; 1015 shdr->sh_size = 0; 1016 /* sh_link and sh_info have different meaning for different types of 1017 * sections, so we leave it up to the caller code to fill them in, if 1018 * necessary 1019 */ 1020 shdr->sh_link = 0; 1021 shdr->sh_info = 0; 1022 shdr->sh_addralign = src_sec->shdr->sh_addralign; 1023 shdr->sh_entsize = src_sec->shdr->sh_entsize; 1024 1025 data->d_type = src_sec->data->d_type; 1026 data->d_size = 0; 1027 data->d_buf = NULL; 1028 data->d_align = src_sec->data->d_align; 1029 data->d_off = 0; 1030 1031 return 0; 1032} 1033 1034static struct dst_sec *find_dst_sec_by_name(struct bpf_linker *linker, const char *sec_name) 1035{ 1036 struct dst_sec *sec; 1037 int i; 1038 1039 for (i = 1; i < linker->sec_cnt; i++) { 1040 sec = &linker->secs[i]; 1041 1042 if (strcmp(sec->sec_name, sec_name) == 0) 1043 return sec; 1044 } 1045 1046 return NULL; 1047} 1048 1049static bool secs_match(struct dst_sec *dst, struct src_sec *src) 1050{ 1051 if (dst->ephemeral || src->ephemeral) 1052 return true; 1053 1054 if (dst->shdr->sh_type != src->shdr->sh_type) { 1055 pr_warn("sec %s types mismatch\n", dst->sec_name); 1056 return false; 1057 } 1058 if (dst->shdr->sh_flags != src->shdr->sh_flags) { 1059 pr_warn("sec %s flags mismatch\n", dst->sec_name); 1060 return false; 1061 } 1062 if (dst->shdr->sh_entsize != src->shdr->sh_entsize) { 1063 pr_warn("sec %s entsize mismatch\n", dst->sec_name); 1064 return false; 1065 } 1066 1067 return true; 1068} 1069 1070static bool sec_content_is_same(struct dst_sec *dst_sec, struct src_sec *src_sec) 1071{ 1072 if (dst_sec->sec_sz != src_sec->shdr->sh_size) 1073 return false; 1074 if (memcmp(dst_sec->raw_data, src_sec->data->d_buf, dst_sec->sec_sz) != 0) 1075 return false; 1076 return true; 1077} 1078 1079static int extend_sec(struct bpf_linker *linker, struct dst_sec *dst, struct src_sec *src) 1080{ 1081 void *tmp; 1082 size_t dst_align, src_align; 1083 size_t dst_align_sz, dst_final_sz; 1084 int err; 1085 1086 /* Ephemeral source section doesn't contribute anything to ELF 1087 * section data. 1088 */ 1089 if (src->ephemeral) 1090 return 0; 1091 1092 /* Some sections (like .maps) can contain both externs (and thus be 1093 * ephemeral) and non-externs (map definitions). So it's possible that 1094 * it has to be "upgraded" from ephemeral to non-ephemeral when the 1095 * first non-ephemeral entity appears. In such case, we add ELF 1096 * section, data, etc. 1097 */ 1098 if (dst->ephemeral) { 1099 err = init_sec(linker, dst, src); 1100 if (err) 1101 return err; 1102 } 1103 1104 dst_align = dst->shdr->sh_addralign; 1105 src_align = src->shdr->sh_addralign; 1106 if (dst_align == 0) 1107 dst_align = 1; 1108 if (dst_align < src_align) 1109 dst_align = src_align; 1110 1111 dst_align_sz = (dst->sec_sz + dst_align - 1) / dst_align * dst_align; 1112 1113 /* no need to re-align final size */ 1114 dst_final_sz = dst_align_sz + src->shdr->sh_size; 1115 1116 if (src->shdr->sh_type != SHT_NOBITS) { 1117 tmp = realloc(dst->raw_data, dst_final_sz); 1118 /* If dst_align_sz == 0, realloc() behaves in a special way: 1119 * 1. When dst->raw_data is NULL it returns: 1120 * "either NULL or a pointer suitable to be passed to free()" [1]. 1121 * 2. When dst->raw_data is not-NULL it frees dst->raw_data and returns NULL, 1122 * thus invalidating any "pointer suitable to be passed to free()" obtained 1123 * at step (1). 1124 * 1125 * The dst_align_sz > 0 check avoids error exit after (2), otherwise 1126 * dst->raw_data would be freed again in bpf_linker__free(). 1127 * 1128 * [1] man 3 realloc 1129 */ 1130 if (!tmp && dst_align_sz > 0) 1131 return -ENOMEM; 1132 dst->raw_data = tmp; 1133 1134 /* pad dst section, if it's alignment forced size increase */ 1135 memset(dst->raw_data + dst->sec_sz, 0, dst_align_sz - dst->sec_sz); 1136 /* now copy src data at a properly aligned offset */ 1137 memcpy(dst->raw_data + dst_align_sz, src->data->d_buf, src->shdr->sh_size); 1138 } 1139 1140 dst->sec_sz = dst_final_sz; 1141 dst->shdr->sh_size = dst_final_sz; 1142 dst->data->d_size = dst_final_sz; 1143 1144 dst->shdr->sh_addralign = dst_align; 1145 dst->data->d_align = dst_align; 1146 1147 src->dst_off = dst_align_sz; 1148 1149 return 0; 1150} 1151 1152static bool is_data_sec(struct src_sec *sec) 1153{ 1154 if (!sec || sec->skipped) 1155 return false; 1156 /* ephemeral sections are data sections, e.g., .kconfig, .ksyms */ 1157 if (sec->ephemeral) 1158 return true; 1159 return sec->shdr->sh_type == SHT_PROGBITS || sec->shdr->sh_type == SHT_NOBITS; 1160} 1161 1162static bool is_relo_sec(struct src_sec *sec) 1163{ 1164 if (!sec || sec->skipped || sec->ephemeral) 1165 return false; 1166 return sec->shdr->sh_type == SHT_REL; 1167} 1168 1169static int linker_append_sec_data(struct bpf_linker *linker, struct src_obj *obj) 1170{ 1171 int i, err; 1172 1173 for (i = 1; i < obj->sec_cnt; i++) { 1174 struct src_sec *src_sec; 1175 struct dst_sec *dst_sec; 1176 1177 src_sec = &obj->secs[i]; 1178 if (!is_data_sec(src_sec)) 1179 continue; 1180 1181 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 1182 if (!dst_sec) { 1183 dst_sec = add_dst_sec(linker, src_sec->sec_name); 1184 if (!dst_sec) 1185 return -ENOMEM; 1186 err = init_sec(linker, dst_sec, src_sec); 1187 if (err) { 1188 pr_warn("failed to init section '%s'\n", src_sec->sec_name); 1189 return err; 1190 } 1191 } else { 1192 if (!secs_match(dst_sec, src_sec)) { 1193 pr_warn("ELF sections %s are incompatible\n", src_sec->sec_name); 1194 return -1; 1195 } 1196 1197 /* "license" and "version" sections are deduped */ 1198 if (strcmp(src_sec->sec_name, "license") == 0 1199 || strcmp(src_sec->sec_name, "version") == 0) { 1200 if (!sec_content_is_same(dst_sec, src_sec)) { 1201 pr_warn("non-identical contents of section '%s' are not supported\n", src_sec->sec_name); 1202 return -EINVAL; 1203 } 1204 src_sec->skipped = true; 1205 src_sec->dst_id = dst_sec->id; 1206 continue; 1207 } 1208 } 1209 1210 /* record mapped section index */ 1211 src_sec->dst_id = dst_sec->id; 1212 1213 err = extend_sec(linker, dst_sec, src_sec); 1214 if (err) 1215 return err; 1216 } 1217 1218 return 0; 1219} 1220 1221static int linker_append_elf_syms(struct bpf_linker *linker, struct src_obj *obj) 1222{ 1223 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 1224 Elf64_Sym *sym = symtab->data->d_buf; 1225 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize, err; 1226 int str_sec_idx = symtab->shdr->sh_link; 1227 const char *sym_name; 1228 1229 obj->sym_map = calloc(n + 1, sizeof(*obj->sym_map)); 1230 if (!obj->sym_map) 1231 return -ENOMEM; 1232 1233 for (i = 0; i < n; i++, sym++) { 1234 /* We already validated all-zero symbol #0 and we already 1235 * appended it preventively to the final SYMTAB, so skip it. 1236 */ 1237 if (i == 0) 1238 continue; 1239 1240 sym_name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 1241 if (!sym_name) { 1242 pr_warn("can't fetch symbol name for symbol #%d in '%s'\n", i, obj->filename); 1243 return -EINVAL; 1244 } 1245 1246 err = linker_append_elf_sym(linker, obj, sym, sym_name, i); 1247 if (err) 1248 return err; 1249 } 1250 1251 return 0; 1252} 1253 1254static Elf64_Sym *get_sym_by_idx(struct bpf_linker *linker, size_t sym_idx) 1255{ 1256 struct dst_sec *symtab = &linker->secs[linker->symtab_sec_idx]; 1257 Elf64_Sym *syms = symtab->raw_data; 1258 1259 return &syms[sym_idx]; 1260} 1261 1262static struct glob_sym *find_glob_sym(struct bpf_linker *linker, const char *sym_name) 1263{ 1264 struct glob_sym *glob_sym; 1265 const char *name; 1266 int i; 1267 1268 for (i = 0; i < linker->glob_sym_cnt; i++) { 1269 glob_sym = &linker->glob_syms[i]; 1270 name = strset__data(linker->strtab_strs) + glob_sym->name_off; 1271 1272 if (strcmp(name, sym_name) == 0) 1273 return glob_sym; 1274 } 1275 1276 return NULL; 1277} 1278 1279static struct glob_sym *add_glob_sym(struct bpf_linker *linker) 1280{ 1281 struct glob_sym *syms, *sym; 1282 1283 syms = libbpf_reallocarray(linker->glob_syms, linker->glob_sym_cnt + 1, 1284 sizeof(*linker->glob_syms)); 1285 if (!syms) 1286 return NULL; 1287 1288 sym = &syms[linker->glob_sym_cnt]; 1289 memset(sym, 0, sizeof(*sym)); 1290 sym->var_idx = -1; 1291 1292 linker->glob_syms = syms; 1293 linker->glob_sym_cnt++; 1294 1295 return sym; 1296} 1297 1298static bool glob_sym_btf_matches(const char *sym_name, bool exact, 1299 const struct btf *btf1, __u32 id1, 1300 const struct btf *btf2, __u32 id2) 1301{ 1302 const struct btf_type *t1, *t2; 1303 bool is_static1, is_static2; 1304 const char *n1, *n2; 1305 int i, n; 1306 1307recur: 1308 n1 = n2 = NULL; 1309 t1 = skip_mods_and_typedefs(btf1, id1, &id1); 1310 t2 = skip_mods_and_typedefs(btf2, id2, &id2); 1311 1312 /* check if only one side is FWD, otherwise handle with common logic */ 1313 if (!exact && btf_is_fwd(t1) != btf_is_fwd(t2)) { 1314 n1 = btf__str_by_offset(btf1, t1->name_off); 1315 n2 = btf__str_by_offset(btf2, t2->name_off); 1316 if (strcmp(n1, n2) != 0) { 1317 pr_warn("global '%s': incompatible forward declaration names '%s' and '%s'\n", 1318 sym_name, n1, n2); 1319 return false; 1320 } 1321 /* validate if FWD kind matches concrete kind */ 1322 if (btf_is_fwd(t1)) { 1323 if (btf_kflag(t1) && btf_is_union(t2)) 1324 return true; 1325 if (!btf_kflag(t1) && btf_is_struct(t2)) 1326 return true; 1327 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n", 1328 sym_name, btf_kflag(t1) ? "union" : "struct", btf_kind_str(t2)); 1329 } else { 1330 if (btf_kflag(t2) && btf_is_union(t1)) 1331 return true; 1332 if (!btf_kflag(t2) && btf_is_struct(t1)) 1333 return true; 1334 pr_warn("global '%s': incompatible %s forward declaration and concrete kind %s\n", 1335 sym_name, btf_kflag(t2) ? "union" : "struct", btf_kind_str(t1)); 1336 } 1337 return false; 1338 } 1339 1340 if (btf_kind(t1) != btf_kind(t2)) { 1341 pr_warn("global '%s': incompatible BTF kinds %s and %s\n", 1342 sym_name, btf_kind_str(t1), btf_kind_str(t2)); 1343 return false; 1344 } 1345 1346 switch (btf_kind(t1)) { 1347 case BTF_KIND_STRUCT: 1348 case BTF_KIND_UNION: 1349 case BTF_KIND_ENUM: 1350 case BTF_KIND_ENUM64: 1351 case BTF_KIND_FWD: 1352 case BTF_KIND_FUNC: 1353 case BTF_KIND_VAR: 1354 n1 = btf__str_by_offset(btf1, t1->name_off); 1355 n2 = btf__str_by_offset(btf2, t2->name_off); 1356 if (strcmp(n1, n2) != 0) { 1357 pr_warn("global '%s': incompatible %s names '%s' and '%s'\n", 1358 sym_name, btf_kind_str(t1), n1, n2); 1359 return false; 1360 } 1361 break; 1362 default: 1363 break; 1364 } 1365 1366 switch (btf_kind(t1)) { 1367 case BTF_KIND_UNKN: /* void */ 1368 case BTF_KIND_FWD: 1369 return true; 1370 case BTF_KIND_INT: 1371 case BTF_KIND_FLOAT: 1372 case BTF_KIND_ENUM: 1373 case BTF_KIND_ENUM64: 1374 /* ignore encoding for int and enum values for enum */ 1375 if (t1->size != t2->size) { 1376 pr_warn("global '%s': incompatible %s '%s' size %u and %u\n", 1377 sym_name, btf_kind_str(t1), n1, t1->size, t2->size); 1378 return false; 1379 } 1380 return true; 1381 case BTF_KIND_PTR: 1382 /* just validate overall shape of the referenced type, so no 1383 * contents comparison for struct/union, and allowd fwd vs 1384 * struct/union 1385 */ 1386 exact = false; 1387 id1 = t1->type; 1388 id2 = t2->type; 1389 goto recur; 1390 case BTF_KIND_ARRAY: 1391 /* ignore index type and array size */ 1392 id1 = btf_array(t1)->type; 1393 id2 = btf_array(t2)->type; 1394 goto recur; 1395 case BTF_KIND_FUNC: 1396 /* extern and global linkages are compatible */ 1397 is_static1 = btf_func_linkage(t1) == BTF_FUNC_STATIC; 1398 is_static2 = btf_func_linkage(t2) == BTF_FUNC_STATIC; 1399 if (is_static1 != is_static2) { 1400 pr_warn("global '%s': incompatible func '%s' linkage\n", sym_name, n1); 1401 return false; 1402 } 1403 1404 id1 = t1->type; 1405 id2 = t2->type; 1406 goto recur; 1407 case BTF_KIND_VAR: 1408 /* extern and global linkages are compatible */ 1409 is_static1 = btf_var(t1)->linkage == BTF_VAR_STATIC; 1410 is_static2 = btf_var(t2)->linkage == BTF_VAR_STATIC; 1411 if (is_static1 != is_static2) { 1412 pr_warn("global '%s': incompatible var '%s' linkage\n", sym_name, n1); 1413 return false; 1414 } 1415 1416 id1 = t1->type; 1417 id2 = t2->type; 1418 goto recur; 1419 case BTF_KIND_STRUCT: 1420 case BTF_KIND_UNION: { 1421 const struct btf_member *m1, *m2; 1422 1423 if (!exact) 1424 return true; 1425 1426 if (btf_vlen(t1) != btf_vlen(t2)) { 1427 pr_warn("global '%s': incompatible number of %s fields %u and %u\n", 1428 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2)); 1429 return false; 1430 } 1431 1432 n = btf_vlen(t1); 1433 m1 = btf_members(t1); 1434 m2 = btf_members(t2); 1435 for (i = 0; i < n; i++, m1++, m2++) { 1436 n1 = btf__str_by_offset(btf1, m1->name_off); 1437 n2 = btf__str_by_offset(btf2, m2->name_off); 1438 if (strcmp(n1, n2) != 0) { 1439 pr_warn("global '%s': incompatible field #%d names '%s' and '%s'\n", 1440 sym_name, i, n1, n2); 1441 return false; 1442 } 1443 if (m1->offset != m2->offset) { 1444 pr_warn("global '%s': incompatible field #%d ('%s') offsets\n", 1445 sym_name, i, n1); 1446 return false; 1447 } 1448 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) 1449 return false; 1450 } 1451 1452 return true; 1453 } 1454 case BTF_KIND_FUNC_PROTO: { 1455 const struct btf_param *m1, *m2; 1456 1457 if (btf_vlen(t1) != btf_vlen(t2)) { 1458 pr_warn("global '%s': incompatible number of %s params %u and %u\n", 1459 sym_name, btf_kind_str(t1), btf_vlen(t1), btf_vlen(t2)); 1460 return false; 1461 } 1462 1463 n = btf_vlen(t1); 1464 m1 = btf_params(t1); 1465 m2 = btf_params(t2); 1466 for (i = 0; i < n; i++, m1++, m2++) { 1467 /* ignore func arg names */ 1468 if (!glob_sym_btf_matches(sym_name, exact, btf1, m1->type, btf2, m2->type)) 1469 return false; 1470 } 1471 1472 /* now check return type as well */ 1473 id1 = t1->type; 1474 id2 = t2->type; 1475 goto recur; 1476 } 1477 1478 /* skip_mods_and_typedefs() make this impossible */ 1479 case BTF_KIND_TYPEDEF: 1480 case BTF_KIND_VOLATILE: 1481 case BTF_KIND_CONST: 1482 case BTF_KIND_RESTRICT: 1483 /* DATASECs are never compared with each other */ 1484 case BTF_KIND_DATASEC: 1485 default: 1486 pr_warn("global '%s': unsupported BTF kind %s\n", 1487 sym_name, btf_kind_str(t1)); 1488 return false; 1489 } 1490} 1491 1492static bool map_defs_match(const char *sym_name, 1493 const struct btf *main_btf, 1494 const struct btf_map_def *main_def, 1495 const struct btf_map_def *main_inner_def, 1496 const struct btf *extra_btf, 1497 const struct btf_map_def *extra_def, 1498 const struct btf_map_def *extra_inner_def) 1499{ 1500 const char *reason; 1501 1502 if (main_def->map_type != extra_def->map_type) { 1503 reason = "type"; 1504 goto mismatch; 1505 } 1506 1507 /* check key type/size match */ 1508 if (main_def->key_size != extra_def->key_size) { 1509 reason = "key_size"; 1510 goto mismatch; 1511 } 1512 if (!!main_def->key_type_id != !!extra_def->key_type_id) { 1513 reason = "key type"; 1514 goto mismatch; 1515 } 1516 if ((main_def->parts & MAP_DEF_KEY_TYPE) 1517 && !glob_sym_btf_matches(sym_name, true /*exact*/, 1518 main_btf, main_def->key_type_id, 1519 extra_btf, extra_def->key_type_id)) { 1520 reason = "key type"; 1521 goto mismatch; 1522 } 1523 1524 /* validate value type/size match */ 1525 if (main_def->value_size != extra_def->value_size) { 1526 reason = "value_size"; 1527 goto mismatch; 1528 } 1529 if (!!main_def->value_type_id != !!extra_def->value_type_id) { 1530 reason = "value type"; 1531 goto mismatch; 1532 } 1533 if ((main_def->parts & MAP_DEF_VALUE_TYPE) 1534 && !glob_sym_btf_matches(sym_name, true /*exact*/, 1535 main_btf, main_def->value_type_id, 1536 extra_btf, extra_def->value_type_id)) { 1537 reason = "key type"; 1538 goto mismatch; 1539 } 1540 1541 if (main_def->max_entries != extra_def->max_entries) { 1542 reason = "max_entries"; 1543 goto mismatch; 1544 } 1545 if (main_def->map_flags != extra_def->map_flags) { 1546 reason = "map_flags"; 1547 goto mismatch; 1548 } 1549 if (main_def->numa_node != extra_def->numa_node) { 1550 reason = "numa_node"; 1551 goto mismatch; 1552 } 1553 if (main_def->pinning != extra_def->pinning) { 1554 reason = "pinning"; 1555 goto mismatch; 1556 } 1557 1558 if ((main_def->parts & MAP_DEF_INNER_MAP) != (extra_def->parts & MAP_DEF_INNER_MAP)) { 1559 reason = "inner map"; 1560 goto mismatch; 1561 } 1562 1563 if (main_def->parts & MAP_DEF_INNER_MAP) { 1564 char inner_map_name[128]; 1565 1566 snprintf(inner_map_name, sizeof(inner_map_name), "%s.inner", sym_name); 1567 1568 return map_defs_match(inner_map_name, 1569 main_btf, main_inner_def, NULL, 1570 extra_btf, extra_inner_def, NULL); 1571 } 1572 1573 return true; 1574 1575mismatch: 1576 pr_warn("global '%s': map %s mismatch\n", sym_name, reason); 1577 return false; 1578} 1579 1580static bool glob_map_defs_match(const char *sym_name, 1581 struct bpf_linker *linker, struct glob_sym *glob_sym, 1582 struct src_obj *obj, Elf64_Sym *sym, int btf_id) 1583{ 1584 struct btf_map_def dst_def = {}, dst_inner_def = {}; 1585 struct btf_map_def src_def = {}, src_inner_def = {}; 1586 const struct btf_type *t; 1587 int err; 1588 1589 t = btf__type_by_id(obj->btf, btf_id); 1590 if (!btf_is_var(t)) { 1591 pr_warn("global '%s': invalid map definition type [%d]\n", sym_name, btf_id); 1592 return false; 1593 } 1594 t = skip_mods_and_typedefs(obj->btf, t->type, NULL); 1595 1596 err = parse_btf_map_def(sym_name, obj->btf, t, true /*strict*/, &src_def, &src_inner_def); 1597 if (err) { 1598 pr_warn("global '%s': invalid map definition\n", sym_name); 1599 return false; 1600 } 1601 1602 /* re-parse existing map definition */ 1603 t = btf__type_by_id(linker->btf, glob_sym->btf_id); 1604 t = skip_mods_and_typedefs(linker->btf, t->type, NULL); 1605 err = parse_btf_map_def(sym_name, linker->btf, t, true /*strict*/, &dst_def, &dst_inner_def); 1606 if (err) { 1607 /* this should not happen, because we already validated it */ 1608 pr_warn("global '%s': invalid dst map definition\n", sym_name); 1609 return false; 1610 } 1611 1612 /* Currently extern map definition has to be complete and match 1613 * concrete map definition exactly. This restriction might be lifted 1614 * in the future. 1615 */ 1616 return map_defs_match(sym_name, linker->btf, &dst_def, &dst_inner_def, 1617 obj->btf, &src_def, &src_inner_def); 1618} 1619 1620static bool glob_syms_match(const char *sym_name, 1621 struct bpf_linker *linker, struct glob_sym *glob_sym, 1622 struct src_obj *obj, Elf64_Sym *sym, size_t sym_idx, int btf_id) 1623{ 1624 const struct btf_type *src_t; 1625 1626 /* if we are dealing with externs, BTF types describing both global 1627 * and extern VARs/FUNCs should be completely present in all files 1628 */ 1629 if (!glob_sym->btf_id || !btf_id) { 1630 pr_warn("BTF info is missing for global symbol '%s'\n", sym_name); 1631 return false; 1632 } 1633 1634 src_t = btf__type_by_id(obj->btf, btf_id); 1635 if (!btf_is_var(src_t) && !btf_is_func(src_t)) { 1636 pr_warn("only extern variables and functions are supported, but got '%s' for '%s'\n", 1637 btf_kind_str(src_t), sym_name); 1638 return false; 1639 } 1640 1641 /* deal with .maps definitions specially */ 1642 if (glob_sym->sec_id && strcmp(linker->secs[glob_sym->sec_id].sec_name, MAPS_ELF_SEC) == 0) 1643 return glob_map_defs_match(sym_name, linker, glob_sym, obj, sym, btf_id); 1644 1645 if (!glob_sym_btf_matches(sym_name, true /*exact*/, 1646 linker->btf, glob_sym->btf_id, obj->btf, btf_id)) 1647 return false; 1648 1649 return true; 1650} 1651 1652static bool btf_is_non_static(const struct btf_type *t) 1653{ 1654 return (btf_is_var(t) && btf_var(t)->linkage != BTF_VAR_STATIC) 1655 || (btf_is_func(t) && btf_func_linkage(t) != BTF_FUNC_STATIC); 1656} 1657 1658static int find_glob_sym_btf(struct src_obj *obj, Elf64_Sym *sym, const char *sym_name, 1659 int *out_btf_sec_id, int *out_btf_id) 1660{ 1661 int i, j, n, m, btf_id = 0; 1662 const struct btf_type *t; 1663 const struct btf_var_secinfo *vi; 1664 const char *name; 1665 1666 if (!obj->btf) { 1667 pr_warn("failed to find BTF info for object '%s'\n", obj->filename); 1668 return -EINVAL; 1669 } 1670 1671 n = btf__type_cnt(obj->btf); 1672 for (i = 1; i < n; i++) { 1673 t = btf__type_by_id(obj->btf, i); 1674 1675 /* some global and extern FUNCs and VARs might not be associated with any 1676 * DATASEC, so try to detect them in the same pass 1677 */ 1678 if (btf_is_non_static(t)) { 1679 name = btf__str_by_offset(obj->btf, t->name_off); 1680 if (strcmp(name, sym_name) != 0) 1681 continue; 1682 1683 /* remember and still try to find DATASEC */ 1684 btf_id = i; 1685 continue; 1686 } 1687 1688 if (!btf_is_datasec(t)) 1689 continue; 1690 1691 vi = btf_var_secinfos(t); 1692 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { 1693 t = btf__type_by_id(obj->btf, vi->type); 1694 name = btf__str_by_offset(obj->btf, t->name_off); 1695 1696 if (strcmp(name, sym_name) != 0) 1697 continue; 1698 if (btf_is_var(t) && btf_var(t)->linkage == BTF_VAR_STATIC) 1699 continue; 1700 if (btf_is_func(t) && btf_func_linkage(t) == BTF_FUNC_STATIC) 1701 continue; 1702 1703 if (btf_id && btf_id != vi->type) { 1704 pr_warn("global/extern '%s' BTF is ambiguous: both types #%d and #%u match\n", 1705 sym_name, btf_id, vi->type); 1706 return -EINVAL; 1707 } 1708 1709 *out_btf_sec_id = i; 1710 *out_btf_id = vi->type; 1711 1712 return 0; 1713 } 1714 } 1715 1716 /* free-floating extern or global FUNC */ 1717 if (btf_id) { 1718 *out_btf_sec_id = 0; 1719 *out_btf_id = btf_id; 1720 return 0; 1721 } 1722 1723 pr_warn("failed to find BTF info for global/extern symbol '%s'\n", sym_name); 1724 return -ENOENT; 1725} 1726 1727static struct src_sec *find_src_sec_by_name(struct src_obj *obj, const char *sec_name) 1728{ 1729 struct src_sec *sec; 1730 int i; 1731 1732 for (i = 1; i < obj->sec_cnt; i++) { 1733 sec = &obj->secs[i]; 1734 1735 if (strcmp(sec->sec_name, sec_name) == 0) 1736 return sec; 1737 } 1738 1739 return NULL; 1740} 1741 1742static int complete_extern_btf_info(struct btf *dst_btf, int dst_id, 1743 struct btf *src_btf, int src_id) 1744{ 1745 struct btf_type *dst_t = btf_type_by_id(dst_btf, dst_id); 1746 struct btf_type *src_t = btf_type_by_id(src_btf, src_id); 1747 struct btf_param *src_p, *dst_p; 1748 const char *s; 1749 int i, n, off; 1750 1751 /* We already made sure that source and destination types (FUNC or 1752 * VAR) match in terms of types and argument names. 1753 */ 1754 if (btf_is_var(dst_t)) { 1755 btf_var(dst_t)->linkage = BTF_VAR_GLOBAL_ALLOCATED; 1756 return 0; 1757 } 1758 1759 dst_t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_GLOBAL, 0); 1760 1761 /* now onto FUNC_PROTO types */ 1762 src_t = btf_type_by_id(src_btf, src_t->type); 1763 dst_t = btf_type_by_id(dst_btf, dst_t->type); 1764 1765 /* Fill in all the argument names, which for extern FUNCs are missing. 1766 * We'll end up with two copies of FUNCs/VARs for externs, but that 1767 * will be taken care of by BTF dedup at the very end. 1768 * It might be that BTF types for extern in one file has less/more BTF 1769 * information (e.g., FWD instead of full STRUCT/UNION information), 1770 * but that should be (in most cases, subject to BTF dedup rules) 1771 * handled and resolved by BTF dedup algorithm as well, so we won't 1772 * worry about it. Our only job is to make sure that argument names 1773 * are populated on both sides, otherwise BTF dedup will pedantically 1774 * consider them different. 1775 */ 1776 src_p = btf_params(src_t); 1777 dst_p = btf_params(dst_t); 1778 for (i = 0, n = btf_vlen(dst_t); i < n; i++, src_p++, dst_p++) { 1779 if (!src_p->name_off) 1780 continue; 1781 1782 /* src_btf has more complete info, so add name to dst_btf */ 1783 s = btf__str_by_offset(src_btf, src_p->name_off); 1784 off = btf__add_str(dst_btf, s); 1785 if (off < 0) 1786 return off; 1787 dst_p->name_off = off; 1788 } 1789 return 0; 1790} 1791 1792static void sym_update_bind(Elf64_Sym *sym, int sym_bind) 1793{ 1794 sym->st_info = ELF64_ST_INFO(sym_bind, ELF64_ST_TYPE(sym->st_info)); 1795} 1796 1797static void sym_update_type(Elf64_Sym *sym, int sym_type) 1798{ 1799 sym->st_info = ELF64_ST_INFO(ELF64_ST_BIND(sym->st_info), sym_type); 1800} 1801 1802static void sym_update_visibility(Elf64_Sym *sym, int sym_vis) 1803{ 1804 /* libelf doesn't provide setters for ST_VISIBILITY, 1805 * but it is stored in the lower 2 bits of st_other 1806 */ 1807 sym->st_other &= ~0x03; 1808 sym->st_other |= sym_vis; 1809} 1810 1811static int linker_append_elf_sym(struct bpf_linker *linker, struct src_obj *obj, 1812 Elf64_Sym *sym, const char *sym_name, int src_sym_idx) 1813{ 1814 struct src_sec *src_sec = NULL; 1815 struct dst_sec *dst_sec = NULL; 1816 struct glob_sym *glob_sym = NULL; 1817 int name_off, sym_type, sym_bind, sym_vis, err; 1818 int btf_sec_id = 0, btf_id = 0; 1819 size_t dst_sym_idx; 1820 Elf64_Sym *dst_sym; 1821 bool sym_is_extern; 1822 1823 sym_type = ELF64_ST_TYPE(sym->st_info); 1824 sym_bind = ELF64_ST_BIND(sym->st_info); 1825 sym_vis = ELF64_ST_VISIBILITY(sym->st_other); 1826 sym_is_extern = sym->st_shndx == SHN_UNDEF; 1827 1828 if (sym_is_extern) { 1829 if (!obj->btf) { 1830 pr_warn("externs without BTF info are not supported\n"); 1831 return -ENOTSUP; 1832 } 1833 } else if (sym->st_shndx < SHN_LORESERVE) { 1834 src_sec = &obj->secs[sym->st_shndx]; 1835 if (src_sec->skipped) 1836 return 0; 1837 dst_sec = &linker->secs[src_sec->dst_id]; 1838 1839 /* allow only one STT_SECTION symbol per section */ 1840 if (sym_type == STT_SECTION && dst_sec->sec_sym_idx) { 1841 obj->sym_map[src_sym_idx] = dst_sec->sec_sym_idx; 1842 return 0; 1843 } 1844 } 1845 1846 if (sym_bind == STB_LOCAL) 1847 goto add_sym; 1848 1849 /* find matching BTF info */ 1850 err = find_glob_sym_btf(obj, sym, sym_name, &btf_sec_id, &btf_id); 1851 if (err) 1852 return err; 1853 1854 if (sym_is_extern && btf_sec_id) { 1855 const char *sec_name = NULL; 1856 const struct btf_type *t; 1857 1858 t = btf__type_by_id(obj->btf, btf_sec_id); 1859 sec_name = btf__str_by_offset(obj->btf, t->name_off); 1860 1861 /* Clang puts unannotated extern vars into 1862 * '.extern' BTF DATASEC. Treat them the same 1863 * as unannotated extern funcs (which are 1864 * currently not put into any DATASECs). 1865 * Those don't have associated src_sec/dst_sec. 1866 */ 1867 if (strcmp(sec_name, BTF_EXTERN_SEC) != 0) { 1868 src_sec = find_src_sec_by_name(obj, sec_name); 1869 if (!src_sec) { 1870 pr_warn("failed to find matching ELF sec '%s'\n", sec_name); 1871 return -ENOENT; 1872 } 1873 dst_sec = &linker->secs[src_sec->dst_id]; 1874 } 1875 } 1876 1877 glob_sym = find_glob_sym(linker, sym_name); 1878 if (glob_sym) { 1879 /* Preventively resolve to existing symbol. This is 1880 * needed for further relocation symbol remapping in 1881 * the next step of linking. 1882 */ 1883 obj->sym_map[src_sym_idx] = glob_sym->sym_idx; 1884 1885 /* If both symbols are non-externs, at least one of 1886 * them has to be STB_WEAK, otherwise they are in 1887 * a conflict with each other. 1888 */ 1889 if (!sym_is_extern && !glob_sym->is_extern 1890 && !glob_sym->is_weak && sym_bind != STB_WEAK) { 1891 pr_warn("conflicting non-weak symbol #%d (%s) definition in '%s'\n", 1892 src_sym_idx, sym_name, obj->filename); 1893 return -EINVAL; 1894 } 1895 1896 if (!glob_syms_match(sym_name, linker, glob_sym, obj, sym, src_sym_idx, btf_id)) 1897 return -EINVAL; 1898 1899 dst_sym = get_sym_by_idx(linker, glob_sym->sym_idx); 1900 1901 /* If new symbol is strong, then force dst_sym to be strong as 1902 * well; this way a mix of weak and non-weak extern 1903 * definitions will end up being strong. 1904 */ 1905 if (sym_bind == STB_GLOBAL) { 1906 /* We still need to preserve type (NOTYPE or 1907 * OBJECT/FUNC, depending on whether the symbol is 1908 * extern or not) 1909 */ 1910 sym_update_bind(dst_sym, STB_GLOBAL); 1911 glob_sym->is_weak = false; 1912 } 1913 1914 /* Non-default visibility is "contaminating", with stricter 1915 * visibility overwriting more permissive ones, even if more 1916 * permissive visibility comes from just an extern definition. 1917 * Currently only STV_DEFAULT and STV_HIDDEN are allowed and 1918 * ensured by ELF symbol sanity checks above. 1919 */ 1920 if (sym_vis > ELF64_ST_VISIBILITY(dst_sym->st_other)) 1921 sym_update_visibility(dst_sym, sym_vis); 1922 1923 /* If the new symbol is extern, then regardless if 1924 * existing symbol is extern or resolved global, just 1925 * keep the existing one untouched. 1926 */ 1927 if (sym_is_extern) 1928 return 0; 1929 1930 /* If existing symbol is a strong resolved symbol, bail out, 1931 * because we lost resolution battle have nothing to 1932 * contribute. We already checked abover that there is no 1933 * strong-strong conflict. We also already tightened binding 1934 * and visibility, so nothing else to contribute at that point. 1935 */ 1936 if (!glob_sym->is_extern && sym_bind == STB_WEAK) 1937 return 0; 1938 1939 /* At this point, new symbol is strong non-extern, 1940 * so overwrite glob_sym with new symbol information. 1941 * Preserve binding and visibility. 1942 */ 1943 sym_update_type(dst_sym, sym_type); 1944 dst_sym->st_shndx = dst_sec->sec_idx; 1945 dst_sym->st_value = src_sec->dst_off + sym->st_value; 1946 dst_sym->st_size = sym->st_size; 1947 1948 /* see comment below about dst_sec->id vs dst_sec->sec_idx */ 1949 glob_sym->sec_id = dst_sec->id; 1950 glob_sym->is_extern = false; 1951 1952 if (complete_extern_btf_info(linker->btf, glob_sym->btf_id, 1953 obj->btf, btf_id)) 1954 return -EINVAL; 1955 1956 /* request updating VAR's/FUNC's underlying BTF type when appending BTF type */ 1957 glob_sym->underlying_btf_id = 0; 1958 1959 obj->sym_map[src_sym_idx] = glob_sym->sym_idx; 1960 return 0; 1961 } 1962 1963add_sym: 1964 name_off = strset__add_str(linker->strtab_strs, sym_name); 1965 if (name_off < 0) 1966 return name_off; 1967 1968 dst_sym = add_new_sym(linker, &dst_sym_idx); 1969 if (!dst_sym) 1970 return -ENOMEM; 1971 1972 dst_sym->st_name = name_off; 1973 dst_sym->st_info = sym->st_info; 1974 dst_sym->st_other = sym->st_other; 1975 dst_sym->st_shndx = dst_sec ? dst_sec->sec_idx : sym->st_shndx; 1976 dst_sym->st_value = (src_sec ? src_sec->dst_off : 0) + sym->st_value; 1977 dst_sym->st_size = sym->st_size; 1978 1979 obj->sym_map[src_sym_idx] = dst_sym_idx; 1980 1981 if (sym_type == STT_SECTION && dst_sym) { 1982 dst_sec->sec_sym_idx = dst_sym_idx; 1983 dst_sym->st_value = 0; 1984 } 1985 1986 if (sym_bind != STB_LOCAL) { 1987 glob_sym = add_glob_sym(linker); 1988 if (!glob_sym) 1989 return -ENOMEM; 1990 1991 glob_sym->sym_idx = dst_sym_idx; 1992 /* we use dst_sec->id (and not dst_sec->sec_idx), because 1993 * ephemeral sections (.kconfig, .ksyms, etc) don't have 1994 * sec_idx (as they don't have corresponding ELF section), but 1995 * still have id. .extern doesn't have even ephemeral section 1996 * associated with it, so dst_sec->id == dst_sec->sec_idx == 0. 1997 */ 1998 glob_sym->sec_id = dst_sec ? dst_sec->id : 0; 1999 glob_sym->name_off = name_off; 2000 /* we will fill btf_id in during BTF merging step */ 2001 glob_sym->btf_id = 0; 2002 glob_sym->is_extern = sym_is_extern; 2003 glob_sym->is_weak = sym_bind == STB_WEAK; 2004 } 2005 2006 return 0; 2007} 2008 2009static int linker_append_elf_relos(struct bpf_linker *linker, struct src_obj *obj) 2010{ 2011 struct src_sec *src_symtab = &obj->secs[obj->symtab_sec_idx]; 2012 int i, err; 2013 2014 for (i = 1; i < obj->sec_cnt; i++) { 2015 struct src_sec *src_sec, *src_linked_sec; 2016 struct dst_sec *dst_sec, *dst_linked_sec; 2017 Elf64_Rel *src_rel, *dst_rel; 2018 int j, n; 2019 2020 src_sec = &obj->secs[i]; 2021 if (!is_relo_sec(src_sec)) 2022 continue; 2023 2024 /* shdr->sh_info points to relocatable section */ 2025 src_linked_sec = &obj->secs[src_sec->shdr->sh_info]; 2026 if (src_linked_sec->skipped) 2027 continue; 2028 2029 dst_sec = find_dst_sec_by_name(linker, src_sec->sec_name); 2030 if (!dst_sec) { 2031 dst_sec = add_dst_sec(linker, src_sec->sec_name); 2032 if (!dst_sec) 2033 return -ENOMEM; 2034 err = init_sec(linker, dst_sec, src_sec); 2035 if (err) { 2036 pr_warn("failed to init section '%s'\n", src_sec->sec_name); 2037 return err; 2038 } 2039 } else if (!secs_match(dst_sec, src_sec)) { 2040 pr_warn("sections %s are not compatible\n", src_sec->sec_name); 2041 return -1; 2042 } 2043 2044 /* shdr->sh_link points to SYMTAB */ 2045 dst_sec->shdr->sh_link = linker->symtab_sec_idx; 2046 2047 /* shdr->sh_info points to relocated section */ 2048 dst_linked_sec = &linker->secs[src_linked_sec->dst_id]; 2049 dst_sec->shdr->sh_info = dst_linked_sec->sec_idx; 2050 2051 src_sec->dst_id = dst_sec->id; 2052 err = extend_sec(linker, dst_sec, src_sec); 2053 if (err) 2054 return err; 2055 2056 src_rel = src_sec->data->d_buf; 2057 dst_rel = dst_sec->raw_data + src_sec->dst_off; 2058 n = src_sec->shdr->sh_size / src_sec->shdr->sh_entsize; 2059 for (j = 0; j < n; j++, src_rel++, dst_rel++) { 2060 size_t src_sym_idx, dst_sym_idx, sym_type; 2061 Elf64_Sym *src_sym; 2062 2063 src_sym_idx = ELF64_R_SYM(src_rel->r_info); 2064 src_sym = src_symtab->data->d_buf + sizeof(*src_sym) * src_sym_idx; 2065 2066 dst_sym_idx = obj->sym_map[src_sym_idx]; 2067 dst_rel->r_offset += src_linked_sec->dst_off; 2068 sym_type = ELF64_R_TYPE(src_rel->r_info); 2069 dst_rel->r_info = ELF64_R_INFO(dst_sym_idx, sym_type); 2070 2071 if (ELF64_ST_TYPE(src_sym->st_info) == STT_SECTION) { 2072 struct src_sec *sec = &obj->secs[src_sym->st_shndx]; 2073 struct bpf_insn *insn; 2074 2075 if (src_linked_sec->shdr->sh_flags & SHF_EXECINSTR) { 2076 /* calls to the very first static function inside 2077 * .text section at offset 0 will 2078 * reference section symbol, not the 2079 * function symbol. Fix that up, 2080 * otherwise it won't be possible to 2081 * relocate calls to two different 2082 * static functions with the same name 2083 * (rom two different object files) 2084 */ 2085 insn = dst_linked_sec->raw_data + dst_rel->r_offset; 2086 if (insn->code == (BPF_JMP | BPF_CALL)) 2087 insn->imm += sec->dst_off / sizeof(struct bpf_insn); 2088 else 2089 insn->imm += sec->dst_off; 2090 } else { 2091 pr_warn("relocation against STT_SECTION in non-exec section is not supported!\n"); 2092 return -EINVAL; 2093 } 2094 } 2095 2096 } 2097 } 2098 2099 return 0; 2100} 2101 2102static Elf64_Sym *find_sym_by_name(struct src_obj *obj, size_t sec_idx, 2103 int sym_type, const char *sym_name) 2104{ 2105 struct src_sec *symtab = &obj->secs[obj->symtab_sec_idx]; 2106 Elf64_Sym *sym = symtab->data->d_buf; 2107 int i, n = symtab->shdr->sh_size / symtab->shdr->sh_entsize; 2108 int str_sec_idx = symtab->shdr->sh_link; 2109 const char *name; 2110 2111 for (i = 0; i < n; i++, sym++) { 2112 if (sym->st_shndx != sec_idx) 2113 continue; 2114 if (ELF64_ST_TYPE(sym->st_info) != sym_type) 2115 continue; 2116 2117 name = elf_strptr(obj->elf, str_sec_idx, sym->st_name); 2118 if (!name) 2119 return NULL; 2120 2121 if (strcmp(sym_name, name) != 0) 2122 continue; 2123 2124 return sym; 2125 } 2126 2127 return NULL; 2128} 2129 2130static int linker_fixup_btf(struct src_obj *obj) 2131{ 2132 const char *sec_name; 2133 struct src_sec *sec; 2134 int i, j, n, m; 2135 2136 if (!obj->btf) 2137 return 0; 2138 2139 n = btf__type_cnt(obj->btf); 2140 for (i = 1; i < n; i++) { 2141 struct btf_var_secinfo *vi; 2142 struct btf_type *t; 2143 2144 t = btf_type_by_id(obj->btf, i); 2145 if (btf_kind(t) != BTF_KIND_DATASEC) 2146 continue; 2147 2148 sec_name = btf__str_by_offset(obj->btf, t->name_off); 2149 sec = find_src_sec_by_name(obj, sec_name); 2150 if (sec) { 2151 /* record actual section size, unless ephemeral */ 2152 if (sec->shdr) 2153 t->size = sec->shdr->sh_size; 2154 } else { 2155 /* BTF can have some sections that are not represented 2156 * in ELF, e.g., .kconfig, .ksyms, .extern, which are used 2157 * for special extern variables. 2158 * 2159 * For all but one such special (ephemeral) 2160 * sections, we pre-create "section shells" to be able 2161 * to keep track of extra per-section metadata later 2162 * (e.g., those BTF extern variables). 2163 * 2164 * .extern is even more special, though, because it 2165 * contains extern variables that need to be resolved 2166 * by static linker, not libbpf and kernel. When such 2167 * externs are resolved, we are going to remove them 2168 * from .extern BTF section and might end up not 2169 * needing it at all. Each resolved extern should have 2170 * matching non-extern VAR/FUNC in other sections. 2171 * 2172 * We do support leaving some of the externs 2173 * unresolved, though, to support cases of building 2174 * libraries, which will later be linked against final 2175 * BPF applications. So if at finalization we still 2176 * see unresolved externs, we'll create .extern 2177 * section on our own. 2178 */ 2179 if (strcmp(sec_name, BTF_EXTERN_SEC) == 0) 2180 continue; 2181 2182 sec = add_src_sec(obj, sec_name); 2183 if (!sec) 2184 return -ENOMEM; 2185 2186 sec->ephemeral = true; 2187 sec->sec_idx = 0; /* will match UNDEF shndx in ELF */ 2188 } 2189 2190 /* remember ELF section and its BTF type ID match */ 2191 sec->sec_type_id = i; 2192 2193 /* fix up variable offsets */ 2194 vi = btf_var_secinfos(t); 2195 for (j = 0, m = btf_vlen(t); j < m; j++, vi++) { 2196 const struct btf_type *vt = btf__type_by_id(obj->btf, vi->type); 2197 const char *var_name = btf__str_by_offset(obj->btf, vt->name_off); 2198 int var_linkage = btf_var(vt)->linkage; 2199 Elf64_Sym *sym; 2200 2201 /* no need to patch up static or extern vars */ 2202 if (var_linkage != BTF_VAR_GLOBAL_ALLOCATED) 2203 continue; 2204 2205 sym = find_sym_by_name(obj, sec->sec_idx, STT_OBJECT, var_name); 2206 if (!sym) { 2207 pr_warn("failed to find symbol for variable '%s' in section '%s'\n", var_name, sec_name); 2208 return -ENOENT; 2209 } 2210 2211 vi->offset = sym->st_value; 2212 } 2213 } 2214 2215 return 0; 2216} 2217 2218static int remap_type_id(__u32 *type_id, void *ctx) 2219{ 2220 int *id_map = ctx; 2221 int new_id = id_map[*type_id]; 2222 2223 /* Error out if the type wasn't remapped. Ignore VOID which stays VOID. */ 2224 if (new_id == 0 && *type_id != 0) { 2225 pr_warn("failed to find new ID mapping for original BTF type ID %u\n", *type_id); 2226 return -EINVAL; 2227 } 2228 2229 *type_id = id_map[*type_id]; 2230 2231 return 0; 2232} 2233 2234static int linker_append_btf(struct bpf_linker *linker, struct src_obj *obj) 2235{ 2236 const struct btf_type *t; 2237 int i, j, n, start_id, id; 2238 const char *name; 2239 2240 if (!obj->btf) 2241 return 0; 2242 2243 start_id = btf__type_cnt(linker->btf); 2244 n = btf__type_cnt(obj->btf); 2245 2246 obj->btf_type_map = calloc(n + 1, sizeof(int)); 2247 if (!obj->btf_type_map) 2248 return -ENOMEM; 2249 2250 for (i = 1; i < n; i++) { 2251 struct glob_sym *glob_sym = NULL; 2252 2253 t = btf__type_by_id(obj->btf, i); 2254 2255 /* DATASECs are handled specially below */ 2256 if (btf_kind(t) == BTF_KIND_DATASEC) 2257 continue; 2258 2259 if (btf_is_non_static(t)) { 2260 /* there should be glob_sym already */ 2261 name = btf__str_by_offset(obj->btf, t->name_off); 2262 glob_sym = find_glob_sym(linker, name); 2263 2264 /* VARs without corresponding glob_sym are those that 2265 * belong to skipped/deduplicated sections (i.e., 2266 * license and version), so just skip them 2267 */ 2268 if (!glob_sym) 2269 continue; 2270 2271 /* linker_append_elf_sym() might have requested 2272 * updating underlying type ID, if extern was resolved 2273 * to strong symbol or weak got upgraded to non-weak 2274 */ 2275 if (glob_sym->underlying_btf_id == 0) 2276 glob_sym->underlying_btf_id = -t->type; 2277 2278 /* globals from previous object files that match our 2279 * VAR/FUNC already have a corresponding associated 2280 * BTF type, so just make sure to use it 2281 */ 2282 if (glob_sym->btf_id) { 2283 /* reuse existing BTF type for global var/func */ 2284 obj->btf_type_map[i] = glob_sym->btf_id; 2285 continue; 2286 } 2287 } 2288 2289 id = btf__add_type(linker->btf, obj->btf, t); 2290 if (id < 0) { 2291 pr_warn("failed to append BTF type #%d from file '%s'\n", i, obj->filename); 2292 return id; 2293 } 2294 2295 obj->btf_type_map[i] = id; 2296 2297 /* record just appended BTF type for var/func */ 2298 if (glob_sym) { 2299 glob_sym->btf_id = id; 2300 glob_sym->underlying_btf_id = -t->type; 2301 } 2302 } 2303 2304 /* remap all the types except DATASECs */ 2305 n = btf__type_cnt(linker->btf); 2306 for (i = start_id; i < n; i++) { 2307 struct btf_type *dst_t = btf_type_by_id(linker->btf, i); 2308 2309 if (btf_type_visit_type_ids(dst_t, remap_type_id, obj->btf_type_map)) 2310 return -EINVAL; 2311 } 2312 2313 /* Rewrite VAR/FUNC underlying types (i.e., FUNC's FUNC_PROTO and VAR's 2314 * actual type), if necessary 2315 */ 2316 for (i = 0; i < linker->glob_sym_cnt; i++) { 2317 struct glob_sym *glob_sym = &linker->glob_syms[i]; 2318 struct btf_type *glob_t; 2319 2320 if (glob_sym->underlying_btf_id >= 0) 2321 continue; 2322 2323 glob_sym->underlying_btf_id = obj->btf_type_map[-glob_sym->underlying_btf_id]; 2324 2325 glob_t = btf_type_by_id(linker->btf, glob_sym->btf_id); 2326 glob_t->type = glob_sym->underlying_btf_id; 2327 } 2328 2329 /* append DATASEC info */ 2330 for (i = 1; i < obj->sec_cnt; i++) { 2331 struct src_sec *src_sec; 2332 struct dst_sec *dst_sec; 2333 const struct btf_var_secinfo *src_var; 2334 struct btf_var_secinfo *dst_var; 2335 2336 src_sec = &obj->secs[i]; 2337 if (!src_sec->sec_type_id || src_sec->skipped) 2338 continue; 2339 dst_sec = &linker->secs[src_sec->dst_id]; 2340 2341 /* Mark section as having BTF regardless of the presence of 2342 * variables. In some cases compiler might generate empty BTF 2343 * with no variables information. E.g., when promoting local 2344 * array/structure variable initial values and BPF object 2345 * file otherwise has no read-only static variables in 2346 * .rodata. We need to preserve such empty BTF and just set 2347 * correct section size. 2348 */ 2349 dst_sec->has_btf = true; 2350 2351 t = btf__type_by_id(obj->btf, src_sec->sec_type_id); 2352 src_var = btf_var_secinfos(t); 2353 n = btf_vlen(t); 2354 for (j = 0; j < n; j++, src_var++) { 2355 void *sec_vars = dst_sec->sec_vars; 2356 int new_id = obj->btf_type_map[src_var->type]; 2357 struct glob_sym *glob_sym = NULL; 2358 2359 t = btf_type_by_id(linker->btf, new_id); 2360 if (btf_is_non_static(t)) { 2361 name = btf__str_by_offset(linker->btf, t->name_off); 2362 glob_sym = find_glob_sym(linker, name); 2363 if (glob_sym->sec_id != dst_sec->id) { 2364 pr_warn("global '%s': section mismatch %d vs %d\n", 2365 name, glob_sym->sec_id, dst_sec->id); 2366 return -EINVAL; 2367 } 2368 } 2369 2370 /* If there is already a member (VAR or FUNC) mapped 2371 * to the same type, don't add a duplicate entry. 2372 * This will happen when multiple object files define 2373 * the same extern VARs/FUNCs. 2374 */ 2375 if (glob_sym && glob_sym->var_idx >= 0) { 2376 __s64 sz; 2377 2378 dst_var = &dst_sec->sec_vars[glob_sym->var_idx]; 2379 /* Because underlying BTF type might have 2380 * changed, so might its size have changed, so 2381 * re-calculate and update it in sec_var. 2382 */ 2383 sz = btf__resolve_size(linker->btf, glob_sym->underlying_btf_id); 2384 if (sz < 0) { 2385 pr_warn("global '%s': failed to resolve size of underlying type: %d\n", 2386 name, (int)sz); 2387 return -EINVAL; 2388 } 2389 dst_var->size = sz; 2390 continue; 2391 } 2392 2393 sec_vars = libbpf_reallocarray(sec_vars, 2394 dst_sec->sec_var_cnt + 1, 2395 sizeof(*dst_sec->sec_vars)); 2396 if (!sec_vars) 2397 return -ENOMEM; 2398 2399 dst_sec->sec_vars = sec_vars; 2400 dst_sec->sec_var_cnt++; 2401 2402 dst_var = &dst_sec->sec_vars[dst_sec->sec_var_cnt - 1]; 2403 dst_var->type = obj->btf_type_map[src_var->type]; 2404 dst_var->size = src_var->size; 2405 dst_var->offset = src_sec->dst_off + src_var->offset; 2406 2407 if (glob_sym) 2408 glob_sym->var_idx = dst_sec->sec_var_cnt - 1; 2409 } 2410 } 2411 2412 return 0; 2413} 2414 2415static void *add_btf_ext_rec(struct btf_ext_sec_data *ext_data, const void *src_rec) 2416{ 2417 void *tmp; 2418 2419 tmp = libbpf_reallocarray(ext_data->recs, ext_data->rec_cnt + 1, ext_data->rec_sz); 2420 if (!tmp) 2421 return NULL; 2422 ext_data->recs = tmp; 2423 2424 tmp += ext_data->rec_cnt * ext_data->rec_sz; 2425 memcpy(tmp, src_rec, ext_data->rec_sz); 2426 2427 ext_data->rec_cnt++; 2428 2429 return tmp; 2430} 2431 2432static int linker_append_btf_ext(struct bpf_linker *linker, struct src_obj *obj) 2433{ 2434 const struct btf_ext_info_sec *ext_sec; 2435 const char *sec_name, *s; 2436 struct src_sec *src_sec; 2437 struct dst_sec *dst_sec; 2438 int rec_sz, str_off, i; 2439 2440 if (!obj->btf_ext) 2441 return 0; 2442 2443 rec_sz = obj->btf_ext->func_info.rec_size; 2444 for_each_btf_ext_sec(&obj->btf_ext->func_info, ext_sec) { 2445 struct bpf_func_info_min *src_rec, *dst_rec; 2446 2447 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 2448 src_sec = find_src_sec_by_name(obj, sec_name); 2449 if (!src_sec) { 2450 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 2451 return -EINVAL; 2452 } 2453 dst_sec = &linker->secs[src_sec->dst_id]; 2454 2455 if (dst_sec->func_info.rec_sz == 0) 2456 dst_sec->func_info.rec_sz = rec_sz; 2457 if (dst_sec->func_info.rec_sz != rec_sz) { 2458 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 2459 return -EINVAL; 2460 } 2461 2462 for_each_btf_ext_rec(&obj->btf_ext->func_info, ext_sec, i, src_rec) { 2463 dst_rec = add_btf_ext_rec(&dst_sec->func_info, src_rec); 2464 if (!dst_rec) 2465 return -ENOMEM; 2466 2467 dst_rec->insn_off += src_sec->dst_off; 2468 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 2469 } 2470 } 2471 2472 rec_sz = obj->btf_ext->line_info.rec_size; 2473 for_each_btf_ext_sec(&obj->btf_ext->line_info, ext_sec) { 2474 struct bpf_line_info_min *src_rec, *dst_rec; 2475 2476 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 2477 src_sec = find_src_sec_by_name(obj, sec_name); 2478 if (!src_sec) { 2479 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 2480 return -EINVAL; 2481 } 2482 dst_sec = &linker->secs[src_sec->dst_id]; 2483 2484 if (dst_sec->line_info.rec_sz == 0) 2485 dst_sec->line_info.rec_sz = rec_sz; 2486 if (dst_sec->line_info.rec_sz != rec_sz) { 2487 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 2488 return -EINVAL; 2489 } 2490 2491 for_each_btf_ext_rec(&obj->btf_ext->line_info, ext_sec, i, src_rec) { 2492 dst_rec = add_btf_ext_rec(&dst_sec->line_info, src_rec); 2493 if (!dst_rec) 2494 return -ENOMEM; 2495 2496 dst_rec->insn_off += src_sec->dst_off; 2497 2498 s = btf__str_by_offset(obj->btf, src_rec->file_name_off); 2499 str_off = btf__add_str(linker->btf, s); 2500 if (str_off < 0) 2501 return -ENOMEM; 2502 dst_rec->file_name_off = str_off; 2503 2504 s = btf__str_by_offset(obj->btf, src_rec->line_off); 2505 str_off = btf__add_str(linker->btf, s); 2506 if (str_off < 0) 2507 return -ENOMEM; 2508 dst_rec->line_off = str_off; 2509 2510 /* dst_rec->line_col is fine */ 2511 } 2512 } 2513 2514 rec_sz = obj->btf_ext->core_relo_info.rec_size; 2515 for_each_btf_ext_sec(&obj->btf_ext->core_relo_info, ext_sec) { 2516 struct bpf_core_relo *src_rec, *dst_rec; 2517 2518 sec_name = btf__name_by_offset(obj->btf, ext_sec->sec_name_off); 2519 src_sec = find_src_sec_by_name(obj, sec_name); 2520 if (!src_sec) { 2521 pr_warn("can't find section '%s' referenced from .BTF.ext\n", sec_name); 2522 return -EINVAL; 2523 } 2524 dst_sec = &linker->secs[src_sec->dst_id]; 2525 2526 if (dst_sec->core_relo_info.rec_sz == 0) 2527 dst_sec->core_relo_info.rec_sz = rec_sz; 2528 if (dst_sec->core_relo_info.rec_sz != rec_sz) { 2529 pr_warn("incompatible .BTF.ext record sizes for section '%s'\n", sec_name); 2530 return -EINVAL; 2531 } 2532 2533 for_each_btf_ext_rec(&obj->btf_ext->core_relo_info, ext_sec, i, src_rec) { 2534 dst_rec = add_btf_ext_rec(&dst_sec->core_relo_info, src_rec); 2535 if (!dst_rec) 2536 return -ENOMEM; 2537 2538 dst_rec->insn_off += src_sec->dst_off; 2539 dst_rec->type_id = obj->btf_type_map[dst_rec->type_id]; 2540 2541 s = btf__str_by_offset(obj->btf, src_rec->access_str_off); 2542 str_off = btf__add_str(linker->btf, s); 2543 if (str_off < 0) 2544 return -ENOMEM; 2545 dst_rec->access_str_off = str_off; 2546 2547 /* dst_rec->kind is fine */ 2548 } 2549 } 2550 2551 return 0; 2552} 2553 2554int bpf_linker__finalize(struct bpf_linker *linker) 2555{ 2556 struct dst_sec *sec; 2557 size_t strs_sz; 2558 const void *strs; 2559 int err, i; 2560 2561 if (!linker->elf) 2562 return libbpf_err(-EINVAL); 2563 2564 err = finalize_btf(linker); 2565 if (err) 2566 return libbpf_err(err); 2567 2568 /* Finalize strings */ 2569 strs_sz = strset__data_size(linker->strtab_strs); 2570 strs = strset__data(linker->strtab_strs); 2571 2572 sec = &linker->secs[linker->strtab_sec_idx]; 2573 sec->data->d_align = 1; 2574 sec->data->d_off = 0LL; 2575 sec->data->d_buf = (void *)strs; 2576 sec->data->d_type = ELF_T_BYTE; 2577 sec->data->d_size = strs_sz; 2578 sec->shdr->sh_size = strs_sz; 2579 2580 for (i = 1; i < linker->sec_cnt; i++) { 2581 sec = &linker->secs[i]; 2582 2583 /* STRTAB is handled specially above */ 2584 if (sec->sec_idx == linker->strtab_sec_idx) 2585 continue; 2586 2587 /* special ephemeral sections (.ksyms, .kconfig, etc) */ 2588 if (!sec->scn) 2589 continue; 2590 2591 sec->data->d_buf = sec->raw_data; 2592 } 2593 2594 /* Finalize ELF layout */ 2595 if (elf_update(linker->elf, ELF_C_NULL) < 0) { 2596 err = -errno; 2597 pr_warn_elf("failed to finalize ELF layout"); 2598 return libbpf_err(err); 2599 } 2600 2601 /* Write out final ELF contents */ 2602 if (elf_update(linker->elf, ELF_C_WRITE) < 0) { 2603 err = -errno; 2604 pr_warn_elf("failed to write ELF contents"); 2605 return libbpf_err(err); 2606 } 2607 2608 elf_end(linker->elf); 2609 close(linker->fd); 2610 2611 linker->elf = NULL; 2612 linker->fd = -1; 2613 2614 return 0; 2615} 2616 2617static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, 2618 size_t align, const void *raw_data, size_t raw_sz) 2619{ 2620 Elf_Scn *scn; 2621 Elf_Data *data; 2622 Elf64_Shdr *shdr; 2623 int name_off; 2624 2625 name_off = strset__add_str(linker->strtab_strs, sec_name); 2626 if (name_off < 0) 2627 return name_off; 2628 2629 scn = elf_newscn(linker->elf); 2630 if (!scn) 2631 return -ENOMEM; 2632 data = elf_newdata(scn); 2633 if (!data) 2634 return -ENOMEM; 2635 shdr = elf64_getshdr(scn); 2636 if (!shdr) 2637 return -EINVAL; 2638 2639 shdr->sh_name = name_off; 2640 shdr->sh_type = SHT_PROGBITS; 2641 shdr->sh_flags = 0; 2642 shdr->sh_size = raw_sz; 2643 shdr->sh_link = 0; 2644 shdr->sh_info = 0; 2645 shdr->sh_addralign = align; 2646 shdr->sh_entsize = 0; 2647 2648 data->d_type = ELF_T_BYTE; 2649 data->d_size = raw_sz; 2650 data->d_buf = (void *)raw_data; 2651 data->d_align = align; 2652 data->d_off = 0; 2653 2654 return 0; 2655} 2656 2657static int finalize_btf(struct bpf_linker *linker) 2658{ 2659 LIBBPF_OPTS(btf_dedup_opts, opts); 2660 struct btf *btf = linker->btf; 2661 const void *raw_data; 2662 int i, j, id, err; 2663 __u32 raw_sz; 2664 2665 /* bail out if no BTF data was produced */ 2666 if (btf__type_cnt(linker->btf) == 1) 2667 return 0; 2668 2669 for (i = 1; i < linker->sec_cnt; i++) { 2670 struct dst_sec *sec = &linker->secs[i]; 2671 2672 if (!sec->has_btf) 2673 continue; 2674 2675 id = btf__add_datasec(btf, sec->sec_name, sec->sec_sz); 2676 if (id < 0) { 2677 pr_warn("failed to add consolidated BTF type for datasec '%s': %d\n", 2678 sec->sec_name, id); 2679 return id; 2680 } 2681 2682 for (j = 0; j < sec->sec_var_cnt; j++) { 2683 struct btf_var_secinfo *vi = &sec->sec_vars[j]; 2684 2685 if (btf__add_datasec_var_info(btf, vi->type, vi->offset, vi->size)) 2686 return -EINVAL; 2687 } 2688 } 2689 2690 err = finalize_btf_ext(linker); 2691 if (err) { 2692 pr_warn(".BTF.ext generation failed: %d\n", err); 2693 return err; 2694 } 2695 2696 opts.btf_ext = linker->btf_ext; 2697 err = btf__dedup(linker->btf, &opts); 2698 if (err) { 2699 pr_warn("BTF dedup failed: %d\n", err); 2700 return err; 2701 } 2702 2703 /* Emit .BTF section */ 2704 raw_data = btf__raw_data(linker->btf, &raw_sz); 2705 if (!raw_data) 2706 return -ENOMEM; 2707 2708 err = emit_elf_data_sec(linker, BTF_ELF_SEC, 8, raw_data, raw_sz); 2709 if (err) { 2710 pr_warn("failed to write out .BTF ELF section: %d\n", err); 2711 return err; 2712 } 2713 2714 /* Emit .BTF.ext section */ 2715 if (linker->btf_ext) { 2716 raw_data = btf_ext__get_raw_data(linker->btf_ext, &raw_sz); 2717 if (!raw_data) 2718 return -ENOMEM; 2719 2720 err = emit_elf_data_sec(linker, BTF_EXT_ELF_SEC, 8, raw_data, raw_sz); 2721 if (err) { 2722 pr_warn("failed to write out .BTF.ext ELF section: %d\n", err); 2723 return err; 2724 } 2725 } 2726 2727 return 0; 2728} 2729 2730static int emit_btf_ext_data(struct bpf_linker *linker, void *output, 2731 const char *sec_name, struct btf_ext_sec_data *sec_data) 2732{ 2733 struct btf_ext_info_sec *sec_info; 2734 void *cur = output; 2735 int str_off; 2736 size_t sz; 2737 2738 if (!sec_data->rec_cnt) 2739 return 0; 2740 2741 str_off = btf__add_str(linker->btf, sec_name); 2742 if (str_off < 0) 2743 return -ENOMEM; 2744 2745 sec_info = cur; 2746 sec_info->sec_name_off = str_off; 2747 sec_info->num_info = sec_data->rec_cnt; 2748 cur += sizeof(struct btf_ext_info_sec); 2749 2750 sz = sec_data->rec_cnt * sec_data->rec_sz; 2751 memcpy(cur, sec_data->recs, sz); 2752 cur += sz; 2753 2754 return cur - output; 2755} 2756 2757static int finalize_btf_ext(struct bpf_linker *linker) 2758{ 2759 size_t funcs_sz = 0, lines_sz = 0, core_relos_sz = 0, total_sz = 0; 2760 size_t func_rec_sz = 0, line_rec_sz = 0, core_relo_rec_sz = 0; 2761 struct btf_ext_header *hdr; 2762 void *data, *cur; 2763 int i, err, sz; 2764 2765 /* validate that all sections have the same .BTF.ext record sizes 2766 * and calculate total data size for each type of data (func info, 2767 * line info, core relos) 2768 */ 2769 for (i = 1; i < linker->sec_cnt; i++) { 2770 struct dst_sec *sec = &linker->secs[i]; 2771 2772 if (sec->func_info.rec_cnt) { 2773 if (func_rec_sz == 0) 2774 func_rec_sz = sec->func_info.rec_sz; 2775 if (func_rec_sz != sec->func_info.rec_sz) { 2776 pr_warn("mismatch in func_info record size %zu != %u\n", 2777 func_rec_sz, sec->func_info.rec_sz); 2778 return -EINVAL; 2779 } 2780 2781 funcs_sz += sizeof(struct btf_ext_info_sec) + func_rec_sz * sec->func_info.rec_cnt; 2782 } 2783 if (sec->line_info.rec_cnt) { 2784 if (line_rec_sz == 0) 2785 line_rec_sz = sec->line_info.rec_sz; 2786 if (line_rec_sz != sec->line_info.rec_sz) { 2787 pr_warn("mismatch in line_info record size %zu != %u\n", 2788 line_rec_sz, sec->line_info.rec_sz); 2789 return -EINVAL; 2790 } 2791 2792 lines_sz += sizeof(struct btf_ext_info_sec) + line_rec_sz * sec->line_info.rec_cnt; 2793 } 2794 if (sec->core_relo_info.rec_cnt) { 2795 if (core_relo_rec_sz == 0) 2796 core_relo_rec_sz = sec->core_relo_info.rec_sz; 2797 if (core_relo_rec_sz != sec->core_relo_info.rec_sz) { 2798 pr_warn("mismatch in core_relo_info record size %zu != %u\n", 2799 core_relo_rec_sz, sec->core_relo_info.rec_sz); 2800 return -EINVAL; 2801 } 2802 2803 core_relos_sz += sizeof(struct btf_ext_info_sec) + core_relo_rec_sz * sec->core_relo_info.rec_cnt; 2804 } 2805 } 2806 2807 if (!funcs_sz && !lines_sz && !core_relos_sz) 2808 return 0; 2809 2810 total_sz += sizeof(struct btf_ext_header); 2811 if (funcs_sz) { 2812 funcs_sz += sizeof(__u32); /* record size prefix */ 2813 total_sz += funcs_sz; 2814 } 2815 if (lines_sz) { 2816 lines_sz += sizeof(__u32); /* record size prefix */ 2817 total_sz += lines_sz; 2818 } 2819 if (core_relos_sz) { 2820 core_relos_sz += sizeof(__u32); /* record size prefix */ 2821 total_sz += core_relos_sz; 2822 } 2823 2824 cur = data = calloc(1, total_sz); 2825 if (!data) 2826 return -ENOMEM; 2827 2828 hdr = cur; 2829 hdr->magic = BTF_MAGIC; 2830 hdr->version = BTF_VERSION; 2831 hdr->flags = 0; 2832 hdr->hdr_len = sizeof(struct btf_ext_header); 2833 cur += sizeof(struct btf_ext_header); 2834 2835 /* All offsets are in bytes relative to the end of this header */ 2836 hdr->func_info_off = 0; 2837 hdr->func_info_len = funcs_sz; 2838 hdr->line_info_off = funcs_sz; 2839 hdr->line_info_len = lines_sz; 2840 hdr->core_relo_off = funcs_sz + lines_sz; 2841 hdr->core_relo_len = core_relos_sz; 2842 2843 if (funcs_sz) { 2844 *(__u32 *)cur = func_rec_sz; 2845 cur += sizeof(__u32); 2846 2847 for (i = 1; i < linker->sec_cnt; i++) { 2848 struct dst_sec *sec = &linker->secs[i]; 2849 2850 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->func_info); 2851 if (sz < 0) { 2852 err = sz; 2853 goto out; 2854 } 2855 2856 cur += sz; 2857 } 2858 } 2859 2860 if (lines_sz) { 2861 *(__u32 *)cur = line_rec_sz; 2862 cur += sizeof(__u32); 2863 2864 for (i = 1; i < linker->sec_cnt; i++) { 2865 struct dst_sec *sec = &linker->secs[i]; 2866 2867 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->line_info); 2868 if (sz < 0) { 2869 err = sz; 2870 goto out; 2871 } 2872 2873 cur += sz; 2874 } 2875 } 2876 2877 if (core_relos_sz) { 2878 *(__u32 *)cur = core_relo_rec_sz; 2879 cur += sizeof(__u32); 2880 2881 for (i = 1; i < linker->sec_cnt; i++) { 2882 struct dst_sec *sec = &linker->secs[i]; 2883 2884 sz = emit_btf_ext_data(linker, cur, sec->sec_name, &sec->core_relo_info); 2885 if (sz < 0) { 2886 err = sz; 2887 goto out; 2888 } 2889 2890 cur += sz; 2891 } 2892 } 2893 2894 linker->btf_ext = btf_ext__new(data, total_sz); 2895 err = libbpf_get_error(linker->btf_ext); 2896 if (err) { 2897 linker->btf_ext = NULL; 2898 pr_warn("failed to parse final .BTF.ext data: %d\n", err); 2899 goto out; 2900 } 2901 2902out: 2903 free(data); 2904 return err; 2905} 2906