1// SPDX-License-Identifier: GPL-2.0+ 2/* 3* Copyright (C) 2020 Loongson Technology Corporation Limited 4* 5* Author: Hanlu Li <lihanlu@loongson.cn> 6* Author: Huacai Chen <chenhuacai@loongson.cn> 7*/ 8 9#define pr_fmt(fmt) "kmod: " fmt 10 11#include <linux/moduleloader.h> 12#include <linux/elf.h> 13#include <linux/ftrace.h> 14#include <linux/mm.h> 15#include <linux/numa.h> 16#include <linux/vmalloc.h> 17#include <linux/slab.h> 18#include <linux/fs.h> 19#include <linux/string.h> 20#include <linux/kernel.h> 21#include <asm/alternative.h> 22#include <asm/inst.h> 23#include <asm/unwind.h> 24 25static inline bool signed_imm_check(long val, unsigned int bit) 26{ 27 return -(1L << (bit - 1)) <= val && val < (1L << (bit - 1)); 28} 29 30static inline bool unsigned_imm_check(unsigned long val, unsigned int bit) 31{ 32 return val < (1UL << bit); 33} 34 35static int rela_stack_push(s64 stack_value, s64 *rela_stack, size_t *rela_stack_top) 36{ 37 if (RELA_STACK_DEPTH <= *rela_stack_top) 38 return -ENOEXEC; 39 40 rela_stack[(*rela_stack_top)++] = stack_value; 41 pr_debug("%s stack_value = 0x%llx\n", __func__, stack_value); 42 43 return 0; 44} 45 46static int rela_stack_pop(s64 *stack_value, s64 *rela_stack, size_t *rela_stack_top) 47{ 48 if (*rela_stack_top == 0) 49 return -ENOEXEC; 50 51 *stack_value = rela_stack[--(*rela_stack_top)]; 52 pr_debug("%s stack_value = 0x%llx\n", __func__, *stack_value); 53 54 return 0; 55} 56 57static int apply_r_larch_none(struct module *mod, u32 *location, Elf_Addr v, 58 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 59{ 60 return 0; 61} 62 63static int apply_r_larch_error(struct module *me, u32 *location, Elf_Addr v, 64 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 65{ 66 pr_err("%s: Unsupport relocation type %u, please add its support.\n", me->name, type); 67 return -EINVAL; 68} 69 70static int apply_r_larch_32(struct module *mod, u32 *location, Elf_Addr v, 71 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 72{ 73 *location = v; 74 return 0; 75} 76 77static int apply_r_larch_64(struct module *mod, u32 *location, Elf_Addr v, 78 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 79{ 80 *(Elf_Addr *)location = v; 81 return 0; 82} 83 84static int apply_r_larch_sop_push_pcrel(struct module *mod, u32 *location, Elf_Addr v, 85 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 86{ 87 return rela_stack_push(v - (u64)location, rela_stack, rela_stack_top); 88} 89 90static int apply_r_larch_sop_push_absolute(struct module *mod, u32 *location, Elf_Addr v, 91 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 92{ 93 return rela_stack_push(v, rela_stack, rela_stack_top); 94} 95 96static int apply_r_larch_sop_push_dup(struct module *mod, u32 *location, Elf_Addr v, 97 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 98{ 99 int err = 0; 100 s64 opr1; 101 102 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 103 if (err) 104 return err; 105 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 106 if (err) 107 return err; 108 err = rela_stack_push(opr1, rela_stack, rela_stack_top); 109 if (err) 110 return err; 111 112 return 0; 113} 114 115static int apply_r_larch_sop_push_plt_pcrel(struct module *mod, 116 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 117 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 118{ 119 ptrdiff_t offset = (void *)v - (void *)location; 120 121 if (offset >= SZ_128M) 122 v = module_emit_plt_entry(mod, sechdrs, v); 123 124 if (offset < -SZ_128M) 125 v = module_emit_plt_entry(mod, sechdrs, v); 126 127 return apply_r_larch_sop_push_pcrel(mod, location, v, rela_stack, rela_stack_top, type); 128} 129 130static int apply_r_larch_sop(struct module *mod, u32 *location, Elf_Addr v, 131 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 132{ 133 int err = 0; 134 s64 opr1, opr2, opr3; 135 136 if (type == R_LARCH_SOP_IF_ELSE) { 137 err = rela_stack_pop(&opr3, rela_stack, rela_stack_top); 138 if (err) 139 return err; 140 } 141 142 err = rela_stack_pop(&opr2, rela_stack, rela_stack_top); 143 if (err) 144 return err; 145 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 146 if (err) 147 return err; 148 149 switch (type) { 150 case R_LARCH_SOP_AND: 151 err = rela_stack_push(opr1 & opr2, rela_stack, rela_stack_top); 152 break; 153 case R_LARCH_SOP_ADD: 154 err = rela_stack_push(opr1 + opr2, rela_stack, rela_stack_top); 155 break; 156 case R_LARCH_SOP_SUB: 157 err = rela_stack_push(opr1 - opr2, rela_stack, rela_stack_top); 158 break; 159 case R_LARCH_SOP_SL: 160 err = rela_stack_push(opr1 << opr2, rela_stack, rela_stack_top); 161 break; 162 case R_LARCH_SOP_SR: 163 err = rela_stack_push(opr1 >> opr2, rela_stack, rela_stack_top); 164 break; 165 case R_LARCH_SOP_IF_ELSE: 166 err = rela_stack_push(opr1 ? opr2 : opr3, rela_stack, rela_stack_top); 167 break; 168 default: 169 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 170 return -EINVAL; 171 } 172 173 return err; 174} 175 176static int apply_r_larch_sop_imm_field(struct module *mod, u32 *location, Elf_Addr v, 177 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 178{ 179 int err = 0; 180 s64 opr1; 181 union loongarch_instruction *insn = (union loongarch_instruction *)location; 182 183 err = rela_stack_pop(&opr1, rela_stack, rela_stack_top); 184 if (err) 185 return err; 186 187 switch (type) { 188 case R_LARCH_SOP_POP_32_U_10_12: 189 if (!unsigned_imm_check(opr1, 12)) 190 goto overflow; 191 192 /* (*(uint32_t *) PC) [21 ... 10] = opr [11 ... 0] */ 193 insn->reg2ui12_format.simmediate = opr1 & 0xfff; 194 return 0; 195 case R_LARCH_SOP_POP_32_S_10_12: 196 if (!signed_imm_check(opr1, 12)) 197 goto overflow; 198 199 insn->reg2i12_format.simmediate = opr1 & 0xfff; 200 return 0; 201 case R_LARCH_SOP_POP_32_S_10_16: 202 if (!signed_imm_check(opr1, 16)) 203 goto overflow; 204 205 insn->reg2i16_format.simmediate = opr1 & 0xffff; 206 return 0; 207 case R_LARCH_SOP_POP_32_S_10_16_S2: 208 if (opr1 % 4) 209 goto unaligned; 210 211 if (!signed_imm_check(opr1, 18)) 212 goto overflow; 213 214 insn->reg2i16_format.simmediate = (opr1 >> 2) & 0xffff; 215 return 0; 216 case R_LARCH_SOP_POP_32_S_5_20: 217 if (!signed_imm_check(opr1, 20)) 218 goto overflow; 219 220 insn->reg1i20_format.simmediate = (opr1) & 0xfffff; 221 return 0; 222 case R_LARCH_SOP_POP_32_S_0_5_10_16_S2: 223 if (opr1 % 4) 224 goto unaligned; 225 226 if (!signed_imm_check(opr1, 23)) 227 goto overflow; 228 229 opr1 >>= 2; 230 insn->reg1i21_format.simmediate_l = opr1 & 0xffff; 231 insn->reg1i21_format.simmediate_h = (opr1 >> 16) & 0x1f; 232 return 0; 233 case R_LARCH_SOP_POP_32_S_0_10_10_16_S2: 234 if (opr1 % 4) 235 goto unaligned; 236 237 if (!signed_imm_check(opr1, 28)) 238 goto overflow; 239 240 opr1 >>= 2; 241 insn->reg0i26_format.simmediate_l = opr1 & 0xffff; 242 insn->reg0i26_format.simmediate_h = (opr1 >> 16) & 0x3ff; 243 return 0; 244 case R_LARCH_SOP_POP_32_U: 245 if (!unsigned_imm_check(opr1, 32)) 246 goto overflow; 247 248 /* (*(uint32_t *) PC) = opr */ 249 *location = (u32)opr1; 250 return 0; 251 default: 252 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 253 return -EINVAL; 254 } 255 256overflow: 257 pr_err("module %s: opr1 = 0x%llx overflow! dangerous %s (%u) relocation\n", 258 mod->name, opr1, __func__, type); 259 return -ENOEXEC; 260 261unaligned: 262 pr_err("module %s: opr1 = 0x%llx unaligned! dangerous %s (%u) relocation\n", 263 mod->name, opr1, __func__, type); 264 return -ENOEXEC; 265} 266 267static int apply_r_larch_add_sub(struct module *mod, u32 *location, Elf_Addr v, 268 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 269{ 270 switch (type) { 271 case R_LARCH_ADD32: 272 *(s32 *)location += v; 273 return 0; 274 case R_LARCH_ADD64: 275 *(s64 *)location += v; 276 return 0; 277 case R_LARCH_SUB32: 278 *(s32 *)location -= v; 279 return 0; 280 case R_LARCH_SUB64: 281 *(s64 *)location -= v; 282 return 0; 283 default: 284 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 285 return -EINVAL; 286 } 287} 288 289static int apply_r_larch_b26(struct module *mod, 290 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 291 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 292{ 293 ptrdiff_t offset = (void *)v - (void *)location; 294 union loongarch_instruction *insn = (union loongarch_instruction *)location; 295 296 if (offset >= SZ_128M) 297 v = module_emit_plt_entry(mod, sechdrs, v); 298 299 if (offset < -SZ_128M) 300 v = module_emit_plt_entry(mod, sechdrs, v); 301 302 offset = (void *)v - (void *)location; 303 304 if (offset & 3) { 305 pr_err("module %s: jump offset = 0x%llx unaligned! dangerous R_LARCH_B26 (%u) relocation\n", 306 mod->name, (long long)offset, type); 307 return -ENOEXEC; 308 } 309 310 if (!signed_imm_check(offset, 28)) { 311 pr_err("module %s: jump offset = 0x%llx overflow! dangerous R_LARCH_B26 (%u) relocation\n", 312 mod->name, (long long)offset, type); 313 return -ENOEXEC; 314 } 315 316 offset >>= 2; 317 insn->reg0i26_format.simmediate_l = offset & 0xffff; 318 insn->reg0i26_format.simmediate_h = (offset >> 16) & 0x3ff; 319 320 return 0; 321} 322 323static int apply_r_larch_pcala(struct module *mod, u32 *location, Elf_Addr v, 324 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 325{ 326 union loongarch_instruction *insn = (union loongarch_instruction *)location; 327 /* Use s32 for a sign-extension deliberately. */ 328 s32 offset_hi20 = (void *)((v + 0x800) & ~0xfff) - 329 (void *)((Elf_Addr)location & ~0xfff); 330 Elf_Addr anchor = (((Elf_Addr)location) & ~0xfff) + offset_hi20; 331 ptrdiff_t offset_rem = (void *)v - (void *)anchor; 332 333 switch (type) { 334 case R_LARCH_PCALA_LO12: 335 insn->reg2i12_format.simmediate = v & 0xfff; 336 break; 337 case R_LARCH_PCALA_HI20: 338 v = offset_hi20 >> 12; 339 insn->reg1i20_format.simmediate = v & 0xfffff; 340 break; 341 case R_LARCH_PCALA64_LO20: 342 v = offset_rem >> 32; 343 insn->reg1i20_format.simmediate = v & 0xfffff; 344 break; 345 case R_LARCH_PCALA64_HI12: 346 v = offset_rem >> 52; 347 insn->reg2i12_format.simmediate = v & 0xfff; 348 break; 349 default: 350 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 351 return -EINVAL; 352 } 353 354 return 0; 355} 356 357static int apply_r_larch_got_pc(struct module *mod, 358 Elf_Shdr *sechdrs, u32 *location, Elf_Addr v, 359 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 360{ 361 Elf_Addr got = module_emit_got_entry(mod, sechdrs, v); 362 363 if (!got) 364 return -EINVAL; 365 366 switch (type) { 367 case R_LARCH_GOT_PC_LO12: 368 type = R_LARCH_PCALA_LO12; 369 break; 370 case R_LARCH_GOT_PC_HI20: 371 type = R_LARCH_PCALA_HI20; 372 break; 373 default: 374 pr_err("%s: Unsupport relocation type %u\n", mod->name, type); 375 return -EINVAL; 376 } 377 378 return apply_r_larch_pcala(mod, location, got, rela_stack, rela_stack_top, type); 379} 380 381static int apply_r_larch_32_pcrel(struct module *mod, u32 *location, Elf_Addr v, 382 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 383{ 384 ptrdiff_t offset = (void *)v - (void *)location; 385 386 *(u32 *)location = offset; 387 return 0; 388} 389 390static int apply_r_larch_64_pcrel(struct module *mod, u32 *location, Elf_Addr v, 391 s64 *rela_stack, size_t *rela_stack_top, unsigned int type) 392{ 393 ptrdiff_t offset = (void *)v - (void *)location; 394 395 *(u64 *)location = offset; 396 return 0; 397} 398 399/* 400 * reloc_handlers_rela() - Apply a particular relocation to a module 401 * @mod: the module to apply the reloc to 402 * @location: the address at which the reloc is to be applied 403 * @v: the value of the reloc, with addend for RELA-style 404 * @rela_stack: the stack used for store relocation info, LOCAL to THIS module 405 * @rela_stac_top: where the stack operation(pop/push) applies to 406 * 407 * Return: 0 upon success, else -ERRNO 408 */ 409typedef int (*reloc_rela_handler)(struct module *mod, u32 *location, Elf_Addr v, 410 s64 *rela_stack, size_t *rela_stack_top, unsigned int type); 411 412/* The handlers for known reloc types */ 413static reloc_rela_handler reloc_rela_handlers[] = { 414 [R_LARCH_NONE ... R_LARCH_64_PCREL] = apply_r_larch_error, 415 416 [R_LARCH_NONE] = apply_r_larch_none, 417 [R_LARCH_32] = apply_r_larch_32, 418 [R_LARCH_64] = apply_r_larch_64, 419 [R_LARCH_MARK_LA] = apply_r_larch_none, 420 [R_LARCH_MARK_PCREL] = apply_r_larch_none, 421 [R_LARCH_SOP_PUSH_PCREL] = apply_r_larch_sop_push_pcrel, 422 [R_LARCH_SOP_PUSH_ABSOLUTE] = apply_r_larch_sop_push_absolute, 423 [R_LARCH_SOP_PUSH_DUP] = apply_r_larch_sop_push_dup, 424 [R_LARCH_SOP_SUB ... R_LARCH_SOP_IF_ELSE] = apply_r_larch_sop, 425 [R_LARCH_SOP_POP_32_S_10_5 ... R_LARCH_SOP_POP_32_U] = apply_r_larch_sop_imm_field, 426 [R_LARCH_ADD32 ... R_LARCH_SUB64] = apply_r_larch_add_sub, 427 [R_LARCH_PCALA_HI20...R_LARCH_PCALA64_HI12] = apply_r_larch_pcala, 428 [R_LARCH_32_PCREL] = apply_r_larch_32_pcrel, 429 [R_LARCH_64_PCREL] = apply_r_larch_64_pcrel, 430}; 431 432int apply_relocate_add(Elf_Shdr *sechdrs, const char *strtab, 433 unsigned int symindex, unsigned int relsec, 434 struct module *mod) 435{ 436 int i, err; 437 unsigned int type; 438 s64 rela_stack[RELA_STACK_DEPTH]; 439 size_t rela_stack_top = 0; 440 reloc_rela_handler handler; 441 void *location; 442 Elf_Addr v; 443 Elf_Sym *sym; 444 Elf_Rela *rel = (void *) sechdrs[relsec].sh_addr; 445 446 pr_debug("%s: Applying relocate section %u to %u\n", __func__, relsec, 447 sechdrs[relsec].sh_info); 448 449 rela_stack_top = 0; 450 for (i = 0; i < sechdrs[relsec].sh_size / sizeof(*rel); i++) { 451 /* This is where to make the change */ 452 location = (void *)sechdrs[sechdrs[relsec].sh_info].sh_addr + rel[i].r_offset; 453 /* This is the symbol it is referring to */ 454 sym = (Elf_Sym *)sechdrs[symindex].sh_addr + ELF_R_SYM(rel[i].r_info); 455 if (IS_ERR_VALUE(sym->st_value)) { 456 /* Ignore unresolved weak symbol */ 457 if (ELF_ST_BIND(sym->st_info) == STB_WEAK) 458 continue; 459 pr_warn("%s: Unknown symbol %s\n", mod->name, strtab + sym->st_name); 460 return -ENOENT; 461 } 462 463 type = ELF_R_TYPE(rel[i].r_info); 464 465 if (type < ARRAY_SIZE(reloc_rela_handlers)) 466 handler = reloc_rela_handlers[type]; 467 else 468 handler = NULL; 469 470 if (!handler) { 471 pr_err("%s: Unknown relocation type %u\n", mod->name, type); 472 return -EINVAL; 473 } 474 475 pr_debug("type %d st_value %llx r_addend %llx loc %llx\n", 476 (int)ELF_R_TYPE(rel[i].r_info), 477 sym->st_value, rel[i].r_addend, (u64)location); 478 479 v = sym->st_value + rel[i].r_addend; 480 switch (type) { 481 case R_LARCH_B26: 482 err = apply_r_larch_b26(mod, sechdrs, location, 483 v, rela_stack, &rela_stack_top, type); 484 break; 485 case R_LARCH_GOT_PC_HI20...R_LARCH_GOT_PC_LO12: 486 err = apply_r_larch_got_pc(mod, sechdrs, location, 487 v, rela_stack, &rela_stack_top, type); 488 break; 489 case R_LARCH_SOP_PUSH_PLT_PCREL: 490 err = apply_r_larch_sop_push_plt_pcrel(mod, sechdrs, location, 491 v, rela_stack, &rela_stack_top, type); 492 break; 493 default: 494 err = handler(mod, location, v, rela_stack, &rela_stack_top, type); 495 } 496 if (err) 497 return err; 498 } 499 500 return 0; 501} 502 503void *module_alloc(unsigned long size) 504{ 505 return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END, 506 GFP_KERNEL, PAGE_KERNEL, 0, NUMA_NO_NODE, __builtin_return_address(0)); 507} 508 509static int module_init_ftrace_plt(const Elf_Ehdr *hdr, 510 const Elf_Shdr *sechdrs, struct module *mod) 511{ 512#ifdef CONFIG_DYNAMIC_FTRACE 513 struct plt_entry *ftrace_plts; 514 515 ftrace_plts = (void *)sechdrs->sh_addr; 516 517 ftrace_plts[FTRACE_PLT_IDX] = emit_plt_entry(FTRACE_ADDR); 518 519 if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) 520 ftrace_plts[FTRACE_REGS_PLT_IDX] = emit_plt_entry(FTRACE_REGS_ADDR); 521 522 mod->arch.ftrace_trampolines = ftrace_plts; 523#endif 524 return 0; 525} 526 527int module_finalize(const Elf_Ehdr *hdr, 528 const Elf_Shdr *sechdrs, struct module *mod) 529{ 530 char *secstrings; 531 const Elf_Shdr *s, *orc = NULL, *orc_ip = NULL, *alt = NULL, *ftrace = NULL; 532 533 secstrings = (void *)hdr + sechdrs[hdr->e_shstrndx].sh_offset; 534 535 for (s = sechdrs; s < sechdrs + hdr->e_shnum; s++) { 536 if (!strcmp(".altinstructions", secstrings + s->sh_name)) 537 alt = s; 538 if (!strcmp(".orc_unwind", secstrings + s->sh_name)) 539 orc = s; 540 if (!strcmp(".orc_unwind_ip", secstrings + s->sh_name)) 541 orc_ip = s; 542 if (!strcmp(".ftrace_trampoline", secstrings + s->sh_name)) 543 ftrace = s; 544 } 545 546 if (alt) 547 apply_alternatives((void *)alt->sh_addr, (void *)alt->sh_addr + alt->sh_size); 548 549 if (orc && orc_ip) 550 unwind_module_init(mod, (void *)orc_ip->sh_addr, orc_ip->sh_size, (void *)orc->sh_addr, orc->sh_size); 551 552 return module_init_ftrace_plt(hdr, ftrace, mod); 553} 554