Lines Matching refs:jit
117 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
121 if (r1 >= 6 && r1 <= 15 && !jit->seen_reg[r1])
122 jit->seen_reg[r1] = 1;
127 reg_set_seen(jit, b1); \
130 #define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
138 if (jit->prg_buf) \
139 *(u16 *) (jit->prg_buf + jit->prg) = (op); \
140 jit->prg += 2; \
152 if (jit->prg_buf) \
153 *(u32 *) (jit->prg_buf + jit->prg) = (op); \
154 jit->prg += 4; \
201 int __rel = ((target) - jit->prg) / 2; \
207 if (jit->prg_buf) { \
208 *(u32 *) (jit->prg_buf + jit->prg) = (op1); \
209 *(u16 *) (jit->prg_buf + jit->prg + 4) = (op2); \
211 jit->prg += 6; \
239 unsigned int rel = (int)((target) - jit->prg) / 2; \
248 unsigned int rel = (int)((target) - jit->prg) / 2; \
257 int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2; \
265 unsigned int rel = (int)((target) - jit->prg) / 2; \
272 unsigned int rel = (int)((target) - jit->prg) / 2; \
296 ret = jit->lit32; \
297 if (jit->prg_buf) \
298 *(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\
299 jit->lit32 += 4; \
305 jit->seen |= SEEN_LITERAL; \
306 _EMIT_CONST_U32(val) - jit->base_ip; \
312 ret = jit->lit64; \
313 if (jit->prg_buf) \
314 *(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\
315 jit->lit64 += 8; \
321 jit->seen |= SEEN_LITERAL; \
322 _EMIT_CONST_U64(val) - jit->base_ip; \
338 static bool is_first_pass(struct bpf_jit *jit)
340 return jit->size == 0;
347 static bool is_codegen_pass(struct bpf_jit *jit)
349 return jit->prg_buf;
363 static bool can_use_rel(struct bpf_jit *jit, int off)
365 return is_valid_rel(off - jit->prg);
381 static bool can_use_ldisp_for_lit32(struct bpf_jit *jit)
383 return is_valid_ldisp(jit->lit32 - jit->base_ip);
390 static bool can_use_ldisp_for_lit64(struct bpf_jit *jit)
392 return is_valid_ldisp(jit->lit64 - jit->base_ip);
406 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)
421 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth)
425 if (jit->seen & SEEN_STACK)
439 static int get_start(struct bpf_jit *jit, int start)
444 if (jit->seen_reg[i])
453 static int get_end(struct bpf_jit *jit, int start)
458 if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])
461 return jit->seen_reg[15] ? 15 : 14;
470 static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)
475 if (is_first_pass(jit)) {
480 jit->prg += (last - re + 1) * save_restore_size;
485 rs = get_start(jit, re);
488 re = get_end(jit, rs + 1);
490 save_regs(jit, rs, re);
492 restore_regs(jit, rs, re, stack_depth);
497 static void bpf_skip(struct bpf_jit *jit, int size)
551 static void bpf_jit_prologue(struct bpf_jit *jit, struct bpf_prog *fp,
556 EMIT6_PCREL_RILC(0xc0040000, 0, jit->prologue_plt);
557 jit->prologue_plt_ret = jit->prg;
569 bpf_skip(jit, 6);
572 jit->tail_call_start = jit->prg;
574 save_restore_regs(jit, REGS_SAVE, stack_depth);
576 if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) {
577 if (!is_first_pass(jit) &&
578 is_valid_ldisp(jit->size - (jit->prg + 2))) {
581 jit->base_ip = jit->prg;
584 EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start);
585 jit->base_ip = jit->lit32_start;
589 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) {
590 if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
597 if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
607 static void emit_expoline(struct bpf_jit *jit)
610 EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
618 static void emit_r1_thunk(struct bpf_jit *jit)
621 jit->r1_thunk_ip = jit->prg;
622 emit_expoline(jit);
631 static void call_r1(struct bpf_jit *jit)
635 EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
644 static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
646 jit->exit_ip = jit->prg;
650 save_restore_regs(jit, REGS_RESTORE, stack_depth);
652 jit->r14_thunk_ip = jit->prg;
654 emit_expoline(jit);
659 if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
660 emit_r1_thunk(jit);
662 jit->prg = ALIGN(jit->prg, 8);
663 jit->prologue_plt = jit->prg;
664 if (jit->prg_buf)
665 bpf_jit_plt(jit->prg_buf + jit->prg,
666 jit->prg_buf + jit->prologue_plt_ret, NULL);
667 jit->prg += BPF_PLT_SIZE;
693 static int bpf_jit_probe_mem(struct bpf_jit *jit, struct bpf_prog *fp,
705 insn = jit->prg_buf + probe_prg;
714 if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries))
717 ex = &fp->aux->extable[jit->excnt];
720 delta = jit->prg_buf + prg - (u8 *)&ex->insn;
729 delta = jit->prg_buf + nop_prg - (u8 *)&ex->fixup;
736 jit->excnt++;
744 static int sign_extend(struct bpf_jit *jit, int r, u8 size, u8 flags)
775 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
782 u32 *addrs = jit->addrs;
792 probe_prg = jit->prg;
964 if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {
972 jit->seen |= SEEN_LITERAL;
997 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1005 jit->seen |= SEEN_LITERAL;
1031 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1040 jit->seen |= SEEN_LITERAL;
1063 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1072 jit->seen |= SEEN_LITERAL;
1097 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1106 jit->seen |= SEEN_LITERAL;
1255 jit->seen |= SEEN_MEM;
1260 jit->seen |= SEEN_MEM;
1265 jit->seen |= SEEN_MEM;
1270 jit->seen |= SEEN_MEM;
1277 jit->seen |= SEEN_MEM;
1284 jit->seen |= SEEN_MEM;
1291 jit->seen |= SEEN_MEM;
1298 jit->seen |= SEEN_MEM;
1347 EMIT4_PCREL_RIC(0xa7040000, 4, jit->prg - 6);
1363 jit->seen |= SEEN_MEM;
1373 jit->seen |= SEEN_MEM;
1381 jit->seen |= SEEN_MEM;
1388 jit->seen |= SEEN_MEM;
1396 jit->seen |= SEEN_MEM;
1415 jit->seen |= SEEN_FUNC;
1437 if (sign_extend(jit, BPF_REG_1 + j,
1447 call_r1(jit);
1470 patch_1_clrj = jit->prg;
1472 jit->prg);
1479 if (jit->seen & SEEN_STACK)
1488 patch_2_clij = jit->prg;
1490 2, jit->prg);
1506 patch_3_brc = jit->prg;
1507 EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg);
1512 save_restore_regs(jit, REGS_RESTORE, stack_depth);
1522 jit->seen |= SEEN_FUNC;
1524 EMIT4_IMM(0xa70b0000, REG_1, jit->tail_call_start);
1526 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->r1_thunk_ip);
1529 _EMIT4(0x47f01000 + jit->tail_call_start);
1532 if (jit->prg_buf) {
1533 *(u16 *)(jit->prg_buf + patch_1_clrj + 2) =
1534 (jit->prg - patch_1_clrj) >> 1;
1535 *(u16 *)(jit->prg_buf + patch_2_clij + 2) =
1536 (jit->prg - patch_2_clij) >> 1;
1537 *(u16 *)(jit->prg_buf + patch_3_brc + 2) =
1538 (jit->prg - patch_3_brc) >> 1;
1546 if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip))
1548 EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip);
1551 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip);
1687 if (!is_first_pass(jit) &&
1688 can_use_rel(jit, addrs[i + off + 1])) {
1705 if (!is_first_pass(jit) &&
1706 can_use_rel(jit, addrs[i + off + 1])) {
1723 if (!is_first_pass(jit) &&
1724 can_use_rel(jit, addrs[i + off + 1])) {
1740 if (!is_first_pass(jit) &&
1741 can_use_rel(jit, addrs[i + off + 1])) {
1765 nop_prg = jit->prg;
1768 err = bpf_jit_probe_mem(jit, fp, probe_prg, nop_prg);
1779 static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i)
1782 if (is_first_pass(jit))
1786 if (is_codegen_pass(jit))
1787 return jit->addrs[i] == jit->prg;
1790 return jit->addrs[i] >= jit->prg;
1796 static int bpf_set_addr(struct bpf_jit *jit, int i)
1800 if (is_codegen_pass(jit)) {
1801 delta = jit->prg - jit->addrs[i];
1803 bpf_skip(jit, -delta);
1805 if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i)))
1807 jit->addrs[i] = jit->prg;
1814 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,
1819 jit->lit32 = jit->lit32_start;
1820 jit->lit64 = jit->lit64_start;
1821 jit->prg = 0;
1822 jit->excnt = 0;
1824 bpf_jit_prologue(jit, fp, stack_depth);
1825 if (bpf_set_addr(jit, 0) < 0)
1828 insn_count = bpf_jit_insn(jit, fp, i, extra_pass, stack_depth);
1832 if (bpf_set_addr(jit, i + insn_count) < 0)
1835 bpf_jit_epilogue(jit, stack_depth);
1837 lit32_size = jit->lit32 - jit->lit32_start;
1838 lit64_size = jit->lit64 - jit->lit64_start;
1839 jit->lit32_start = jit->prg;
1841 jit->lit32_start = ALIGN(jit->lit32_start, 4);
1842 jit->lit64_start = jit->lit32_start + lit32_size;
1844 jit->lit64_start = ALIGN(jit->lit64_start, 8);
1845 jit->size = jit->lit64_start + lit64_size;
1846 jit->size_prg = jit->prg;
1849 jit->excnt != fp->aux->num_exentries))
1867 static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
1877 code_size = roundup(jit->size,
1881 header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf,
1886 (jit->prg_buf + code_size);
1901 struct bpf_jit jit;
1932 jit = jit_data->ctx;
1939 memset(&jit, 0, sizeof(jit));
1940 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
1941 if (jit.addrs == NULL) {
1951 if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {
1959 header = bpf_jit_alloc(&jit, fp);
1965 if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {
1971 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
1972 print_fn_code(jit.prg_buf, jit.size_prg);
1978 jit_data->ctx = jit;
1981 fp->bpf_func = (void *) jit.prg_buf;
1983 fp->jited_len = jit.size;
1986 bpf_prog_fill_jited_linfo(fp, jit.addrs + 1);
1988 kvfree(jit.addrs);
2097 static void load_imm64(struct bpf_jit *jit, int dst_reg, u64 val)
2109 struct bpf_jit *jit = &tjit->common;
2120 load_imm64(jit, REG_W0, tlink->cookie);
2130 load_imm64(jit, REG_1, (u64)bpf_trampoline_enter(p));
2132 load_imm64(jit, REG_2, (u64)p);
2136 call_r1(jit);
2140 patch = jit->prg;
2148 load_imm64(jit, REG_1, (u64)p->bpf_func);
2153 load_imm64(jit, REG_3, (u64)p->insnsi);
2155 call_r1(jit);
2158 if (sign_extend(jit, REG_2, m->ret_size, m->ret_flags))
2165 if (jit->prg_buf)
2166 *(u32 *)&jit->prg_buf[patch + 2] = (jit->prg - patch) >> 1;
2173 load_imm64(jit, REG_1, (u64)bpf_trampoline_exit(p));
2175 load_imm64(jit, REG_2, (u64)p);
2181 call_r1(jit);
2215 struct bpf_jit *jit = &tjit->common;
2342 load_imm64(jit, REG_8, (u64)func_addr + S390X_PATCH_SIZE);
2352 load_imm64(jit, REG_0, (u64)func_addr);
2369 load_imm64(jit, REG_1, (u64)__bpf_tramp_enter);
2371 load_imm64(jit, REG_2, (u64)im);
2373 call_r1(jit);
2428 call_r1(jit);
2433 im->ip_after_call = jit->prg_buf + jit->prg;
2444 tjit->do_fexit = jit->prg;
2450 im->ip_epilogue = jit->prg_buf + jit->prg;
2457 load_imm64(jit, REG_1, (u64)__bpf_tramp_exit);
2459 load_imm64(jit, REG_2, (u64)im);
2461 call_r1(jit);
2488 emit_expoline(jit);
2496 emit_r1_thunk(jit);