Lines Matching refs:jit
111 static inline void reg_set_seen(struct bpf_jit *jit, u32 b1)
115 if (r1 >= 6 && r1 <= 15 && !jit->seen_reg[r1])
116 jit->seen_reg[r1] = 1;
121 reg_set_seen(jit, b1); \
124 #define REG_SEEN(b1) jit->seen_reg[reg2hex[(b1)]]
132 if (jit->prg_buf) \
133 *(u16 *) (jit->prg_buf + jit->prg) = (op); \
134 jit->prg += 2; \
146 if (jit->prg_buf) \
147 *(u32 *) (jit->prg_buf + jit->prg) = (op); \
148 jit->prg += 4; \
195 int __rel = ((target) - jit->prg) / 2; \
201 if (jit->prg_buf) { \
202 *(u32 *) (jit->prg_buf + jit->prg) = (op1); \
203 *(u16 *) (jit->prg_buf + jit->prg + 4) = (op2); \
205 jit->prg += 6; \
233 unsigned int rel = (int)((target) - jit->prg) / 2; \
242 unsigned int rel = (int)((target) - jit->prg) / 2; \
251 int rel = (addrs[(i) + (off) + 1] - jit->prg) / 2; \
259 unsigned int rel = (int)((target) - jit->prg) / 2; \
266 unsigned int rel = (int)((target) - jit->prg) / 2; \
290 ret = jit->lit32; \
291 if (jit->prg_buf) \
292 *(u32 *)(jit->prg_buf + jit->lit32) = (u32)(val);\
293 jit->lit32 += 4; \
299 jit->seen |= SEEN_LITERAL; \
300 _EMIT_CONST_U32(val) - jit->base_ip; \
306 ret = jit->lit64; \
307 if (jit->prg_buf) \
308 *(u64 *)(jit->prg_buf + jit->lit64) = (u64)(val);\
309 jit->lit64 += 8; \
315 jit->seen |= SEEN_LITERAL; \
316 _EMIT_CONST_U64(val) - jit->base_ip; \
332 static bool is_first_pass(struct bpf_jit *jit)
334 return jit->size == 0;
341 static bool is_codegen_pass(struct bpf_jit *jit)
343 return jit->prg_buf;
357 static bool can_use_rel(struct bpf_jit *jit, int off)
359 return is_valid_rel(off - jit->prg);
375 static bool can_use_ldisp_for_lit32(struct bpf_jit *jit)
377 return is_valid_ldisp(jit->lit32 - jit->base_ip);
384 static bool can_use_ldisp_for_lit64(struct bpf_jit *jit)
386 return is_valid_ldisp(jit->lit64 - jit->base_ip);
400 static void save_regs(struct bpf_jit *jit, u32 rs, u32 re)
415 static void restore_regs(struct bpf_jit *jit, u32 rs, u32 re, u32 stack_depth)
419 if (jit->seen & SEEN_STACK)
433 static int get_start(struct bpf_jit *jit, int start)
438 if (jit->seen_reg[i])
447 static int get_end(struct bpf_jit *jit, int start)
452 if (!jit->seen_reg[i] && !jit->seen_reg[i + 1])
455 return jit->seen_reg[15] ? 15 : 14;
464 static void save_restore_regs(struct bpf_jit *jit, int op, u32 stack_depth)
469 if (is_first_pass(jit)) {
474 jit->prg += (last - re + 1) * save_restore_size;
479 rs = get_start(jit, re);
482 re = get_end(jit, rs + 1);
484 save_regs(jit, rs, re);
486 restore_regs(jit, rs, re, stack_depth);
491 static void bpf_skip(struct bpf_jit *jit, int size)
515 static void bpf_jit_prologue(struct bpf_jit *jit, u32 stack_depth)
517 if (jit->seen & SEEN_TAIL_CALL) {
525 bpf_skip(jit, 6);
528 jit->tail_call_start = jit->prg;
530 save_restore_regs(jit, REGS_SAVE, stack_depth);
532 if (is_first_pass(jit) || (jit->seen & SEEN_LITERAL)) {
533 if (!is_first_pass(jit) &&
534 is_valid_ldisp(jit->size - (jit->prg + 2))) {
537 jit->base_ip = jit->prg;
540 EMIT6_PCREL_RILB(0xc0000000, REG_L, jit->lit32_start);
541 jit->base_ip = jit->lit32_start;
545 if (is_first_pass(jit) || (jit->seen & SEEN_STACK)) {
546 if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
553 if (is_first_pass(jit) || (jit->seen & SEEN_FUNC))
563 static void bpf_jit_epilogue(struct bpf_jit *jit, u32 stack_depth)
565 jit->exit_ip = jit->prg;
569 save_restore_regs(jit, REGS_RESTORE, stack_depth);
571 jit->r14_thunk_ip = jit->prg;
575 EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
578 EMIT6_PCREL_RILB(0xc0000000, REG_1, jit->prg + 14);
589 (is_first_pass(jit) || (jit->seen & SEEN_FUNC))) {
590 jit->r1_thunk_ip = jit->prg;
594 EMIT6_PCREL_RIL(0xc6000000, jit->prg + 10);
641 static int bpf_jit_probe_mem(struct bpf_jit *jit, struct bpf_prog *fp,
653 insn = jit->prg_buf + probe_prg;
661 if (WARN_ON_ONCE(jit->excnt >= fp->aux->num_exentries))
664 ex = &fp->aux->extable[jit->excnt];
667 delta = jit->prg_buf + prg - (u8 *)&ex->insn;
676 delta = jit->prg_buf + nop_prg - (u8 *)&ex->fixup;
682 jit->excnt++;
693 static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
700 u32 *addrs = jit->addrs;
710 probe_prg = jit->prg;
882 if (!is_first_pass(jit) && can_use_ldisp_for_lit32(jit)) {
890 jit->seen |= SEEN_LITERAL;
915 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
923 jit->seen |= SEEN_LITERAL;
949 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
958 jit->seen |= SEEN_LITERAL;
981 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
990 jit->seen |= SEEN_LITERAL;
1015 if (!is_first_pass(jit) && can_use_ldisp_for_lit64(jit)) {
1024 jit->seen |= SEEN_LITERAL;
1173 jit->seen |= SEEN_MEM;
1178 jit->seen |= SEEN_MEM;
1183 jit->seen |= SEEN_MEM;
1188 jit->seen |= SEEN_MEM;
1195 jit->seen |= SEEN_MEM;
1202 jit->seen |= SEEN_MEM;
1209 jit->seen |= SEEN_MEM;
1216 jit->seen |= SEEN_MEM;
1225 jit->seen |= SEEN_MEM;
1231 jit->seen |= SEEN_MEM;
1240 jit->seen |= SEEN_MEM;
1248 jit->seen |= SEEN_MEM;
1255 jit->seen |= SEEN_MEM;
1263 jit->seen |= SEEN_MEM;
1281 jit->seen |= SEEN_FUNC;
1286 EMIT6_PCREL_RILB(0xc0050000, REG_14, jit->r1_thunk_ip);
1304 jit->seen |= SEEN_TAIL_CALL;
1316 patch_1_clrj = jit->prg;
1318 jit->prg);
1325 if (jit->seen & SEEN_STACK)
1334 patch_2_clij = jit->prg;
1336 2, jit->prg);
1352 patch_3_brc = jit->prg;
1353 EMIT4_PCREL_RIC(0xa7040000, 8, jit->prg);
1358 save_restore_regs(jit, REGS_RESTORE, stack_depth);
1368 _EMIT4(0x47f01000 + jit->tail_call_start);
1370 if (jit->prg_buf) {
1371 *(u16 *)(jit->prg_buf + patch_1_clrj + 2) =
1372 (jit->prg - patch_1_clrj) >> 1;
1373 *(u16 *)(jit->prg_buf + patch_2_clij + 2) =
1374 (jit->prg - patch_2_clij) >> 1;
1375 *(u16 *)(jit->prg_buf + patch_3_brc + 2) =
1376 (jit->prg - patch_3_brc) >> 1;
1384 if (!is_first_pass(jit) && can_use_rel(jit, jit->exit_ip))
1386 EMIT4_PCREL_RIC(0xa7040000, 0xf, jit->exit_ip);
1389 EMIT6_PCREL_RILC(0xc0040000, 0xf, jit->exit_ip);
1525 if (!is_first_pass(jit) &&
1526 can_use_rel(jit, addrs[i + off + 1])) {
1543 if (!is_first_pass(jit) &&
1544 can_use_rel(jit, addrs[i + off + 1])) {
1561 if (!is_first_pass(jit) &&
1562 can_use_rel(jit, addrs[i + off + 1])) {
1578 if (!is_first_pass(jit) &&
1579 can_use_rel(jit, addrs[i + off + 1])) {
1603 nop_prg = jit->prg;
1606 err = bpf_jit_probe_mem(jit, fp, probe_prg, nop_prg);
1617 static bool bpf_is_new_addr_sane(struct bpf_jit *jit, int i)
1620 if (is_first_pass(jit))
1624 if (is_codegen_pass(jit))
1625 return jit->addrs[i] == jit->prg;
1628 return jit->addrs[i] >= jit->prg;
1634 static int bpf_set_addr(struct bpf_jit *jit, int i)
1638 if (is_codegen_pass(jit)) {
1639 delta = jit->prg - jit->addrs[i];
1641 bpf_skip(jit, -delta);
1643 if (WARN_ON_ONCE(!bpf_is_new_addr_sane(jit, i)))
1645 jit->addrs[i] = jit->prg;
1652 static int bpf_jit_prog(struct bpf_jit *jit, struct bpf_prog *fp,
1657 jit->lit32 = jit->lit32_start;
1658 jit->lit64 = jit->lit64_start;
1659 jit->prg = 0;
1660 jit->excnt = 0;
1662 bpf_jit_prologue(jit, stack_depth);
1663 if (bpf_set_addr(jit, 0) < 0)
1666 insn_count = bpf_jit_insn(jit, fp, i, extra_pass, stack_depth);
1670 if (bpf_set_addr(jit, i + insn_count) < 0)
1673 bpf_jit_epilogue(jit, stack_depth);
1675 lit32_size = jit->lit32 - jit->lit32_start;
1676 lit64_size = jit->lit64 - jit->lit64_start;
1677 jit->lit32_start = jit->prg;
1679 jit->lit32_start = ALIGN(jit->lit32_start, 4);
1680 jit->lit64_start = jit->lit32_start + lit32_size;
1682 jit->lit64_start = ALIGN(jit->lit64_start, 8);
1683 jit->size = jit->lit64_start + lit64_size;
1684 jit->size_prg = jit->prg;
1687 jit->excnt != fp->aux->num_exentries))
1705 static struct bpf_binary_header *bpf_jit_alloc(struct bpf_jit *jit,
1715 code_size = roundup(jit->size,
1719 header = bpf_jit_binary_alloc(code_size + extable_size, &jit->prg_buf,
1724 (jit->prg_buf + code_size);
1739 struct bpf_jit jit;
1767 jit = jit_data->ctx;
1774 memset(&jit, 0, sizeof(jit));
1775 jit.addrs = kvcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
1776 if (jit.addrs == NULL) {
1786 if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {
1794 header = bpf_jit_alloc(&jit, fp);
1800 if (bpf_jit_prog(&jit, fp, extra_pass, stack_depth)) {
1806 bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
1807 print_fn_code(jit.prg_buf, jit.size_prg);
1813 jit_data->ctx = jit;
1816 fp->bpf_func = (void *) jit.prg_buf;
1818 fp->jited_len = jit.size;
1821 bpf_prog_fill_jited_linfo(fp, jit.addrs + 1);
1823 kvfree(jit.addrs);