Lines Matching refs:ctx
65 static inline void emit(const u32 insn, struct jit_ctx *ctx)
67 if (ctx->image != NULL)
68 ctx->image[ctx->idx] = cpu_to_le32(insn);
70 ctx->idx++;
74 const s32 val, struct jit_ctx *ctx)
81 emit(A64_MOVN(is64, reg, (u16)~lo, 0), ctx);
83 emit(A64_MOVN(is64, reg, (u16)~hi, 16), ctx);
85 emit(A64_MOVK(is64, reg, lo, 0), ctx);
88 emit(A64_MOVZ(is64, reg, lo, 0), ctx);
90 emit(A64_MOVK(is64, reg, hi, 16), ctx);
103 struct jit_ctx *ctx)
110 return emit_a64_mov_i(0, reg, (u32)val, ctx);
116 emit(A64_MOVN(1, reg, (rev_tmp >> shift) & 0xffff, shift), ctx);
118 emit(A64_MOVZ(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx);
122 emit(A64_MOVK(1, reg, (nrm_tmp >> shift) & 0xffff, shift), ctx);
133 struct jit_ctx *ctx)
138 emit(A64_MOVN(1, reg, ~tmp & 0xffff, shift), ctx);
142 emit(A64_MOVK(1, reg, tmp & 0xffff, shift), ctx);
147 const struct jit_ctx *ctx)
156 return ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1);
167 static inline int epilogue_offset(const struct jit_ctx *ctx)
169 int to = ctx->epilogue_offset;
170 int from = ctx->idx;
191 static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
193 const struct bpf_prog *prog = ctx->prog;
200 const int idx0 = ctx->idx;
217 * current A64_SP => +-----+ <= (BPF_FP - ctx->stack_size)
228 emit(A64_BTI_C, ctx);
231 emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
232 emit(A64_MOV(1, A64_FP, A64_SP), ctx);
235 emit(A64_PUSH(r6, r7, A64_SP), ctx);
236 emit(A64_PUSH(r8, r9, A64_SP), ctx);
237 emit(A64_PUSH(fp, tcc, A64_SP), ctx);
240 emit(A64_MOV(1, fp, A64_SP), ctx);
244 emit(A64_MOVZ(1, tcc, 0, 0), ctx);
246 cur_offset = ctx->idx - idx0;
255 emit(A64_BTI_J, ctx);
258 ctx->stack_size = STACK_ALIGN(prog->aux->stack_depth);
261 emit(A64_SUB_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
266 static int emit_bpf_tail_call(struct jit_ctx *ctx)
275 const int idx0 = ctx->idx;
276 #define cur_offset (ctx->idx - idx0)
284 emit_a64_mov_i64(tmp, off, ctx);
285 emit(A64_LDR32(tmp, r2, tmp), ctx);
286 emit(A64_MOV(0, r3, r3), ctx);
287 emit(A64_CMP(0, r3, tmp), ctx);
288 emit(A64_B_(A64_COND_CS, jmp_offset), ctx);
294 emit_a64_mov_i64(tmp, MAX_TAIL_CALL_CNT, ctx);
295 emit(A64_CMP(1, tcc, tmp), ctx);
296 emit(A64_B_(A64_COND_HI, jmp_offset), ctx);
297 emit(A64_ADD_I(1, tcc, tcc, 1), ctx);
304 emit_a64_mov_i64(tmp, off, ctx);
305 emit(A64_ADD(1, tmp, r2, tmp), ctx);
306 emit(A64_LSL(1, prg, r3, 3), ctx);
307 emit(A64_LDR64(prg, tmp, prg), ctx);
308 emit(A64_CBZ(1, prg, jmp_offset), ctx);
312 emit_a64_mov_i64(tmp, off, ctx);
313 emit(A64_LDR64(tmp, prg, tmp), ctx);
314 emit(A64_ADD_I(1, tmp, tmp, sizeof(u32) * PROLOGUE_OFFSET), ctx);
315 emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
316 emit(A64_BR(tmp), ctx);
331 static void build_epilogue(struct jit_ctx *ctx)
341 emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
344 emit(A64_POP(fp, A64_R(26), A64_SP), ctx);
347 emit(A64_POP(r8, r9, A64_SP), ctx);
348 emit(A64_POP(r6, r7, A64_SP), ctx);
351 emit(A64_POP(A64_FP, A64_LR, A64_SP), ctx);
354 emit(A64_MOV(1, A64_R(0), r0), ctx);
356 emit(A64_RET(A64_LR), ctx);
375 struct jit_ctx *ctx,
382 if (!ctx->image)
389 if (!ctx->prog->aux->extable ||
390 WARN_ON_ONCE(ctx->exentry_idx >= ctx->prog->aux->num_exentries))
393 ex = &ctx->prog->aux->extable[ctx->exentry_idx];
394 pc = (unsigned long)&ctx->image[ctx->idx - 1];
416 ctx->exentry_idx++;
426 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
437 const int i = insn - ctx->prog->insnsi;
461 emit(A64_MOV(is64, dst, src), ctx);
466 emit(A64_ADD(is64, dst, dst, src), ctx);
470 emit(A64_SUB(is64, dst, dst, src), ctx);
474 emit(A64_AND(is64, dst, dst, src), ctx);
478 emit(A64_ORR(is64, dst, dst, src), ctx);
482 emit(A64_EOR(is64, dst, dst, src), ctx);
486 emit(A64_MUL(is64, dst, dst, src), ctx);
494 emit(A64_UDIV(is64, dst, dst, src), ctx);
497 emit(A64_UDIV(is64, tmp, dst, src), ctx);
498 emit(A64_MSUB(is64, dst, dst, tmp, src), ctx);
504 emit(A64_LSLV(is64, dst, dst, src), ctx);
508 emit(A64_LSRV(is64, dst, dst, src), ctx);
512 emit(A64_ASRV(is64, dst, dst, src), ctx);
517 emit(A64_NEG(is64, dst, dst), ctx);
531 emit(A64_REV16(is64, dst, dst), ctx);
533 emit(A64_UXTH(is64, dst, dst), ctx);
536 emit(A64_REV32(is64, dst, dst), ctx);
540 emit(A64_REV64(dst, dst), ctx);
548 emit(A64_UXTH(is64, dst, dst), ctx);
552 emit(A64_UXTW(is64, dst, dst), ctx);
562 emit_a64_mov_i(is64, dst, imm, ctx);
568 emit(A64_ADD_I(is64, dst, dst, imm), ctx);
570 emit(A64_SUB_I(is64, dst, dst, -imm), ctx);
572 emit_a64_mov_i(is64, tmp, imm, ctx);
573 emit(A64_ADD(is64, dst, dst, tmp), ctx);
579 emit(A64_SUB_I(is64, dst, dst, imm), ctx);
581 emit(A64_ADD_I(is64, dst, dst, -imm), ctx);
583 emit_a64_mov_i(is64, tmp, imm, ctx);
584 emit(A64_SUB(is64, dst, dst, tmp), ctx);
591 emit(a64_insn, ctx);
593 emit_a64_mov_i(is64, tmp, imm, ctx);
594 emit(A64_AND(is64, dst, dst, tmp), ctx);
601 emit(a64_insn, ctx);
603 emit_a64_mov_i(is64, tmp, imm, ctx);
604 emit(A64_ORR(is64, dst, dst, tmp), ctx);
611 emit(a64_insn, ctx);
613 emit_a64_mov_i(is64, tmp, imm, ctx);
614 emit(A64_EOR(is64, dst, dst, tmp), ctx);
619 emit_a64_mov_i(is64, tmp, imm, ctx);
620 emit(A64_MUL(is64, dst, dst, tmp), ctx);
624 emit_a64_mov_i(is64, tmp, imm, ctx);
625 emit(A64_UDIV(is64, dst, dst, tmp), ctx);
629 emit_a64_mov_i(is64, tmp2, imm, ctx);
630 emit(A64_UDIV(is64, tmp, dst, tmp2), ctx);
631 emit(A64_MSUB(is64, dst, dst, tmp, tmp2), ctx);
635 emit(A64_LSL(is64, dst, dst, imm), ctx);
639 emit(A64_LSR(is64, dst, dst, imm), ctx);
643 emit(A64_ASR(is64, dst, dst, imm), ctx);
648 jmp_offset = bpf2a64_offset(i, off, ctx);
650 emit(A64_B(jmp_offset), ctx);
673 emit(A64_CMP(is64, dst, src), ctx);
675 jmp_offset = bpf2a64_offset(i, off, ctx);
712 emit(A64_B_(jmp_cond, jmp_offset), ctx);
716 emit(A64_TST(is64, dst, src), ctx);
740 emit(A64_CMP_I(is64, dst, imm), ctx);
742 emit(A64_CMN_I(is64, dst, -imm), ctx);
744 emit_a64_mov_i(is64, tmp, imm, ctx);
745 emit(A64_CMP(is64, dst, tmp), ctx);
752 emit(a64_insn, ctx);
754 emit_a64_mov_i(is64, tmp, imm, ctx);
755 emit(A64_TST(is64, dst, tmp), ctx);
765 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
769 emit_addr_mov_i64(tmp, func_addr, ctx);
770 emit(A64_BLR(tmp), ctx);
771 emit(A64_MOV(1, r0, A64_R(0)), ctx);
776 if (emit_bpf_tail_call(ctx))
783 if (i == ctx->prog->len - 1)
785 jmp_offset = epilogue_offset(ctx);
787 emit(A64_B(jmp_offset), ctx);
797 emit_a64_mov_i64(dst, imm64, ctx);
811 emit_a64_mov_i(1, tmp, off, ctx);
814 emit(A64_LDR32(dst, src, tmp), ctx);
817 emit(A64_LDRH(dst, src, tmp), ctx);
820 emit(A64_LDRB(dst, src, tmp), ctx);
823 emit(A64_LDR64(dst, src, tmp), ctx);
827 ret = add_exception_handler(insn, ctx, dst);
851 emit_a64_mov_i(1, tmp2, off, ctx);
852 emit_a64_mov_i(1, tmp, imm, ctx);
855 emit(A64_STR32(tmp, dst, tmp2), ctx);
858 emit(A64_STRH(tmp, dst, tmp2), ctx);
861 emit(A64_STRB(tmp, dst, tmp2), ctx);
864 emit(A64_STR64(tmp, dst, tmp2), ctx);
874 emit_a64_mov_i(1, tmp, off, ctx);
877 emit(A64_STR32(src, dst, tmp), ctx);
880 emit(A64_STRH(src, dst, tmp), ctx);
883 emit(A64_STRB(src, dst, tmp), ctx);
886 emit(A64_STR64(src, dst, tmp), ctx);
898 emit_a64_mov_i(1, tmp, off, ctx);
899 emit(A64_ADD(1, tmp, tmp, dst), ctx);
903 emit(A64_STADD(isdw, reg, src), ctx);
905 emit(A64_LDXR(isdw, tmp2, reg), ctx);
906 emit(A64_ADD(isdw, tmp2, tmp2, src), ctx);
907 emit(A64_STXR(isdw, tmp2, reg, tmp3), ctx);
910 emit(A64_CBNZ(0, tmp3, jmp_offset), ctx);
922 static int build_body(struct jit_ctx *ctx, bool extra_pass)
924 const struct bpf_prog *prog = ctx->prog;
940 if (ctx->image == NULL)
941 ctx->offset[i] = ctx->idx;
942 ret = build_insn(insn, ctx, extra_pass);
945 if (ctx->image == NULL)
946 ctx->offset[i] = ctx->idx;
957 if (ctx->image == NULL)
958 ctx->offset[i] = ctx->idx;
963 static int validate_code(struct jit_ctx *ctx)
967 for (i = 0; i < ctx->idx; i++) {
968 u32 a64_insn = le32_to_cpu(ctx->image[i]);
974 if (WARN_ON_ONCE(ctx->exentry_idx != ctx->prog->aux->num_exentries))
988 struct jit_ctx ctx;
1000 struct jit_ctx ctx;
1026 if (jit_data->ctx.offset) {
1027 ctx = jit_data->ctx;
1031 prog_size = sizeof(u32) * ctx.idx;
1034 memset(&ctx, 0, sizeof(ctx));
1035 ctx.prog = prog;
1037 ctx.offset = kcalloc(prog->len + 1, sizeof(int), GFP_KERNEL);
1038 if (ctx.offset == NULL) {
1044 * 1. Initial fake pass to compute ctx->idx and ctx->offset.
1046 * BPF line info needs ctx->offset[i] to be the offset of
1049 if (build_prologue(&ctx, was_classic)) {
1054 if (build_body(&ctx, extra_pass)) {
1059 ctx.epilogue_offset = ctx.idx;
1060 build_epilogue(&ctx);
1066 prog_size = sizeof(u32) * ctx.idx;
1077 ctx.image = (__le32 *)image_ptr;
1081 ctx.idx = 0;
1082 ctx.exentry_idx = 0;
1084 build_prologue(&ctx, was_classic);
1086 if (build_body(&ctx, extra_pass)) {
1092 build_epilogue(&ctx);
1095 if (validate_code(&ctx)) {
1103 bpf_jit_dump(prog->len, prog_size, 2, ctx.image);
1105 bpf_flush_icache(header, ctx.image + ctx.idx);
1108 if (extra_pass && ctx.idx != jit_data->ctx.idx) {
1110 ctx.idx, jit_data->ctx.idx);
1119 jit_data->ctx = ctx;
1123 prog->bpf_func = (void *)ctx.image;
1132 ctx.offset[i] *= AARCH64_INSN_SIZE;
1133 bpf_prog_fill_jited_linfo(prog, ctx.offset + 1);
1135 kfree(ctx.offset);