Lines Matching defs:ctx
46 static void mark_call(struct jit_ctx *ctx)
48 ctx->flags |= SAVE_RA;
51 static void mark_tail_call(struct jit_ctx *ctx)
53 ctx->flags |= SAVE_TCC;
56 static bool seen_call(struct jit_ctx *ctx)
58 return (ctx->flags & SAVE_RA);
61 static bool seen_tail_call(struct jit_ctx *ctx)
63 return (ctx->flags & SAVE_TCC);
66 static u8 tail_call_reg(struct jit_ctx *ctx)
68 if (seen_call(ctx))
100 static void build_prologue(struct jit_ctx *ctx)
104 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16);
123 emit_insn(ctx, addid, regmap[REG_TCC], LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT);
125 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust);
128 emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, store_offset);
131 emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, store_offset);
134 emit_insn(ctx, std, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, store_offset);
137 emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, store_offset);
140 emit_insn(ctx, std, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, store_offset);
143 emit_insn(ctx, std, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, store_offset);
146 emit_insn(ctx, std, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, store_offset);
149 emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset);
151 emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust);
154 emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust);
160 if (seen_tail_call(ctx) && seen_call(ctx))
161 move_reg(ctx, regmap[TCC_SAVED], regmap[REG_TCC]);
163 ctx->stack_size = stack_adjust;
166 static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call)
168 int stack_adjust = ctx->stack_size;
172 emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, load_offset);
175 emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, load_offset);
178 emit_insn(ctx, ldd, LOONGARCH_GPR_S0, LOONGARCH_GPR_SP, load_offset);
181 emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_SP, load_offset);
184 emit_insn(ctx, ldd, LOONGARCH_GPR_S2, LOONGARCH_GPR_SP, load_offset);
187 emit_insn(ctx, ldd, LOONGARCH_GPR_S3, LOONGARCH_GPR_SP, load_offset);
190 emit_insn(ctx, ldd, LOONGARCH_GPR_S4, LOONGARCH_GPR_SP, load_offset);
193 emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset);
195 emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust);
199 move_reg(ctx, LOONGARCH_GPR_A0, regmap[BPF_REG_0]);
201 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0);
207 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, regmap[TMP_REG_3], 1);
211 void build_epilogue(struct jit_ctx *ctx)
213 __build_epilogue(ctx, false);
218 static int emit_bpf_tail_call(struct jit_ctx *ctx)
221 u8 tcc = tail_call_reg(ctx);
227 const int idx0 = ctx->idx;
229 #define cur_offset (ctx->idx - idx0)
233 * a0: &ctx
241 emit_insn(ctx, ldwu, tmp1, a1, off);
243 emit_tailcall_jump(ctx, BPF_JGE, a2, tmp1, jmp_offset);
249 emit_insn(ctx, addid, tmp1, tcc, -1);
250 emit_tailcall_jump(ctx, BPF_JSLT, tcc, LOONGARCH_GPR_ZERO, jmp_offset);
257 emit_insn(ctx, sllid, tmp2, a2, 3);
258 emit_insn(ctx, addd, tmp2, tmp2, a1);
260 emit_insn(ctx, ldd, tmp2, tmp2, off);
262 emit_tailcall_jump(ctx, BPF_JEQ, tmp2, LOONGARCH_GPR_ZERO, jmp_offset);
266 emit_insn(ctx, ldd, tmp3, tmp2, off);
267 move_reg(ctx, tcc, tmp1);
268 __build_epilogue(ctx, true);
284 static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool extra_pass)
295 int i = insn - ctx->prog->insnsi;
306 move_reg(ctx, dst, src);
307 emit_zext_32(ctx, dst, is32);
312 move_imm32(ctx, dst, imm, is32);
318 emit_insn(ctx, addd, dst, dst, src);
319 emit_zext_32(ctx, dst, is32);
325 emit_insn(ctx, addid, dst, dst, imm);
327 move_imm32(ctx, tmp, imm, is32);
328 emit_insn(ctx, addd, dst, dst, tmp);
330 emit_zext_32(ctx, dst, is32);
336 emit_insn(ctx, subd, dst, dst, src);
337 emit_zext_32(ctx, dst, is32);
343 emit_insn(ctx, addid, dst, dst, -imm);
345 move_imm32(ctx, tmp, imm, is32);
346 emit_insn(ctx, subd, dst, dst, tmp);
348 emit_zext_32(ctx, dst, is32);
354 emit_insn(ctx, muld, dst, dst, src);
355 emit_zext_32(ctx, dst, is32);
360 move_imm32(ctx, tmp, imm, is32);
361 emit_insn(ctx, muld, dst, dst, tmp);
362 emit_zext_32(ctx, dst, is32);
368 emit_insn(ctx, divdu, dst, dst, src);
369 emit_zext_32(ctx, dst, is32);
374 move_imm32(ctx, tmp, imm, is32);
375 emit_insn(ctx, divdu, dst, dst, tmp);
376 emit_zext_32(ctx, dst, is32);
382 emit_insn(ctx, moddu, dst, dst, src);
383 emit_zext_32(ctx, dst, is32);
388 move_imm32(ctx, tmp, imm, is32);
389 emit_insn(ctx, moddu, dst, dst, tmp);
390 emit_zext_32(ctx, dst, is32);
396 move_imm32(ctx, tmp, imm, is32);
397 emit_insn(ctx, subd, dst, LOONGARCH_GPR_ZERO, dst);
398 emit_zext_32(ctx, dst, is32);
404 emit_insn(ctx, and, dst, dst, src);
405 emit_zext_32(ctx, dst, is32);
411 emit_insn(ctx, andi, dst, dst, imm);
413 move_imm32(ctx, tmp, imm, is32);
414 emit_insn(ctx, and, dst, dst, tmp);
416 emit_zext_32(ctx, dst, is32);
422 emit_insn(ctx, or, dst, dst, src);
423 emit_zext_32(ctx, dst, is32);
429 emit_insn(ctx, ori, dst, dst, imm);
431 move_imm32(ctx, tmp, imm, is32);
432 emit_insn(ctx, or, dst, dst, tmp);
434 emit_zext_32(ctx, dst, is32);
440 emit_insn(ctx, xor, dst, dst, src);
441 emit_zext_32(ctx, dst, is32);
447 emit_insn(ctx, xori, dst, dst, imm);
449 move_imm32(ctx, tmp, imm, is32);
450 emit_insn(ctx, xor, dst, dst, tmp);
452 emit_zext_32(ctx, dst, is32);
457 emit_insn(ctx, sllw, dst, dst, src);
458 emit_zext_32(ctx, dst, is32);
461 emit_insn(ctx, slld, dst, dst, src);
465 emit_insn(ctx, slliw, dst, dst, imm);
466 emit_zext_32(ctx, dst, is32);
469 emit_insn(ctx, sllid, dst, dst, imm);
474 emit_insn(ctx, srlw, dst, dst, src);
475 emit_zext_32(ctx, dst, is32);
478 emit_insn(ctx, srld, dst, dst, src);
482 emit_insn(ctx, srliw, dst, dst, imm);
483 emit_zext_32(ctx, dst, is32);
486 emit_insn(ctx, srlid, dst, dst, imm);
491 emit_insn(ctx, sraw, dst, dst, src);
492 emit_zext_32(ctx, dst, is32);
495 emit_insn(ctx, srad, dst, dst, src);
499 emit_insn(ctx, sraiw, dst, dst, imm);
500 emit_zext_32(ctx, dst, is32);
503 emit_insn(ctx, sraid, dst, dst, imm);
511 emit_insn(ctx, sllid, dst, dst, 48);
512 emit_insn(ctx, srlid, dst, dst, 48);
516 emit_zext_32(ctx, dst, is32);
526 emit_insn(ctx, revb2h, dst, dst);
528 emit_insn(ctx, sllid, dst, dst, 48);
529 emit_insn(ctx, srlid, dst, dst, 48);
532 emit_insn(ctx, revb2w, dst, dst);
534 emit_zext_32(ctx, dst, is32);
537 emit_insn(ctx, revbd, dst, dst);
553 jmp_offset = bpf2la_offset(i, off, ctx);
554 emit_cond_jump(ctx, cond, dst, src, jmp_offset);
568 jmp_offset = bpf2la_offset(i, off, ctx);
569 move_imm32(ctx, tmp, imm, is32);
570 emit_cond_jump(ctx, cond, dst, tmp, jmp_offset);
575 jmp_offset = bpf2la_offset(i, off, ctx);
576 emit_insn(ctx, and, tmp, dst, src);
577 emit_cond_jump(ctx, cond, tmp, LOONGARCH_GPR_ZERO, jmp_offset);
581 jmp_offset = bpf2la_offset(i, off, ctx);
582 move_imm32(ctx, tmp, imm, is32);
583 emit_insn(ctx, and, tmp, dst, tmp);
584 emit_cond_jump(ctx, cond, tmp, LOONGARCH_GPR_ZERO, jmp_offset);
589 jmp_offset = bpf2la_offset(i, off, ctx);
590 emit_uncond_jump(ctx, jmp_offset, false);
595 mark_call(ctx);
596 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass,
601 move_imm64(ctx, tmp, func_addr, is32);
602 emit_insn(ctx, jirl, LOONGARCH_GPR_RA, tmp, 0);
603 move_reg(ctx, regmap[BPF_REG_0], LOONGARCH_GPR_A0);
608 mark_tail_call(ctx);
609 if (emit_bpf_tail_call(ctx))
615 emit_sext_32(ctx, regmap[BPF_REG_0]);
620 if (i == ctx->prog->len - 1)
623 jmp_offset = epilogue_offset(ctx);
624 emit_uncond_jump(ctx, jmp_offset, true);
630 move_imm64(ctx, dst, imm64, is32);
641 emit_insn(ctx, ldbu, dst, src, off);
644 emit_insn(ctx, ldhu, dst, src, off);
647 emit_insn(ctx, ldwu, dst, src, off);
650 emit_insn(ctx, ldd, dst, src, off);
654 move_imm32(ctx, tmp, off, is32);
657 emit_insn(ctx, ldxbu, dst, src, tmp);
660 emit_insn(ctx, ldxhu, dst, src, tmp);
663 emit_insn(ctx, ldxwu, dst, src, tmp);
666 emit_insn(ctx, ldxd, dst, src, tmp);
677 move_imm32(ctx, tmp, imm, is32);
681 emit_insn(ctx, stb, tmp, dst, off);
684 emit_insn(ctx, sth, tmp, dst, off);
687 emit_insn(ctx, stw, tmp, dst, off);
690 emit_insn(ctx, std, tmp, dst, off);
694 move_imm32(ctx, tmp2, off, is32);
697 emit_insn(ctx, stxb, tmp, dst, tmp2);
700 emit_insn(ctx, stxh, tmp, dst, tmp2);
703 emit_insn(ctx, stxw, tmp, dst, tmp2);
706 emit_insn(ctx, stxd, tmp, dst, tmp2);
720 emit_insn(ctx, stb, src, dst, off);
723 emit_insn(ctx, sth, src, dst, off);
726 emit_insn(ctx, stw, src, dst, off);
729 emit_insn(ctx, std, src, dst, off);
733 move_imm32(ctx, tmp, off, is32);
736 emit_insn(ctx, stxb, src, dst, tmp);
739 emit_insn(ctx, stxh, src, dst, tmp);
742 emit_insn(ctx, stxw, src, dst, tmp);
745 emit_insn(ctx, stxd, src, dst, tmp);
759 move_imm32(ctx, tmp, off, is32);
760 emit_insn(ctx, addd, tmp, dst, tmp);
763 emit_insn(ctx, amaddw, tmp2, src, tmp);
766 emit_insn(ctx, amaddd, tmp2, src, tmp);
779 static int build_body(struct jit_ctx *ctx, bool extra_pass)
781 const struct bpf_prog *prog = ctx->prog;
788 if (!ctx->image)
789 ctx->offset[i] = ctx->idx;
791 ret = build_insn(insn, ctx, extra_pass);
794 if (!ctx->image)
795 ctx->offset[i] = ctx->idx;
802 if (!ctx->image)
803 ctx->offset[i] = ctx->idx;
823 static int validate_code(struct jit_ctx *ctx)
828 for (i = 0; i < ctx->idx; i++) {
829 insn = ctx->image[i];
843 struct jit_ctx ctx;
878 if (jit_data->ctx.offset) {
879 ctx = jit_data->ctx;
883 image_size = sizeof(u32) * ctx.idx;
887 memset(&ctx, 0, sizeof(ctx));
888 ctx.prog = prog;
890 ctx.offset = kcalloc(prog->len + 1, sizeof(*ctx.offset), GFP_KERNEL);
891 if (!ctx.offset) {
896 /* 1. Initial fake pass to compute ctx->idx and set ctx->flags */
897 if (build_body(&ctx, extra_pass)) {
901 build_prologue(&ctx);
902 ctx.epilogue_offset = ctx.idx;
903 build_epilogue(&ctx);
910 image_size = sizeof(u32) * ctx.idx;
920 ctx.image = (union loongarch_instruction *)image_ptr;
922 ctx.idx = 0;
924 build_prologue(&ctx);
925 if (build_body(&ctx, extra_pass)) {
930 build_epilogue(&ctx);
933 if (validate_code(&ctx)) {
941 bpf_jit_dump(prog->len, image_size, 2, ctx.image);
944 bpf_flush_icache(header, ctx.image + ctx.idx);
947 if (extra_pass && ctx.idx != jit_data->ctx.idx) {
949 ctx.idx, jit_data->ctx.idx);
957 jit_data->ctx = ctx;
961 prog->bpf_func = (void *)ctx.image;
967 kfree(ctx.offset);