Lines Matching refs:ctx
119 static enum reg_val_type get_reg_val_type(const struct jit_ctx *ctx,
122 return (ctx->reg_val_types[index] >> (reg * 3)) & 7;
126 #define emit_instr_long(ctx, func64, func32, ...) \
128 if ((ctx)->target != NULL) { \
129 u32 *p = &(ctx)->target[ctx->idx]; \
135 (ctx)->idx++; \
138 #define emit_instr(ctx, func, ...) \
139 emit_instr_long(ctx, func, func, ##__VA_ARGS__)
141 static unsigned int j_target(struct jit_ctx *ctx, int target_idx)
146 if (!ctx->target)
149 base_va = (unsigned long)ctx->target;
150 target_va = base_va + (ctx->offsets[target_idx] & ~OFFSETS_B_CONV);
159 static u32 b_imm(unsigned int tgt, struct jit_ctx *ctx)
161 if (!ctx->gen_b_offsets)
172 * ctx->idx currently points to the branch instruction
176 return (ctx->offsets[tgt] & ~OFFSETS_B_CONV) -
177 (ctx->idx * 4) - 4;
193 static int ebpf_to_mips_reg(struct jit_ctx *ctx,
214 ctx->flags |= EBPF_SAVE_S0;
217 ctx->flags |= EBPF_SAVE_S1;
220 ctx->flags |= EBPF_SAVE_S2;
223 ctx->flags |= EBPF_SAVE_S3;
228 ctx->flags |= EBPF_SEEN_FP;
270 static int gen_int_prologue(struct jit_ctx *ctx)
276 if (ctx->flags & EBPF_SAVE_RA)
282 if (ctx->flags & EBPF_SAVE_S0)
284 if (ctx->flags & EBPF_SAVE_S1)
286 if (ctx->flags & EBPF_SAVE_S2)
288 if (ctx->flags & EBPF_SAVE_S3)
290 if (ctx->flags & EBPF_SAVE_S4)
294 locals_size = (ctx->flags & EBPF_SEEN_FP) ? MAX_BPF_STACK : 0;
298 ctx->stack_size = stack_adjust;
305 emit_instr(ctx, addiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
307 emit_instr_long(ctx, daddiu, addiu,
314 if (ctx->flags & EBPF_SAVE_RA) {
315 emit_instr_long(ctx, sd, sw,
319 if (ctx->flags & EBPF_SAVE_S0) {
320 emit_instr_long(ctx, sd, sw,
324 if (ctx->flags & EBPF_SAVE_S1) {
325 emit_instr_long(ctx, sd, sw,
329 if (ctx->flags & EBPF_SAVE_S2) {
330 emit_instr_long(ctx, sd, sw,
334 if (ctx->flags & EBPF_SAVE_S3) {
335 emit_instr_long(ctx, sd, sw,
339 if (ctx->flags & EBPF_SAVE_S4) {
340 emit_instr_long(ctx, sd, sw,
345 if ((ctx->flags & EBPF_SEEN_TC) && !(ctx->flags & EBPF_TCC_IN_V1))
346 emit_instr_long(ctx, daddu, addu,
352 static int build_int_epilogue(struct jit_ctx *ctx, int dest_reg)
354 const struct bpf_prog *prog = ctx->skf;
355 int stack_adjust = ctx->stack_size;
362 td = get_reg_val_type(ctx, prog->len, BPF_REG_0);
364 emit_instr(ctx, sll, r0, r0, 0);
367 if (ctx->flags & EBPF_SAVE_RA) {
368 emit_instr_long(ctx, ld, lw,
372 if (ctx->flags & EBPF_SAVE_S0) {
373 emit_instr_long(ctx, ld, lw,
377 if (ctx->flags & EBPF_SAVE_S1) {
378 emit_instr_long(ctx, ld, lw,
382 if (ctx->flags & EBPF_SAVE_S2) {
383 emit_instr_long(ctx, ld, lw,
387 if (ctx->flags & EBPF_SAVE_S3) {
388 emit_instr_long(ctx, ld, lw,
392 if (ctx->flags & EBPF_SAVE_S4) {
393 emit_instr_long(ctx, ld, lw,
397 emit_instr(ctx, jr, dest_reg);
400 emit_instr_long(ctx, daddiu, addiu,
403 emit_instr(ctx, nop);
409 struct jit_ctx *ctx)
412 emit_instr(ctx, addiu, reg, MIPS_R_ZERO, insn->imm);
417 emit_instr(ctx, lui, reg, upper >> 16);
418 emit_instr(ctx, addiu, reg, reg, lower);
422 static int gen_imm_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
426 int dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
464 get_reg_val_type(ctx, idx, insn->dst_reg) == REG_32BIT)
465 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
470 get_reg_val_type(ctx, idx, insn->dst_reg) != REG_32BIT)
471 emit_instr(ctx, sll, dst, dst, 0);
477 emit_instr(ctx, daddiu, dst, MIPS_R_ZERO, insn->imm);
481 emit_instr(ctx, andi, dst, dst, insn->imm);
485 emit_instr(ctx, ori, dst, dst, insn->imm);
489 emit_instr(ctx, xori, dst, dst, insn->imm);
492 emit_instr(ctx, daddiu, dst, dst, insn->imm);
495 emit_instr(ctx, daddiu, dst, dst, -insn->imm);
498 emit_instr(ctx, dsrl_safe, dst, dst, insn->imm & 0x3f);
501 emit_instr(ctx, srl, dst, dst, insn->imm & 0x1f);
504 emit_instr(ctx, dsll_safe, dst, dst, insn->imm & 0x3f);
507 emit_instr(ctx, sll, dst, dst, insn->imm & 0x1f);
510 emit_instr(ctx, dsra_safe, dst, dst, insn->imm & 0x3f);
513 emit_instr(ctx, sra, dst, dst, insn->imm & 0x1f);
516 emit_instr(ctx, addiu, dst, MIPS_R_ZERO, insn->imm);
519 emit_instr(ctx, addiu, dst, dst, insn->imm);
522 emit_instr(ctx, addiu, dst, dst, -insn->imm);
530 gen_imm_to_reg(insn, dst, ctx);
532 gen_imm_to_reg(insn, MIPS_R_AT, ctx);
536 emit_instr(ctx, and, dst, dst, MIPS_R_AT);
540 emit_instr(ctx, or, dst, dst, MIPS_R_AT);
544 emit_instr(ctx, xor, dst, dst, MIPS_R_AT);
547 emit_instr(ctx, daddu, dst, dst, MIPS_R_AT);
550 emit_instr(ctx, dsubu, dst, dst, MIPS_R_AT);
553 emit_instr(ctx, addu, dst, dst, MIPS_R_AT);
556 emit_instr(ctx, subu, dst, dst, MIPS_R_AT);
567 static void emit_const_to_reg(struct jit_ctx *ctx, int dst, u64 value)
570 emit_instr(ctx, daddiu, dst, MIPS_R_ZERO, (int)value);
573 emit_instr(ctx, lui, dst, (s32)(s16)(value >> 16));
574 emit_instr(ctx, ori, dst, dst, (unsigned int)(value & 0xffff));
584 emit_instr(ctx, dsll_safe, dst, dst, needed_shift);
589 emit_instr(ctx, lui, dst, (s32)(s16)part);
592 emit_instr(ctx, ori, dst,
604 static int emit_bpf_tail_call(struct jit_ctx *ctx, int this_idx)
609 ctx->flags |= EBPF_SEEN_TC;
615 emit_instr(ctx, lwu, MIPS_R_T5, off, MIPS_R_A1);
616 emit_instr(ctx, sltu, MIPS_R_AT, MIPS_R_T5, MIPS_R_A2);
617 b_off = b_imm(this_idx + 1, ctx);
618 emit_instr(ctx, bne, MIPS_R_AT, MIPS_R_ZERO, b_off);
624 tcc_reg = (ctx->flags & EBPF_TCC_IN_V1) ? MIPS_R_V1 : MIPS_R_S4;
625 emit_instr(ctx, daddiu, MIPS_R_T5, tcc_reg, -1);
626 b_off = b_imm(this_idx + 1, ctx);
627 emit_instr(ctx, bltz, tcc_reg, b_off);
634 emit_instr(ctx, dsll, MIPS_R_T8, MIPS_R_A2, 3);
635 emit_instr(ctx, daddu, MIPS_R_T8, MIPS_R_T8, MIPS_R_A1);
637 emit_instr(ctx, ld, MIPS_R_AT, off, MIPS_R_T8);
638 b_off = b_imm(this_idx + 1, ctx);
639 emit_instr(ctx, beq, MIPS_R_AT, MIPS_R_ZERO, b_off);
641 emit_instr(ctx, nop);
645 emit_instr(ctx, ld, MIPS_R_T9, off, MIPS_R_AT);
647 emit_instr(ctx, daddu, MIPS_R_V1, MIPS_R_T5, MIPS_R_ZERO);
649 emit_instr(ctx, daddiu, MIPS_R_T9, MIPS_R_T9, 4);
650 return build_int_epilogue(ctx, MIPS_R_T9);
659 static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
692 r = gen_imm_insn(insn, ctx, this_idx);
697 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
700 if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT)
701 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
704 gen_imm_to_reg(insn, MIPS_R_AT, ctx);
706 emit_instr(ctx, dmulu, dst, dst, MIPS_R_AT);
708 emit_instr(ctx, dmultu, MIPS_R_AT, dst);
709 emit_instr(ctx, mflo, dst);
713 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
716 if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT)
717 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
718 emit_instr(ctx, dsubu, dst, MIPS_R_ZERO, dst);
721 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
724 td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
727 emit_instr(ctx, sll, dst, dst, 0);
731 gen_imm_to_reg(insn, MIPS_R_AT, ctx);
733 emit_instr(ctx, mulu, dst, dst, MIPS_R_AT);
735 emit_instr(ctx, multu, dst, MIPS_R_AT);
736 emit_instr(ctx, mflo, dst);
740 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
743 td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
746 emit_instr(ctx, sll, dst, dst, 0);
748 emit_instr(ctx, subu, dst, MIPS_R_ZERO, dst);
754 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
757 td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
760 emit_instr(ctx, sll, dst, dst, 0);
764 emit_instr(ctx, addu, dst, MIPS_R_ZERO, MIPS_R_ZERO);
767 gen_imm_to_reg(insn, MIPS_R_AT, ctx);
770 emit_instr(ctx, divu_r6, dst, dst, MIPS_R_AT);
772 emit_instr(ctx, modu, dst, dst, MIPS_R_AT);
775 emit_instr(ctx, divu, dst, MIPS_R_AT);
777 emit_instr(ctx, mflo, dst);
779 emit_instr(ctx, mfhi, dst);
785 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
788 if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT)
789 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
793 emit_instr(ctx, addu, dst, MIPS_R_ZERO, MIPS_R_ZERO);
796 gen_imm_to_reg(insn, MIPS_R_AT, ctx);
799 emit_instr(ctx, ddivu_r6, dst, dst, MIPS_R_AT);
801 emit_instr(ctx, modu, dst, dst, MIPS_R_AT);
804 emit_instr(ctx, ddivu, dst, MIPS_R_AT);
806 emit_instr(ctx, mflo, dst);
808 emit_instr(ctx, mfhi, dst);
822 src = ebpf_to_mips_reg(ctx, insn, src_reg);
823 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
826 if (get_reg_val_type(ctx, this_idx, insn->dst_reg) == REG_32BIT)
827 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
831 emit_instr(ctx, daddiu, dst, MIPS_R_SP, MAX_BPF_STACK);
834 emit_instr(ctx, daddiu, MIPS_R_AT, MIPS_R_SP, MAX_BPF_STACK);
837 } else if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
844 emit_instr(ctx, daddu, tmp_reg, src, MIPS_R_ZERO);
845 emit_instr(ctx, dinsu, tmp_reg, MIPS_R_ZERO, 32, 32);
851 emit_instr(ctx, daddu, dst, src, MIPS_R_ZERO);
854 emit_instr(ctx, daddu, dst, dst, src);
857 emit_instr(ctx, dsubu, dst, dst, src);
860 emit_instr(ctx, xor, dst, dst, src);
863 emit_instr(ctx, or, dst, dst, src);
866 emit_instr(ctx, and, dst, dst, src);
870 emit_instr(ctx, dmulu, dst, dst, src);
872 emit_instr(ctx, dmultu, dst, src);
873 emit_instr(ctx, mflo, dst);
880 emit_instr(ctx, ddivu_r6,
883 emit_instr(ctx, modu, dst, dst, src);
886 emit_instr(ctx, ddivu, dst, src);
888 emit_instr(ctx, mflo, dst);
890 emit_instr(ctx, mfhi, dst);
893 emit_instr(ctx, dsllv, dst, dst, src);
896 emit_instr(ctx, dsrlv, dst, dst, src);
899 emit_instr(ctx, dsrav, dst, dst, src);
918 src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
919 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
922 td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
925 emit_instr(ctx, sll, dst, dst, 0);
928 ts = get_reg_val_type(ctx, this_idx, insn->src_reg);
937 emit_instr(ctx, sll, tmp_reg, src, 0);
943 emit_instr(ctx, addu, dst, src, MIPS_R_ZERO);
946 emit_instr(ctx, addu, dst, dst, src);
949 emit_instr(ctx, subu, dst, dst, src);
952 emit_instr(ctx, xor, dst, dst, src);
955 emit_instr(ctx, or, dst, dst, src);
958 emit_instr(ctx, and, dst, dst, src);
961 emit_instr(ctx, mul, dst, dst, src);
967 emit_instr(ctx, divu_r6, dst, dst, src);
969 emit_instr(ctx, modu, dst, dst, src);
972 emit_instr(ctx, divu, dst, src);
974 emit_instr(ctx, mflo, dst);
976 emit_instr(ctx, mfhi, dst);
979 emit_instr(ctx, sllv, dst, dst, src);
982 emit_instr(ctx, srlv, dst, dst, src);
985 emit_instr(ctx, srav, dst, dst, src);
994 b_off = b_imm(exit_idx, ctx);
997 emit_instr(ctx, beq, MIPS_R_ZERO, MIPS_R_ZERO, b_off);
998 emit_instr(ctx, nop);
1004 dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
1010 gen_imm_to_reg(insn, MIPS_R_AT, ctx);
1025 src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
1026 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
1029 td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
1030 ts = get_reg_val_type(ctx, this_idx, insn->src_reg);
1032 emit_instr(ctx, sll, MIPS_R_AT, src, 0);
1035 emit_instr(ctx, sll, MIPS_R_AT, dst, 0);
1039 emit_instr(ctx, and, MIPS_R_AT, dst, src);
1044 emit_instr(ctx, dsubu, MIPS_R_AT, dst, src);
1046 b_off = b_imm(exit_idx, ctx);
1050 emit_instr(ctx, blez, MIPS_R_AT, b_off);
1052 emit_instr(ctx, bgtz, MIPS_R_AT, b_off);
1053 emit_instr(ctx, nop);
1056 b_off = b_imm(this_idx + insn->off + 1, ctx);
1060 emit_instr(ctx, bgtz, MIPS_R_AT, b_off);
1062 emit_instr(ctx, blez, MIPS_R_AT, b_off);
1063 emit_instr(ctx, nop);
1066 emit_instr(ctx, slt, MIPS_R_AT, dst, src);
1072 emit_instr(ctx, dsubu, MIPS_R_T8, dst, src);
1073 emit_instr(ctx, sltu, MIPS_R_AT, dst, src);
1076 emit_instr(ctx, seleqz, MIPS_R_T9,
1079 emit_instr(ctx, movz, MIPS_R_T9,
1081 emit_instr(ctx, movn, MIPS_R_T9,
1084 emit_instr(ctx, or, MIPS_R_AT, MIPS_R_T9, MIPS_R_AT);
1089 emit_instr(ctx, sltu, MIPS_R_AT, dst, src);
1104 b_off = b_imm(exit_idx, ctx);
1106 target = j_target(ctx, exit_idx);
1111 if (!(ctx->offsets[this_idx] & OFFSETS_B_CONV)) {
1112 ctx->offsets[this_idx] |= OFFSETS_B_CONV;
1113 ctx->long_b_conversion = 1;
1118 emit_instr(ctx, bne, dst, src, b_off);
1120 emit_instr(ctx, beq, dst, src, b_off);
1121 emit_instr(ctx, nop);
1122 if (ctx->offsets[this_idx] & OFFSETS_B_CONV) {
1123 emit_instr(ctx, j, target);
1124 emit_instr(ctx, nop);
1128 b_off = b_imm(this_idx + insn->off + 1, ctx);
1130 target = j_target(ctx, this_idx + insn->off + 1);
1135 if (!(ctx->offsets[this_idx] & OFFSETS_B_CONV)) {
1136 ctx->offsets[this_idx] |= OFFSETS_B_CONV;
1137 ctx->long_b_conversion = 1;
1142 emit_instr(ctx, beq, dst, src, b_off);
1144 emit_instr(ctx, bne, dst, src, b_off);
1145 emit_instr(ctx, nop);
1146 if (ctx->offsets[this_idx] & OFFSETS_B_CONV) {
1147 emit_instr(ctx, j, target);
1148 emit_instr(ctx, nop);
1156 dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
1162 b_off = b_imm(exit_idx, ctx);
1167 emit_instr(ctx, blez, dst, b_off);
1170 emit_instr(ctx, bltz, dst, b_off);
1173 emit_instr(ctx, bgez, dst, b_off);
1176 emit_instr(ctx, bgtz, dst, b_off);
1179 emit_instr(ctx, nop);
1182 b_off = b_imm(this_idx + insn->off + 1, ctx);
1187 emit_instr(ctx, bgtz, dst, b_off);
1190 emit_instr(ctx, bgez, dst, b_off);
1193 emit_instr(ctx, bltz, dst, b_off);
1196 emit_instr(ctx, blez, dst, b_off);
1199 emit_instr(ctx, nop);
1215 emit_instr(ctx, slti, MIPS_R_AT, dst, (int)t64s);
1220 emit_const_to_reg(ctx, MIPS_R_AT, (u64)t64s);
1221 emit_instr(ctx, slt, MIPS_R_AT, dst, MIPS_R_AT);
1231 dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
1247 emit_const_to_reg(ctx, MIPS_R_AT, (u64)t64s);
1248 emit_instr(ctx, sltu, MIPS_R_AT, dst, MIPS_R_AT);
1254 dst = ebpf_to_mips_reg(ctx, insn, dst_reg_fp_ok);
1258 if (ctx->use_bbit_insns && hweight32((u32)insn->imm) == 1) {
1260 b_off = b_imm(exit_idx, ctx);
1263 emit_instr(ctx, bbit0, dst, ffs((u32)insn->imm) - 1, b_off);
1264 emit_instr(ctx, nop);
1267 b_off = b_imm(this_idx + insn->off + 1, ctx);
1270 emit_instr(ctx, bbit1, dst, ffs((u32)insn->imm) - 1, b_off);
1271 emit_instr(ctx, nop);
1275 emit_const_to_reg(ctx, MIPS_R_AT, t64);
1276 emit_instr(ctx, and, MIPS_R_AT, dst, MIPS_R_AT);
1287 b_off = b_imm(this_idx + insn->off + 1, ctx);
1289 target = j_target(ctx, this_idx + insn->off + 1);
1292 emit_instr(ctx, j, target);
1294 emit_instr(ctx, b, b_off);
1296 emit_instr(ctx, nop);
1301 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
1305 emit_const_to_reg(ctx, dst, t64);
1309 ctx->flags |= EBPF_SAVE_RA;
1311 emit_const_to_reg(ctx, MIPS_R_T9, (u64)t64s);
1312 emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
1314 emit_instr(ctx, nop);
1318 if (emit_bpf_tail_call(ctx, this_idx))
1324 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
1327 td = get_reg_val_type(ctx, this_idx, insn->dst_reg);
1329 emit_instr(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32);
1333 emit_instr(ctx, sll, dst, dst, 0);
1343 emit_instr(ctx, wsbh, dst, dst);
1344 emit_instr(ctx, andi, dst, dst, 0xffff);
1347 emit_instr(ctx, wsbh, dst, dst);
1348 emit_instr(ctx, rotr, dst, dst, 16);
1352 emit_instr(ctx, dsbh, dst, dst);
1353 emit_instr(ctx, dshd, dst, dst);
1366 ctx->flags |= EBPF_SEEN_FP;
1370 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
1375 gen_imm_to_reg(insn, MIPS_R_AT, ctx);
1378 emit_instr(ctx, sb, MIPS_R_AT, mem_off, dst);
1381 emit_instr(ctx, sh, MIPS_R_AT, mem_off, dst);
1384 emit_instr(ctx, sw, MIPS_R_AT, mem_off, dst);
1387 emit_instr(ctx, sd, MIPS_R_AT, mem_off, dst);
1397 ctx->flags |= EBPF_SEEN_FP;
1401 src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
1406 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
1411 emit_instr(ctx, lbu, dst, mem_off, src);
1414 emit_instr(ctx, lhu, dst, mem_off, src);
1417 emit_instr(ctx, lw, dst, mem_off, src);
1420 emit_instr(ctx, ld, dst, mem_off, src);
1432 ctx->flags |= EBPF_SEEN_FP;
1436 dst = ebpf_to_mips_reg(ctx, insn, dst_reg);
1441 src = ebpf_to_mips_reg(ctx, insn, src_reg_no_fp);
1451 emit_instr(ctx, daddiu, MIPS_R_T6,
1458 if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
1459 emit_instr(ctx, sll, MIPS_R_AT, src, 0);
1462 emit_instr(ctx, ll, MIPS_R_T8, mem_off, dst);
1463 emit_instr(ctx, addu, MIPS_R_T8, MIPS_R_T8, src);
1464 emit_instr(ctx, sc, MIPS_R_T8, mem_off, dst);
1469 emit_instr(ctx, beq, MIPS_R_T8, MIPS_R_ZERO, -4 * 4);
1470 emit_instr(ctx, nop);
1473 if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
1474 emit_instr(ctx, daddu, MIPS_R_AT, src, MIPS_R_ZERO);
1475 emit_instr(ctx, dinsu, MIPS_R_AT, MIPS_R_ZERO, 32, 32);
1478 emit_instr(ctx, lld, MIPS_R_T8, mem_off, dst);
1479 emit_instr(ctx, daddu, MIPS_R_T8, MIPS_R_T8, src);
1480 emit_instr(ctx, scd, MIPS_R_T8, mem_off, dst);
1481 emit_instr(ctx, beq, MIPS_R_T8, MIPS_R_ZERO, -4 * 4);
1482 emit_instr(ctx, nop);
1488 emit_instr(ctx, sb, src, mem_off, dst);
1491 emit_instr(ctx, sh, src, mem_off, dst);
1494 emit_instr(ctx, sw, src, mem_off, dst);
1497 if (get_reg_val_type(ctx, this_idx, insn->src_reg) == REG_32BIT) {
1498 emit_instr(ctx, daddu, MIPS_R_AT, src, MIPS_R_ZERO);
1499 emit_instr(ctx, dinsu, MIPS_R_AT, MIPS_R_ZERO, 32, 32);
1502 emit_instr(ctx, sd, src, mem_off, dst);
1521 static int build_int_body(struct jit_ctx *ctx)
1523 const struct bpf_prog *prog = ctx->skf;
1529 if ((ctx->reg_val_types[i] & RVT_VISITED_MASK) == 0) {
1535 if (ctx->target == NULL)
1536 ctx->offsets[i] = (ctx->offsets[i] & OFFSETS_B_CONV) | (ctx->idx * 4);
1538 r = build_one_insn(insn, ctx, i, prog->len);
1544 if (ctx->target == NULL)
1545 ctx->offsets[i] = ctx->idx * 4;
1552 if (ctx->target == NULL)
1556 ctx->offsets[i] = ctx->idx * 4;
1562 static int reg_val_propagate_range(struct jit_ctx *ctx, u64 initial_rvt,
1565 const struct bpf_prog *prog = ctx->skf;
1568 u64 *rvt = ctx->reg_val_types;
1739 static int reg_val_propagate(struct jit_ctx *ctx)
1741 const struct bpf_prog *prog = ctx->skf;
1760 reg_val_propagate_range(ctx, exit_rvt, 0, false);
1769 u64 rvt = ctx->reg_val_types[i];
1775 reg_val_propagate_range(ctx, rvt & ~RVT_VISITED_MASK, i, true);
1778 reg_val_propagate_range(ctx, rvt & ~RVT_VISITED_MASK, i, false);
1806 struct jit_ctx ctx;
1824 memset(&ctx, 0, sizeof(ctx));
1832 ctx.use_bbit_insns = 1;
1835 ctx.use_bbit_insns = 0;
1839 ctx.offsets = kcalloc(prog->len + 1, sizeof(*ctx.offsets), GFP_KERNEL);
1840 if (ctx.offsets == NULL)
1843 ctx.reg_val_types = kcalloc(prog->len + 1, sizeof(*ctx.reg_val_types), GFP_KERNEL);
1844 if (ctx.reg_val_types == NULL)
1847 ctx.skf = prog;
1849 if (reg_val_propagate(&ctx))
1856 if (build_int_body(&ctx))
1863 if (ctx.flags & EBPF_SEEN_TC) {
1864 if (ctx.flags & EBPF_SAVE_RA)
1865 ctx.flags |= EBPF_SAVE_S4;
1867 ctx.flags |= EBPF_TCC_IN_V1;
1878 ctx.idx = 0;
1879 ctx.gen_b_offsets = 1;
1880 ctx.long_b_conversion = 0;
1881 if (gen_int_prologue(&ctx))
1883 if (build_int_body(&ctx))
1885 if (build_int_epilogue(&ctx, MIPS_R_RA))
1887 } while (ctx.long_b_conversion);
1889 image_size = 4 * ctx.idx;
1896 ctx.target = (u32 *)image_ptr;
1899 ctx.idx = 0;
1900 if (gen_int_prologue(&ctx))
1902 if (build_int_body(&ctx))
1904 if (build_int_epilogue(&ctx, MIPS_R_RA))
1908 flush_icache_range((unsigned long)ctx.target,
1909 (unsigned long)&ctx.target[ctx.idx]);
1913 bpf_jit_dump(prog->len, image_size, 2, ctx.target);
1916 prog->bpf_func = (void *)ctx.target;
1923 kfree(ctx.offsets);
1924 kfree(ctx.reg_val_types);