Lines Matching refs:ctx
118 static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx);
121 #define emit_instr(ctx, func, ...) \
123 if ((ctx)->target != NULL) { \
124 u32 *p = &(ctx)->target[ctx->idx]; \
127 (ctx)->idx++; \
134 #define emit_long_instr(ctx, func, ...) \
136 if ((ctx)->target != NULL) { \
137 u32 *p = &(ctx)->target[ctx->idx]; \
140 (ctx)->idx++; \
150 unsigned int src2, struct jit_ctx *ctx)
152 emit_instr(ctx, addu, dst, src1, src2);
155 static inline void emit_nop(struct jit_ctx *ctx)
157 emit_instr(ctx, nop);
161 static inline void emit_load_imm(unsigned int dst, u32 imm, struct jit_ctx *ctx)
163 if (ctx->target != NULL) {
166 u32 *p = &ctx->target[ctx->idx];
168 p = &ctx->target[ctx->idx + 1];
171 u32 *p = &ctx->target[ctx->idx];
175 ctx->idx++;
178 ctx->idx++;
182 unsigned int src2, struct jit_ctx *ctx)
184 emit_instr(ctx, or, dst, src1, src2);
188 struct jit_ctx *ctx)
191 emit_load_imm(r_tmp, imm, ctx);
192 emit_or(dst, src, r_tmp, ctx);
194 emit_instr(ctx, ori, dst, src, imm);
199 int imm, struct jit_ctx *ctx)
205 emit_instr(ctx, daddiu, dst, src, imm);
209 u32 imm, struct jit_ctx *ctx)
212 emit_load_imm(r_tmp, imm, ctx);
213 emit_addu(dst, r_tmp, src, ctx);
215 emit_instr(ctx, addiu, dst, src, imm);
220 unsigned int src2, struct jit_ctx *ctx)
222 emit_instr(ctx, and, dst, src1, src2);
226 u32 imm, struct jit_ctx *ctx)
230 emit_load_imm(r_tmp, imm, ctx);
231 emit_and(dst, src, r_tmp, ctx);
233 emit_instr(ctx, andi, dst, src, imm);
238 unsigned int src2, struct jit_ctx *ctx)
240 emit_instr(ctx, xor, dst, src1, src2);
243 static inline void emit_xori(ptr dst, ptr src, u32 imm, struct jit_ctx *ctx)
247 emit_load_imm(r_tmp, imm, ctx);
248 emit_xor(dst, src, r_tmp, ctx);
250 emit_instr(ctx, xori, dst, src, imm);
254 static inline void emit_stack_offset(int offset, struct jit_ctx *ctx)
256 emit_long_instr(ctx, ADDIU, r_sp, r_sp, offset);
260 unsigned int src2, struct jit_ctx *ctx)
262 emit_instr(ctx, subu, dst, src1, src2);
265 static inline void emit_neg(unsigned int reg, struct jit_ctx *ctx)
267 emit_subu(reg, r_zero, reg, ctx);
271 unsigned int sa, struct jit_ctx *ctx)
273 emit_instr(ctx, sllv, dst, src, sa);
277 unsigned int sa, struct jit_ctx *ctx)
282 emit_jit_reg_move(dst, r_zero, ctx);
284 emit_instr(ctx, sll, dst, src, sa);
288 unsigned int sa, struct jit_ctx *ctx)
290 emit_instr(ctx, srlv, dst, src, sa);
294 unsigned int sa, struct jit_ctx *ctx)
299 emit_jit_reg_move(dst, r_zero, ctx);
301 emit_instr(ctx, srl, dst, src, sa);
305 unsigned int src2, struct jit_ctx *ctx)
307 emit_instr(ctx, slt, dst, src1, src2);
311 unsigned int src2, struct jit_ctx *ctx)
313 emit_instr(ctx, sltu, dst, src1, src2);
317 unsigned int imm, struct jit_ctx *ctx)
321 emit_load_imm(r_tmp, imm, ctx);
322 emit_sltu(dst, src, r_tmp, ctx);
324 emit_instr(ctx, sltiu, dst, src, imm);
332 struct jit_ctx *ctx)
334 emit_long_instr(ctx, SW, reg, offset, base);
338 struct jit_ctx *ctx)
340 emit_instr(ctx, sw, reg, offset, base);
345 struct jit_ctx *ctx)
347 emit_long_instr(ctx, LW, reg, offset, base);
351 unsigned int offset, struct jit_ctx *ctx)
353 emit_instr(ctx, lw, reg, offset, base);
357 unsigned int offset, struct jit_ctx *ctx)
359 emit_instr(ctx, lb, reg, offset, base);
363 unsigned int offset, struct jit_ctx *ctx)
365 emit_instr(ctx, lh, reg, offset, base);
369 unsigned int offset, struct jit_ctx *ctx)
371 emit_instr(ctx, lhu, reg, offset, base);
375 unsigned int src2, struct jit_ctx *ctx)
377 emit_instr(ctx, mul, dst, src1, src2);
381 struct jit_ctx *ctx)
383 if (ctx->target != NULL) {
384 u32 *p = &ctx->target[ctx->idx];
386 p = &ctx->target[ctx->idx + 1];
389 ctx->idx += 2; /* 2 insts */
393 struct jit_ctx *ctx)
395 if (ctx->target != NULL) {
396 u32 *p = &ctx->target[ctx->idx];
398 p = &ctx->target[ctx->idx + 1];
401 ctx->idx += 2; /* 2 insts */
405 unsigned int sa, struct jit_ctx *ctx)
407 emit_instr(ctx, dsll, dst, src, sa);
411 unsigned int sa, struct jit_ctx *ctx)
413 emit_instr(ctx, dsrl32, dst, src, sa);
417 struct jit_ctx *ctx)
419 emit_instr(ctx, wsbh, dst, src);
424 int imm, struct jit_ctx *ctx)
427 emit_long_instr(ctx, LW, dst, imm, src);
432 struct jit_ctx *ctx)
436 emit_load_imm(r_tmp, (u64)imm >> 32, ctx);
437 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */
438 emit_ori(r_tmp, r_tmp_imm, (imm >> 16) & 0xffff, ctx);
439 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */
440 emit_ori(reg, r_tmp_imm, imm & 0xffff, ctx);
442 emit_load_imm(reg, imm, ctx);
447 static inline void emit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx)
449 emit_long_instr(ctx, ADDU, dst, src, r_zero);
453 static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx)
455 emit_addu(dst, src, r_zero, ctx);
459 static inline u32 b_imm(unsigned int tgt, struct jit_ctx *ctx)
461 if (ctx->target == NULL)
473 * ctx->idx currently points to the branch instruction
477 return ctx->offsets[tgt] -
478 (ctx->idx * 4 - ctx->prologue_bytes) - 4;
482 unsigned int imm, struct jit_ctx *ctx)
484 if (ctx->target != NULL) {
485 u32 *p = &ctx->target[ctx->idx];
502 ctx->idx++;
505 static inline void emit_b(unsigned int imm, struct jit_ctx *ctx)
507 emit_bcond(MIPS_COND_ALL, r_zero, r_zero, imm, ctx);
511 struct jit_ctx *ctx)
513 emit_instr(ctx, jalr, link, reg);
516 static inline void emit_jr(unsigned int reg, struct jit_ctx *ctx)
518 emit_instr(ctx, jr, reg);
529 static void save_bpf_jit_regs(struct jit_ctx *ctx, unsigned offset)
536 emit_stack_offset(-align_sp(offset), ctx);
538 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT;
543 ctx);
551 if (ctx->flags & SEEN_CALL) {
552 emit_store_stack_reg(r_ra, r_sp, real_off, ctx);
557 if (ctx->flags & SEEN_MEM) {
560 emit_long_instr(ctx, ADDIU, r_M, r_sp, real_off);
564 static void restore_bpf_jit_regs(struct jit_ctx *ctx,
570 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT;
576 ctx);
584 if (ctx->flags & SEEN_CALL)
585 emit_load_stack_reg(r_ra, r_sp, real_off, ctx);
589 emit_stack_offset(align_sp(offset), ctx);
592 static unsigned int get_stack_depth(struct jit_ctx *ctx)
598 sp_off += hweight32(ctx->flags >> SEEN_SREG_SFT) * SZREG;
600 if (ctx->flags & SEEN_MEM)
603 if (ctx->flags & SEEN_CALL)
609 static void build_prologue(struct jit_ctx *ctx)
614 sp_off = get_stack_depth(ctx);
615 save_bpf_jit_regs(ctx, sp_off);
617 if (ctx->flags & SEEN_SKB)
618 emit_reg_move(r_skb, MIPS_R_A0, ctx);
620 if (ctx->flags & SEEN_SKB_DATA) {
623 ctx);
625 ctx);
628 offsetof(struct sk_buff, data), ctx);
630 emit_subu(r_skb_hl, r_skb_len, r_tmp, ctx);
633 if (ctx->flags & SEEN_X)
634 emit_jit_reg_move(r_X, r_zero, ctx);
642 if (bpf_needs_clear_a(&ctx->skf->insns[0]) &&
643 (ctx->flags & SEEN_A))
644 emit_jit_reg_move(r_A, r_zero, ctx);
647 static void build_epilogue(struct jit_ctx *ctx)
653 sp_off = get_stack_depth(ctx);
654 restore_bpf_jit_regs(ctx, sp_off);
657 emit_jr(r_ra, ctx);
658 emit_nop(ctx);
670 static int build_body(struct jit_ctx *ctx)
672 const struct bpf_prog *prog = ctx->skf;
687 if (ctx->target == NULL)
688 ctx->offsets[i] = ctx->idx * 4;
693 ctx->flags |= SEEN_A;
694 emit_load_imm(r_A, k, ctx);
699 ctx->flags |= SEEN_SKB | SEEN_A;
701 emit_load(r_A, r_skb, off, ctx);
705 ctx->flags |= SEEN_MEM | SEEN_A;
706 emit_load(r_A, r_M, SCRATCH_OFF(k), ctx);
720 emit_load_imm(r_off, k, ctx);
722 ctx->flags |= SEEN_CALL | SEEN_OFF |
725 emit_load_func(r_s0, (ptr)sk_load_func, ctx);
726 emit_reg_move(MIPS_R_A0, r_skb, ctx);
727 emit_jalr(MIPS_R_RA, r_s0, ctx);
729 emit_reg_move(MIPS_R_A1, r_off, ctx);
731 emit_bcond(MIPS_COND_EQ, r_ret, 0, b_imm(i + 1, ctx),
732 ctx);
734 emit_reg_move(r_ret, r_zero, ctx);
736 b_off = b_imm(prog->len, ctx);
739 emit_b(b_off, ctx);
740 emit_nop(ctx);
754 ctx->flags |= SEEN_OFF | SEEN_X;
755 emit_addiu(r_off, r_X, k, ctx);
759 ctx->flags |= SEEN_X;
760 emit_load_imm(r_X, k, ctx);
764 ctx->flags |= SEEN_X | SEEN_MEM;
765 emit_load(r_X, r_M, SCRATCH_OFF(k), ctx);
769 ctx->flags |= SEEN_X | SEEN_SKB;
771 emit_load(r_X, r_skb, off, ctx);
775 ctx->flags |= SEEN_X | SEEN_CALL | SEEN_SKB;
777 emit_load_func(r_s0, (ptr)sk_load_byte, ctx);
782 emit_load_imm(MIPS_R_A1, k, ctx);
783 emit_jalr(MIPS_R_RA, r_s0, ctx);
784 emit_reg_move(MIPS_R_A0, r_skb, ctx); /* delay slot */
786 b_off = b_imm(prog->len, ctx);
789 emit_bcond(MIPS_COND_NE, r_ret, 0, b_off, ctx);
790 emit_reg_move(r_ret, r_zero, ctx);
793 emit_andi(r_X, r_A, 0xf, ctx);
795 emit_b(b_imm(i + 1, ctx), ctx);
796 emit_sll(r_X, r_X, 2, ctx); /* delay slot */
800 ctx->flags |= SEEN_MEM | SEEN_A;
801 emit_store(r_A, r_M, SCRATCH_OFF(k), ctx);
805 ctx->flags |= SEEN_MEM | SEEN_X;
806 emit_store(r_X, r_M, SCRATCH_OFF(k), ctx);
810 ctx->flags |= SEEN_A;
811 emit_addiu(r_A, r_A, k, ctx);
815 ctx->flags |= SEEN_A | SEEN_X;
816 emit_addu(r_A, r_A, r_X, ctx);
820 ctx->flags |= SEEN_A;
821 emit_addiu(r_A, r_A, -k, ctx);
825 ctx->flags |= SEEN_A | SEEN_X;
826 emit_subu(r_A, r_A, r_X, ctx);
831 ctx->flags |= SEEN_A;
832 emit_load_imm(r_s0, k, ctx);
833 emit_mul(r_A, r_A, r_s0, ctx);
837 ctx->flags |= SEEN_A | SEEN_X;
838 emit_mul(r_A, r_A, r_X, ctx);
845 ctx->flags |= SEEN_A;
846 emit_srl(r_A, r_A, k, ctx);
849 ctx->flags |= SEEN_A;
850 emit_load_imm(r_s0, k, ctx);
851 emit_div(r_A, r_s0, ctx);
856 ctx->flags |= SEEN_A;
857 emit_jit_reg_move(r_A, r_zero, ctx);
859 ctx->flags |= SEEN_A;
860 emit_load_imm(r_s0, k, ctx);
861 emit_mod(r_A, r_s0, ctx);
866 ctx->flags |= SEEN_X | SEEN_A;
868 b_off = b_imm(prog->len, ctx);
871 emit_bcond(MIPS_COND_EQ, r_X, r_zero, b_off, ctx);
872 emit_load_imm(r_ret, 0, ctx); /* delay slot */
873 emit_div(r_A, r_X, ctx);
877 ctx->flags |= SEEN_X | SEEN_A;
879 b_off = b_imm(prog->len, ctx);
882 emit_bcond(MIPS_COND_EQ, r_X, r_zero, b_off, ctx);
883 emit_load_imm(r_ret, 0, ctx); /* delay slot */
884 emit_mod(r_A, r_X, ctx);
888 ctx->flags |= SEEN_A;
889 emit_ori(r_A, r_A, k, ctx);
893 ctx->flags |= SEEN_A;
894 emit_ori(r_A, r_A, r_X, ctx);
898 ctx->flags |= SEEN_A;
899 emit_xori(r_A, r_A, k, ctx);
904 ctx->flags |= SEEN_A;
905 emit_xor(r_A, r_A, r_X, ctx);
909 ctx->flags |= SEEN_A;
910 emit_andi(r_A, r_A, k, ctx);
914 ctx->flags |= SEEN_A | SEEN_X;
915 emit_and(r_A, r_A, r_X, ctx);
919 ctx->flags |= SEEN_A;
920 emit_sll(r_A, r_A, k, ctx);
924 ctx->flags |= SEEN_A | SEEN_X;
925 emit_sllv(r_A, r_A, r_X, ctx);
929 ctx->flags |= SEEN_A;
930 emit_srl(r_A, r_A, k, ctx);
933 ctx->flags |= SEEN_A | SEEN_X;
934 emit_srlv(r_A, r_A, r_X, ctx);
938 ctx->flags |= SEEN_A;
939 emit_neg(r_A, ctx);
943 b_off = b_imm(i + k + 1, ctx);
946 emit_b(b_off, ctx);
947 emit_nop(ctx);
954 ctx->flags |= SEEN_X;
963 ctx->flags |= SEEN_X;
972 ctx->flags |= SEEN_X;
980 ctx->flags |= SEEN_A;
981 emit_sltiu(r_s0, r_A, k, ctx);
983 ctx->flags |= SEEN_A |
985 emit_sltu(r_s0, r_A, r_X, ctx);
988 b_off = b_imm(i + inst->jf + 1, ctx);
990 ctx);
991 emit_nop(ctx);
995 ctx->flags |= SEEN_A | SEEN_X;
997 emit_load_imm(r_s0, k, ctx);
1000 ctx);
1001 b_off = b_imm(i + inst->jf + 1, ctx);
1003 b_off, ctx);
1004 emit_nop(ctx);
1006 b_off = b_imm(i + inst->jt + 1, ctx);
1007 emit_b(b_off, ctx);
1008 emit_nop(ctx);
1011 b_off = b_imm(i + inst->jt + 1, ctx);
1012 emit_b(b_off, ctx);
1013 emit_nop(ctx);
1018 ctx->flags |= SEEN_A;
1019 emit_load_imm(r_s0, k, ctx);
1021 b_off = b_imm(i + inst->jt + 1, ctx);
1023 b_off, ctx);
1024 emit_nop(ctx);
1027 ctx);
1029 b_off, ctx);
1030 emit_nop(ctx);
1033 ctx->flags |= SEEN_A | SEEN_X;
1035 ctx);
1037 b_off, ctx);
1038 emit_nop(ctx);
1040 b_off = b_imm(i + inst->jf + 1, ctx);
1042 b_off, ctx);
1043 emit_nop(ctx);
1048 ctx->flags |= SEEN_A;
1050 emit_load_imm(r_s1, k, ctx);
1051 emit_and(r_s0, r_A, r_s1, ctx);
1053 b_off = b_imm(i + inst->jt + 1, ctx);
1054 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx);
1055 emit_nop(ctx);
1057 b_off = b_imm(i + inst->jf + 1, ctx);
1058 emit_b(b_off, ctx);
1059 emit_nop(ctx);
1062 ctx->flags |= SEEN_X | SEEN_A;
1064 emit_and(r_s0, r_A, r_X, ctx);
1066 b_off = b_imm(i + inst->jt + 1, ctx);
1067 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx);
1068 emit_nop(ctx);
1070 b_off = b_imm(i + inst->jf + 1, ctx);
1071 emit_b(b_off, ctx);
1072 emit_nop(ctx);
1075 ctx->flags |= SEEN_A;
1081 b_off = b_imm(prog->len, ctx);
1084 emit_b(b_off, ctx);
1086 emit_reg_move(r_ret, r_A, ctx); /* delay slot */
1093 emit_load_imm(r_ret, k, ctx);
1099 b_off = b_imm(prog->len, ctx);
1102 emit_b(b_off, ctx);
1103 emit_nop(ctx);
1108 ctx->flags |= SEEN_X | SEEN_A;
1109 emit_jit_reg_move(r_X, r_A, ctx);
1113 ctx->flags |= SEEN_A | SEEN_X;
1114 emit_jit_reg_move(r_A, r_X, ctx);
1119 ctx->flags |= SEEN_SKB | SEEN_OFF | SEEN_A;
1123 emit_half_load(r_A, r_skb, off, ctx);
1128 emit_wsbh(r_A, r_A, ctx);
1131 emit_andi(r_tmp_imm, r_A, 0xff, ctx);
1133 emit_sll(r_tmp, r_tmp_imm, 8, ctx);
1135 emit_srl(r_tmp_imm, r_A, 8, ctx);
1136 emit_andi(r_tmp_imm, r_tmp_imm, 0xff, ctx);
1138 emit_or(r_A, r_tmp, r_tmp_imm, ctx);
1143 ctx->flags |= SEEN_A | SEEN_OFF;
1149 emit_load(r_A, 28, off, ctx);
1155 ctx->flags |= SEEN_SKB | SEEN_A;
1158 emit_load_ptr(r_s0, r_skb, off, ctx);
1160 b_off = b_imm(prog->len, ctx);
1163 emit_bcond(MIPS_COND_EQ, r_s0, r_zero, b_off, ctx);
1164 emit_reg_move(r_ret, r_zero, ctx);
1168 emit_load(r_A, r_s0, off, ctx);
1172 emit_half_load_unsigned(r_A, r_s0, off, ctx);
1176 ctx->flags |= SEEN_SKB | SEEN_A;
1179 emit_load(r_A, r_skb, off, ctx);
1182 ctx->flags |= SEEN_SKB | SEEN_A;
1185 emit_load(r_A, r_skb, off, ctx);
1188 ctx->flags |= SEEN_SKB | SEEN_A;
1192 emit_half_load_unsigned(r_A, r_skb, off, ctx);
1195 ctx->flags |= SEEN_SKB | SEEN_A;
1196 emit_load_byte(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET(), ctx);
1198 emit_srl(r_A, r_A, PKT_VLAN_PRESENT_BIT, ctx);
1200 emit_andi(r_A, r_A, 1, ctx);
1203 ctx->flags |= SEEN_SKB;
1205 emit_load_byte(r_tmp, r_skb, PKT_TYPE_OFFSET(), ctx);
1207 emit_andi(r_A, r_tmp, PKT_TYPE_MAX, ctx);
1210 emit_srl(r_A, r_A, 5, ctx);
1214 ctx->flags |= SEEN_SKB | SEEN_A;
1220 emit_half_load_unsigned(r_A, r_skb, off, ctx);
1230 if (ctx->target == NULL)
1231 ctx->offsets[i] = ctx->idx * 4;
1238 struct jit_ctx ctx;
1244 memset(&ctx, 0, sizeof(ctx));
1246 ctx.offsets = kcalloc(fp->len + 1, sizeof(*ctx.offsets), GFP_KERNEL);
1247 if (ctx.offsets == NULL)
1250 ctx.skf = fp;
1252 if (build_body(&ctx))
1255 tmp_idx = ctx.idx;
1256 build_prologue(&ctx);
1257 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4;
1258 /* just to complete the ctx.idx count */
1259 build_epilogue(&ctx);
1261 alloc_size = 4 * ctx.idx;
1262 ctx.target = module_alloc(alloc_size);
1263 if (ctx.target == NULL)
1267 memset(ctx.target, 0, alloc_size);
1269 ctx.idx = 0;
1272 build_prologue(&ctx);
1273 if (build_body(&ctx)) {
1274 module_memfree(ctx.target);
1277 build_epilogue(&ctx);
1280 flush_icache_range((ptr)ctx.target, (ptr)(ctx.target + ctx.idx));
1284 bpf_jit_dump(fp->len, alloc_size, 2, ctx.target);
1286 fp->bpf_func = (void *)ctx.target;
1290 kfree(ctx.offsets);