1/* 2 * Just-In-Time compiler for BPF filters on MIPS 3 * 4 * Copyright (c) 2014 Imagination Technologies Ltd. 5 * Author: Markos Chandras <markos.chandras@imgtec.com> 6 * 7 * This program is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License as published by the 9 * Free Software Foundation; version 2 of the License. 10 */ 11 12#include <linux/bitops.h> 13#include <linux/compiler.h> 14#include <linux/errno.h> 15#include <linux/filter.h> 16#include <linux/if_vlan.h> 17#include <linux/moduleloader.h> 18#include <linux/netdevice.h> 19#include <linux/string.h> 20#include <linux/slab.h> 21#include <linux/types.h> 22#include <asm/asm.h> 23#include <asm/bitops.h> 24#include <asm/cacheflush.h> 25#include <asm/cpu-features.h> 26#include <asm/uasm.h> 27 28#include "bpf_jit.h" 29 30/* ABI 31 * r_skb_hl SKB header length 32 * r_data SKB data pointer 33 * r_off Offset 34 * r_A BPF register A 35 * r_X BPF register X 36 * r_skb *skb 37 * r_M *scratch memory 38 * r_skb_len SKB length 39 * 40 * On entry (*bpf_func)(*skb, *filter) 41 * a0 = MIPS_R_A0 = skb; 42 * a1 = MIPS_R_A1 = filter; 43 * 44 * Stack 45 * ... 46 * M[15] 47 * M[14] 48 * M[13] 49 * ... 50 * M[0] <-- r_M 51 * saved reg k-1 52 * saved reg k-2 53 * ... 54 * saved reg 0 <-- r_sp 55 * <no argument area> 56 * 57 * Packet layout 58 * 59 * <--------------------- len ------------------------> 60 * <--skb-len(r_skb_hl)-->< ----- skb->data_len ------> 61 * ---------------------------------------------------- 62 * | skb->data | 63 * ---------------------------------------------------- 64 */ 65 66#define ptr typeof(unsigned long) 67 68#define SCRATCH_OFF(k) (4 * (k)) 69 70/* JIT flags */ 71#define SEEN_CALL (1 << BPF_MEMWORDS) 72#define SEEN_SREG_SFT (BPF_MEMWORDS + 1) 73#define SEEN_SREG_BASE (1 << SEEN_SREG_SFT) 74#define SEEN_SREG(x) (SEEN_SREG_BASE << (x)) 75#define SEEN_OFF SEEN_SREG(2) 76#define SEEN_A SEEN_SREG(3) 77#define SEEN_X SEEN_SREG(4) 78#define SEEN_SKB SEEN_SREG(5) 79#define SEEN_MEM SEEN_SREG(6) 80/* SEEN_SK_DATA also implies skb_hl an skb_len */ 81#define SEEN_SKB_DATA (SEEN_SREG(7) | SEEN_SREG(1) | SEEN_SREG(0)) 82 83/* Arguments used by JIT */ 84#define ARGS_USED_BY_JIT 2 /* only applicable to 64-bit */ 85 86#define SBIT(x) (1 << (x)) /* Signed version of BIT() */ 87 88/** 89 * struct jit_ctx - JIT context 90 * @skf: The sk_filter 91 * @prologue_bytes: Number of bytes for prologue 92 * @idx: Instruction index 93 * @flags: JIT flags 94 * @offsets: Instruction offsets 95 * @target: Memory location for the compiled filter 96 */ 97struct jit_ctx { 98 const struct bpf_prog *skf; 99 unsigned int prologue_bytes; 100 u32 idx; 101 u32 flags; 102 u32 *offsets; 103 u32 *target; 104}; 105 106 107static inline int optimize_div(u32 *k) 108{ 109 /* power of 2 divides can be implemented with right shift */ 110 if (!(*k & (*k-1))) { 111 *k = ilog2(*k); 112 return 1; 113 } 114 115 return 0; 116} 117 118static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx); 119 120/* Simply emit the instruction if the JIT memory space has been allocated */ 121#define emit_instr(ctx, func, ...) \ 122do { \ 123 if ((ctx)->target != NULL) { \ 124 u32 *p = &(ctx)->target[ctx->idx]; \ 125 uasm_i_##func(&p, ##__VA_ARGS__); \ 126 } \ 127 (ctx)->idx++; \ 128} while (0) 129 130/* 131 * Similar to emit_instr but it must be used when we need to emit 132 * 32-bit or 64-bit instructions 133 */ 134#define emit_long_instr(ctx, func, ...) \ 135do { \ 136 if ((ctx)->target != NULL) { \ 137 u32 *p = &(ctx)->target[ctx->idx]; \ 138 UASM_i_##func(&p, ##__VA_ARGS__); \ 139 } \ 140 (ctx)->idx++; \ 141} while (0) 142 143/* Determine if immediate is within the 16-bit signed range */ 144static inline bool is_range16(s32 imm) 145{ 146 return !(imm >= SBIT(15) || imm < -SBIT(15)); 147} 148 149static inline void emit_addu(unsigned int dst, unsigned int src1, 150 unsigned int src2, struct jit_ctx *ctx) 151{ 152 emit_instr(ctx, addu, dst, src1, src2); 153} 154 155static inline void emit_nop(struct jit_ctx *ctx) 156{ 157 emit_instr(ctx, nop); 158} 159 160/* Load a u32 immediate to a register */ 161static inline void emit_load_imm(unsigned int dst, u32 imm, struct jit_ctx *ctx) 162{ 163 if (ctx->target != NULL) { 164 /* addiu can only handle s16 */ 165 if (!is_range16(imm)) { 166 u32 *p = &ctx->target[ctx->idx]; 167 uasm_i_lui(&p, r_tmp_imm, (s32)imm >> 16); 168 p = &ctx->target[ctx->idx + 1]; 169 uasm_i_ori(&p, dst, r_tmp_imm, imm & 0xffff); 170 } else { 171 u32 *p = &ctx->target[ctx->idx]; 172 uasm_i_addiu(&p, dst, r_zero, imm); 173 } 174 } 175 ctx->idx++; 176 177 if (!is_range16(imm)) 178 ctx->idx++; 179} 180 181static inline void emit_or(unsigned int dst, unsigned int src1, 182 unsigned int src2, struct jit_ctx *ctx) 183{ 184 emit_instr(ctx, or, dst, src1, src2); 185} 186 187static inline void emit_ori(unsigned int dst, unsigned src, u32 imm, 188 struct jit_ctx *ctx) 189{ 190 if (imm >= BIT(16)) { 191 emit_load_imm(r_tmp, imm, ctx); 192 emit_or(dst, src, r_tmp, ctx); 193 } else { 194 emit_instr(ctx, ori, dst, src, imm); 195 } 196} 197 198static inline void emit_daddiu(unsigned int dst, unsigned int src, 199 int imm, struct jit_ctx *ctx) 200{ 201 /* 202 * Only used for stack, so the imm is relatively small 203 * and it fits in 15-bits 204 */ 205 emit_instr(ctx, daddiu, dst, src, imm); 206} 207 208static inline void emit_addiu(unsigned int dst, unsigned int src, 209 u32 imm, struct jit_ctx *ctx) 210{ 211 if (!is_range16(imm)) { 212 emit_load_imm(r_tmp, imm, ctx); 213 emit_addu(dst, r_tmp, src, ctx); 214 } else { 215 emit_instr(ctx, addiu, dst, src, imm); 216 } 217} 218 219static inline void emit_and(unsigned int dst, unsigned int src1, 220 unsigned int src2, struct jit_ctx *ctx) 221{ 222 emit_instr(ctx, and, dst, src1, src2); 223} 224 225static inline void emit_andi(unsigned int dst, unsigned int src, 226 u32 imm, struct jit_ctx *ctx) 227{ 228 /* If imm does not fit in u16 then load it to register */ 229 if (imm >= BIT(16)) { 230 emit_load_imm(r_tmp, imm, ctx); 231 emit_and(dst, src, r_tmp, ctx); 232 } else { 233 emit_instr(ctx, andi, dst, src, imm); 234 } 235} 236 237static inline void emit_xor(unsigned int dst, unsigned int src1, 238 unsigned int src2, struct jit_ctx *ctx) 239{ 240 emit_instr(ctx, xor, dst, src1, src2); 241} 242 243static inline void emit_xori(ptr dst, ptr src, u32 imm, struct jit_ctx *ctx) 244{ 245 /* If imm does not fit in u16 then load it to register */ 246 if (imm >= BIT(16)) { 247 emit_load_imm(r_tmp, imm, ctx); 248 emit_xor(dst, src, r_tmp, ctx); 249 } else { 250 emit_instr(ctx, xori, dst, src, imm); 251 } 252} 253 254static inline void emit_stack_offset(int offset, struct jit_ctx *ctx) 255{ 256 emit_long_instr(ctx, ADDIU, r_sp, r_sp, offset); 257} 258 259static inline void emit_subu(unsigned int dst, unsigned int src1, 260 unsigned int src2, struct jit_ctx *ctx) 261{ 262 emit_instr(ctx, subu, dst, src1, src2); 263} 264 265static inline void emit_neg(unsigned int reg, struct jit_ctx *ctx) 266{ 267 emit_subu(reg, r_zero, reg, ctx); 268} 269 270static inline void emit_sllv(unsigned int dst, unsigned int src, 271 unsigned int sa, struct jit_ctx *ctx) 272{ 273 emit_instr(ctx, sllv, dst, src, sa); 274} 275 276static inline void emit_sll(unsigned int dst, unsigned int src, 277 unsigned int sa, struct jit_ctx *ctx) 278{ 279 /* sa is 5-bits long */ 280 if (sa >= BIT(5)) 281 /* Shifting >= 32 results in zero */ 282 emit_jit_reg_move(dst, r_zero, ctx); 283 else 284 emit_instr(ctx, sll, dst, src, sa); 285} 286 287static inline void emit_srlv(unsigned int dst, unsigned int src, 288 unsigned int sa, struct jit_ctx *ctx) 289{ 290 emit_instr(ctx, srlv, dst, src, sa); 291} 292 293static inline void emit_srl(unsigned int dst, unsigned int src, 294 unsigned int sa, struct jit_ctx *ctx) 295{ 296 /* sa is 5-bits long */ 297 if (sa >= BIT(5)) 298 /* Shifting >= 32 results in zero */ 299 emit_jit_reg_move(dst, r_zero, ctx); 300 else 301 emit_instr(ctx, srl, dst, src, sa); 302} 303 304static inline void emit_slt(unsigned int dst, unsigned int src1, 305 unsigned int src2, struct jit_ctx *ctx) 306{ 307 emit_instr(ctx, slt, dst, src1, src2); 308} 309 310static inline void emit_sltu(unsigned int dst, unsigned int src1, 311 unsigned int src2, struct jit_ctx *ctx) 312{ 313 emit_instr(ctx, sltu, dst, src1, src2); 314} 315 316static inline void emit_sltiu(unsigned dst, unsigned int src, 317 unsigned int imm, struct jit_ctx *ctx) 318{ 319 /* 16 bit immediate */ 320 if (!is_range16((s32)imm)) { 321 emit_load_imm(r_tmp, imm, ctx); 322 emit_sltu(dst, src, r_tmp, ctx); 323 } else { 324 emit_instr(ctx, sltiu, dst, src, imm); 325 } 326 327} 328 329/* Store register on the stack */ 330static inline void emit_store_stack_reg(ptr reg, ptr base, 331 unsigned int offset, 332 struct jit_ctx *ctx) 333{ 334 emit_long_instr(ctx, SW, reg, offset, base); 335} 336 337static inline void emit_store(ptr reg, ptr base, unsigned int offset, 338 struct jit_ctx *ctx) 339{ 340 emit_instr(ctx, sw, reg, offset, base); 341} 342 343static inline void emit_load_stack_reg(ptr reg, ptr base, 344 unsigned int offset, 345 struct jit_ctx *ctx) 346{ 347 emit_long_instr(ctx, LW, reg, offset, base); 348} 349 350static inline void emit_load(unsigned int reg, unsigned int base, 351 unsigned int offset, struct jit_ctx *ctx) 352{ 353 emit_instr(ctx, lw, reg, offset, base); 354} 355 356static inline void emit_load_byte(unsigned int reg, unsigned int base, 357 unsigned int offset, struct jit_ctx *ctx) 358{ 359 emit_instr(ctx, lb, reg, offset, base); 360} 361 362static inline void emit_half_load(unsigned int reg, unsigned int base, 363 unsigned int offset, struct jit_ctx *ctx) 364{ 365 emit_instr(ctx, lh, reg, offset, base); 366} 367 368static inline void emit_half_load_unsigned(unsigned int reg, unsigned int base, 369 unsigned int offset, struct jit_ctx *ctx) 370{ 371 emit_instr(ctx, lhu, reg, offset, base); 372} 373 374static inline void emit_mul(unsigned int dst, unsigned int src1, 375 unsigned int src2, struct jit_ctx *ctx) 376{ 377 emit_instr(ctx, mul, dst, src1, src2); 378} 379 380static inline void emit_div(unsigned int dst, unsigned int src, 381 struct jit_ctx *ctx) 382{ 383 if (ctx->target != NULL) { 384 u32 *p = &ctx->target[ctx->idx]; 385 uasm_i_divu(&p, dst, src); 386 p = &ctx->target[ctx->idx + 1]; 387 uasm_i_mflo(&p, dst); 388 } 389 ctx->idx += 2; /* 2 insts */ 390} 391 392static inline void emit_mod(unsigned int dst, unsigned int src, 393 struct jit_ctx *ctx) 394{ 395 if (ctx->target != NULL) { 396 u32 *p = &ctx->target[ctx->idx]; 397 uasm_i_divu(&p, dst, src); 398 p = &ctx->target[ctx->idx + 1]; 399 uasm_i_mfhi(&p, dst); 400 } 401 ctx->idx += 2; /* 2 insts */ 402} 403 404static inline void emit_dsll(unsigned int dst, unsigned int src, 405 unsigned int sa, struct jit_ctx *ctx) 406{ 407 emit_instr(ctx, dsll, dst, src, sa); 408} 409 410static inline void emit_dsrl32(unsigned int dst, unsigned int src, 411 unsigned int sa, struct jit_ctx *ctx) 412{ 413 emit_instr(ctx, dsrl32, dst, src, sa); 414} 415 416static inline void emit_wsbh(unsigned int dst, unsigned int src, 417 struct jit_ctx *ctx) 418{ 419 emit_instr(ctx, wsbh, dst, src); 420} 421 422/* load pointer to register */ 423static inline void emit_load_ptr(unsigned int dst, unsigned int src, 424 int imm, struct jit_ctx *ctx) 425{ 426 /* src contains the base addr of the 32/64-pointer */ 427 emit_long_instr(ctx, LW, dst, imm, src); 428} 429 430/* load a function pointer to register */ 431static inline void emit_load_func(unsigned int reg, ptr imm, 432 struct jit_ctx *ctx) 433{ 434 if (IS_ENABLED(CONFIG_64BIT)) { 435 /* At this point imm is always 64-bit */ 436 emit_load_imm(r_tmp, (u64)imm >> 32, ctx); 437 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ 438 emit_ori(r_tmp, r_tmp_imm, (imm >> 16) & 0xffff, ctx); 439 emit_dsll(r_tmp_imm, r_tmp, 16, ctx); /* left shift by 16 */ 440 emit_ori(reg, r_tmp_imm, imm & 0xffff, ctx); 441 } else { 442 emit_load_imm(reg, imm, ctx); 443 } 444} 445 446/* Move to real MIPS register */ 447static inline void emit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) 448{ 449 emit_long_instr(ctx, ADDU, dst, src, r_zero); 450} 451 452/* Move to JIT (32-bit) register */ 453static inline void emit_jit_reg_move(ptr dst, ptr src, struct jit_ctx *ctx) 454{ 455 emit_addu(dst, src, r_zero, ctx); 456} 457 458/* Compute the immediate value for PC-relative branches. */ 459static inline u32 b_imm(unsigned int tgt, struct jit_ctx *ctx) 460{ 461 if (ctx->target == NULL) 462 return 0; 463 464 /* 465 * We want a pc-relative branch. We only do forward branches 466 * so tgt is always after pc. tgt is the instruction offset 467 * we want to jump to. 468 469 * Branch on MIPS: 470 * I: target_offset <- sign_extend(offset) 471 * I+1: PC += target_offset (delay slot) 472 * 473 * ctx->idx currently points to the branch instruction 474 * but the offset is added to the delay slot so we need 475 * to subtract 4. 476 */ 477 return ctx->offsets[tgt] - 478 (ctx->idx * 4 - ctx->prologue_bytes) - 4; 479} 480 481static inline void emit_bcond(int cond, unsigned int reg1, unsigned int reg2, 482 unsigned int imm, struct jit_ctx *ctx) 483{ 484 if (ctx->target != NULL) { 485 u32 *p = &ctx->target[ctx->idx]; 486 487 switch (cond) { 488 case MIPS_COND_EQ: 489 uasm_i_beq(&p, reg1, reg2, imm); 490 break; 491 case MIPS_COND_NE: 492 uasm_i_bne(&p, reg1, reg2, imm); 493 break; 494 case MIPS_COND_ALL: 495 uasm_i_b(&p, imm); 496 break; 497 default: 498 pr_warn("%s: Unhandled branch conditional: %d\n", 499 __func__, cond); 500 } 501 } 502 ctx->idx++; 503} 504 505static inline void emit_b(unsigned int imm, struct jit_ctx *ctx) 506{ 507 emit_bcond(MIPS_COND_ALL, r_zero, r_zero, imm, ctx); 508} 509 510static inline void emit_jalr(unsigned int link, unsigned int reg, 511 struct jit_ctx *ctx) 512{ 513 emit_instr(ctx, jalr, link, reg); 514} 515 516static inline void emit_jr(unsigned int reg, struct jit_ctx *ctx) 517{ 518 emit_instr(ctx, jr, reg); 519} 520 521static inline u16 align_sp(unsigned int num) 522{ 523 /* Double word alignment for 32-bit, quadword for 64-bit */ 524 unsigned int align = IS_ENABLED(CONFIG_64BIT) ? 16 : 8; 525 num = (num + (align - 1)) & -align; 526 return num; 527} 528 529static void save_bpf_jit_regs(struct jit_ctx *ctx, unsigned offset) 530{ 531 int i = 0, real_off = 0; 532 u32 sflags, tmp_flags; 533 534 /* Adjust the stack pointer */ 535 if (offset) 536 emit_stack_offset(-align_sp(offset), ctx); 537 538 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; 539 /* sflags is essentially a bitmap */ 540 while (tmp_flags) { 541 if ((sflags >> i) & 0x1) { 542 emit_store_stack_reg(MIPS_R_S0 + i, r_sp, real_off, 543 ctx); 544 real_off += SZREG; 545 } 546 i++; 547 tmp_flags >>= 1; 548 } 549 550 /* save return address */ 551 if (ctx->flags & SEEN_CALL) { 552 emit_store_stack_reg(r_ra, r_sp, real_off, ctx); 553 real_off += SZREG; 554 } 555 556 /* Setup r_M leaving the alignment gap if necessary */ 557 if (ctx->flags & SEEN_MEM) { 558 if (real_off % (SZREG * 2)) 559 real_off += SZREG; 560 emit_long_instr(ctx, ADDIU, r_M, r_sp, real_off); 561 } 562} 563 564static void restore_bpf_jit_regs(struct jit_ctx *ctx, 565 unsigned int offset) 566{ 567 int i, real_off = 0; 568 u32 sflags, tmp_flags; 569 570 tmp_flags = sflags = ctx->flags >> SEEN_SREG_SFT; 571 /* sflags is a bitmap */ 572 i = 0; 573 while (tmp_flags) { 574 if ((sflags >> i) & 0x1) { 575 emit_load_stack_reg(MIPS_R_S0 + i, r_sp, real_off, 576 ctx); 577 real_off += SZREG; 578 } 579 i++; 580 tmp_flags >>= 1; 581 } 582 583 /* restore return address */ 584 if (ctx->flags & SEEN_CALL) 585 emit_load_stack_reg(r_ra, r_sp, real_off, ctx); 586 587 /* Restore the sp and discard the scrach memory */ 588 if (offset) 589 emit_stack_offset(align_sp(offset), ctx); 590} 591 592static unsigned int get_stack_depth(struct jit_ctx *ctx) 593{ 594 int sp_off = 0; 595 596 597 /* How may s* regs do we need to preserved? */ 598 sp_off += hweight32(ctx->flags >> SEEN_SREG_SFT) * SZREG; 599 600 if (ctx->flags & SEEN_MEM) 601 sp_off += 4 * BPF_MEMWORDS; /* BPF_MEMWORDS are 32-bit */ 602 603 if (ctx->flags & SEEN_CALL) 604 sp_off += SZREG; /* Space for our ra register */ 605 606 return sp_off; 607} 608 609static void build_prologue(struct jit_ctx *ctx) 610{ 611 int sp_off; 612 613 /* Calculate the total offset for the stack pointer */ 614 sp_off = get_stack_depth(ctx); 615 save_bpf_jit_regs(ctx, sp_off); 616 617 if (ctx->flags & SEEN_SKB) 618 emit_reg_move(r_skb, MIPS_R_A0, ctx); 619 620 if (ctx->flags & SEEN_SKB_DATA) { 621 /* Load packet length */ 622 emit_load(r_skb_len, r_skb, offsetof(struct sk_buff, len), 623 ctx); 624 emit_load(r_tmp, r_skb, offsetof(struct sk_buff, data_len), 625 ctx); 626 /* Load the data pointer */ 627 emit_load_ptr(r_skb_data, r_skb, 628 offsetof(struct sk_buff, data), ctx); 629 /* Load the header length */ 630 emit_subu(r_skb_hl, r_skb_len, r_tmp, ctx); 631 } 632 633 if (ctx->flags & SEEN_X) 634 emit_jit_reg_move(r_X, r_zero, ctx); 635 636 /* 637 * Do not leak kernel data to userspace, we only need to clear 638 * r_A if it is ever used. In fact if it is never used, we 639 * will not save/restore it, so clearing it in this case would 640 * corrupt the state of the caller. 641 */ 642 if (bpf_needs_clear_a(&ctx->skf->insns[0]) && 643 (ctx->flags & SEEN_A)) 644 emit_jit_reg_move(r_A, r_zero, ctx); 645} 646 647static void build_epilogue(struct jit_ctx *ctx) 648{ 649 unsigned int sp_off; 650 651 /* Calculate the total offset for the stack pointer */ 652 653 sp_off = get_stack_depth(ctx); 654 restore_bpf_jit_regs(ctx, sp_off); 655 656 /* Return */ 657 emit_jr(r_ra, ctx); 658 emit_nop(ctx); 659} 660 661#define CHOOSE_LOAD_FUNC(K, func) \ 662 ((int)K < 0 ? ((int)K >= SKF_LL_OFF ? func##_negative : func) : \ 663 func##_positive) 664 665static bool is_bad_offset(int b_off) 666{ 667 return b_off > 0x1ffff || b_off < -0x20000; 668} 669 670static int build_body(struct jit_ctx *ctx) 671{ 672 const struct bpf_prog *prog = ctx->skf; 673 const struct sock_filter *inst; 674 unsigned int i, off, condt; 675 u32 k, b_off __maybe_unused; 676 u8 (*sk_load_func)(unsigned long *skb, int offset); 677 678 for (i = 0; i < prog->len; i++) { 679 u16 code; 680 681 inst = &(prog->insns[i]); 682 pr_debug("%s: code->0x%02x, jt->0x%x, jf->0x%x, k->0x%x\n", 683 __func__, inst->code, inst->jt, inst->jf, inst->k); 684 k = inst->k; 685 code = bpf_anc_helper(inst); 686 687 if (ctx->target == NULL) 688 ctx->offsets[i] = ctx->idx * 4; 689 690 switch (code) { 691 case BPF_LD | BPF_IMM: 692 /* A <- k ==> li r_A, k */ 693 ctx->flags |= SEEN_A; 694 emit_load_imm(r_A, k, ctx); 695 break; 696 case BPF_LD | BPF_W | BPF_LEN: 697 BUILD_BUG_ON(sizeof_field(struct sk_buff, len) != 4); 698 /* A <- len ==> lw r_A, offset(skb) */ 699 ctx->flags |= SEEN_SKB | SEEN_A; 700 off = offsetof(struct sk_buff, len); 701 emit_load(r_A, r_skb, off, ctx); 702 break; 703 case BPF_LD | BPF_MEM: 704 /* A <- M[k] ==> lw r_A, offset(M) */ 705 ctx->flags |= SEEN_MEM | SEEN_A; 706 emit_load(r_A, r_M, SCRATCH_OFF(k), ctx); 707 break; 708 case BPF_LD | BPF_W | BPF_ABS: 709 /* A <- P[k:4] */ 710 sk_load_func = CHOOSE_LOAD_FUNC(k, sk_load_word); 711 goto load; 712 case BPF_LD | BPF_H | BPF_ABS: 713 /* A <- P[k:2] */ 714 sk_load_func = CHOOSE_LOAD_FUNC(k, sk_load_half); 715 goto load; 716 case BPF_LD | BPF_B | BPF_ABS: 717 /* A <- P[k:1] */ 718 sk_load_func = CHOOSE_LOAD_FUNC(k, sk_load_byte); 719load: 720 emit_load_imm(r_off, k, ctx); 721load_common: 722 ctx->flags |= SEEN_CALL | SEEN_OFF | 723 SEEN_SKB | SEEN_A | SEEN_SKB_DATA; 724 725 emit_load_func(r_s0, (ptr)sk_load_func, ctx); 726 emit_reg_move(MIPS_R_A0, r_skb, ctx); 727 emit_jalr(MIPS_R_RA, r_s0, ctx); 728 /* Load second argument to delay slot */ 729 emit_reg_move(MIPS_R_A1, r_off, ctx); 730 /* Check the error value */ 731 emit_bcond(MIPS_COND_EQ, r_ret, 0, b_imm(i + 1, ctx), 732 ctx); 733 /* Load return register on DS for failures */ 734 emit_reg_move(r_ret, r_zero, ctx); 735 /* Return with error */ 736 b_off = b_imm(prog->len, ctx); 737 if (is_bad_offset(b_off)) 738 return -E2BIG; 739 emit_b(b_off, ctx); 740 emit_nop(ctx); 741 break; 742 case BPF_LD | BPF_W | BPF_IND: 743 /* A <- P[X + k:4] */ 744 sk_load_func = sk_load_word; 745 goto load_ind; 746 case BPF_LD | BPF_H | BPF_IND: 747 /* A <- P[X + k:2] */ 748 sk_load_func = sk_load_half; 749 goto load_ind; 750 case BPF_LD | BPF_B | BPF_IND: 751 /* A <- P[X + k:1] */ 752 sk_load_func = sk_load_byte; 753load_ind: 754 ctx->flags |= SEEN_OFF | SEEN_X; 755 emit_addiu(r_off, r_X, k, ctx); 756 goto load_common; 757 case BPF_LDX | BPF_IMM: 758 /* X <- k */ 759 ctx->flags |= SEEN_X; 760 emit_load_imm(r_X, k, ctx); 761 break; 762 case BPF_LDX | BPF_MEM: 763 /* X <- M[k] */ 764 ctx->flags |= SEEN_X | SEEN_MEM; 765 emit_load(r_X, r_M, SCRATCH_OFF(k), ctx); 766 break; 767 case BPF_LDX | BPF_W | BPF_LEN: 768 /* X <- len */ 769 ctx->flags |= SEEN_X | SEEN_SKB; 770 off = offsetof(struct sk_buff, len); 771 emit_load(r_X, r_skb, off, ctx); 772 break; 773 case BPF_LDX | BPF_B | BPF_MSH: 774 /* X <- 4 * (P[k:1] & 0xf) */ 775 ctx->flags |= SEEN_X | SEEN_CALL | SEEN_SKB; 776 /* Load offset to a1 */ 777 emit_load_func(r_s0, (ptr)sk_load_byte, ctx); 778 /* 779 * This may emit two instructions so it may not fit 780 * in the delay slot. So use a0 in the delay slot. 781 */ 782 emit_load_imm(MIPS_R_A1, k, ctx); 783 emit_jalr(MIPS_R_RA, r_s0, ctx); 784 emit_reg_move(MIPS_R_A0, r_skb, ctx); /* delay slot */ 785 /* Check the error value */ 786 b_off = b_imm(prog->len, ctx); 787 if (is_bad_offset(b_off)) 788 return -E2BIG; 789 emit_bcond(MIPS_COND_NE, r_ret, 0, b_off, ctx); 790 emit_reg_move(r_ret, r_zero, ctx); 791 /* We are good */ 792 /* X <- P[1:K] & 0xf */ 793 emit_andi(r_X, r_A, 0xf, ctx); 794 /* X << 2 */ 795 emit_b(b_imm(i + 1, ctx), ctx); 796 emit_sll(r_X, r_X, 2, ctx); /* delay slot */ 797 break; 798 case BPF_ST: 799 /* M[k] <- A */ 800 ctx->flags |= SEEN_MEM | SEEN_A; 801 emit_store(r_A, r_M, SCRATCH_OFF(k), ctx); 802 break; 803 case BPF_STX: 804 /* M[k] <- X */ 805 ctx->flags |= SEEN_MEM | SEEN_X; 806 emit_store(r_X, r_M, SCRATCH_OFF(k), ctx); 807 break; 808 case BPF_ALU | BPF_ADD | BPF_K: 809 /* A += K */ 810 ctx->flags |= SEEN_A; 811 emit_addiu(r_A, r_A, k, ctx); 812 break; 813 case BPF_ALU | BPF_ADD | BPF_X: 814 /* A += X */ 815 ctx->flags |= SEEN_A | SEEN_X; 816 emit_addu(r_A, r_A, r_X, ctx); 817 break; 818 case BPF_ALU | BPF_SUB | BPF_K: 819 /* A -= K */ 820 ctx->flags |= SEEN_A; 821 emit_addiu(r_A, r_A, -k, ctx); 822 break; 823 case BPF_ALU | BPF_SUB | BPF_X: 824 /* A -= X */ 825 ctx->flags |= SEEN_A | SEEN_X; 826 emit_subu(r_A, r_A, r_X, ctx); 827 break; 828 case BPF_ALU | BPF_MUL | BPF_K: 829 /* A *= K */ 830 /* Load K to scratch register before MUL */ 831 ctx->flags |= SEEN_A; 832 emit_load_imm(r_s0, k, ctx); 833 emit_mul(r_A, r_A, r_s0, ctx); 834 break; 835 case BPF_ALU | BPF_MUL | BPF_X: 836 /* A *= X */ 837 ctx->flags |= SEEN_A | SEEN_X; 838 emit_mul(r_A, r_A, r_X, ctx); 839 break; 840 case BPF_ALU | BPF_DIV | BPF_K: 841 /* A /= k */ 842 if (k == 1) 843 break; 844 if (optimize_div(&k)) { 845 ctx->flags |= SEEN_A; 846 emit_srl(r_A, r_A, k, ctx); 847 break; 848 } 849 ctx->flags |= SEEN_A; 850 emit_load_imm(r_s0, k, ctx); 851 emit_div(r_A, r_s0, ctx); 852 break; 853 case BPF_ALU | BPF_MOD | BPF_K: 854 /* A %= k */ 855 if (k == 1) { 856 ctx->flags |= SEEN_A; 857 emit_jit_reg_move(r_A, r_zero, ctx); 858 } else { 859 ctx->flags |= SEEN_A; 860 emit_load_imm(r_s0, k, ctx); 861 emit_mod(r_A, r_s0, ctx); 862 } 863 break; 864 case BPF_ALU | BPF_DIV | BPF_X: 865 /* A /= X */ 866 ctx->flags |= SEEN_X | SEEN_A; 867 /* Check if r_X is zero */ 868 b_off = b_imm(prog->len, ctx); 869 if (is_bad_offset(b_off)) 870 return -E2BIG; 871 emit_bcond(MIPS_COND_EQ, r_X, r_zero, b_off, ctx); 872 emit_load_imm(r_ret, 0, ctx); /* delay slot */ 873 emit_div(r_A, r_X, ctx); 874 break; 875 case BPF_ALU | BPF_MOD | BPF_X: 876 /* A %= X */ 877 ctx->flags |= SEEN_X | SEEN_A; 878 /* Check if r_X is zero */ 879 b_off = b_imm(prog->len, ctx); 880 if (is_bad_offset(b_off)) 881 return -E2BIG; 882 emit_bcond(MIPS_COND_EQ, r_X, r_zero, b_off, ctx); 883 emit_load_imm(r_ret, 0, ctx); /* delay slot */ 884 emit_mod(r_A, r_X, ctx); 885 break; 886 case BPF_ALU | BPF_OR | BPF_K: 887 /* A |= K */ 888 ctx->flags |= SEEN_A; 889 emit_ori(r_A, r_A, k, ctx); 890 break; 891 case BPF_ALU | BPF_OR | BPF_X: 892 /* A |= X */ 893 ctx->flags |= SEEN_A; 894 emit_ori(r_A, r_A, r_X, ctx); 895 break; 896 case BPF_ALU | BPF_XOR | BPF_K: 897 /* A ^= k */ 898 ctx->flags |= SEEN_A; 899 emit_xori(r_A, r_A, k, ctx); 900 break; 901 case BPF_ANC | SKF_AD_ALU_XOR_X: 902 case BPF_ALU | BPF_XOR | BPF_X: 903 /* A ^= X */ 904 ctx->flags |= SEEN_A; 905 emit_xor(r_A, r_A, r_X, ctx); 906 break; 907 case BPF_ALU | BPF_AND | BPF_K: 908 /* A &= K */ 909 ctx->flags |= SEEN_A; 910 emit_andi(r_A, r_A, k, ctx); 911 break; 912 case BPF_ALU | BPF_AND | BPF_X: 913 /* A &= X */ 914 ctx->flags |= SEEN_A | SEEN_X; 915 emit_and(r_A, r_A, r_X, ctx); 916 break; 917 case BPF_ALU | BPF_LSH | BPF_K: 918 /* A <<= K */ 919 ctx->flags |= SEEN_A; 920 emit_sll(r_A, r_A, k, ctx); 921 break; 922 case BPF_ALU | BPF_LSH | BPF_X: 923 /* A <<= X */ 924 ctx->flags |= SEEN_A | SEEN_X; 925 emit_sllv(r_A, r_A, r_X, ctx); 926 break; 927 case BPF_ALU | BPF_RSH | BPF_K: 928 /* A >>= K */ 929 ctx->flags |= SEEN_A; 930 emit_srl(r_A, r_A, k, ctx); 931 break; 932 case BPF_ALU | BPF_RSH | BPF_X: 933 ctx->flags |= SEEN_A | SEEN_X; 934 emit_srlv(r_A, r_A, r_X, ctx); 935 break; 936 case BPF_ALU | BPF_NEG: 937 /* A = -A */ 938 ctx->flags |= SEEN_A; 939 emit_neg(r_A, ctx); 940 break; 941 case BPF_JMP | BPF_JA: 942 /* pc += K */ 943 b_off = b_imm(i + k + 1, ctx); 944 if (is_bad_offset(b_off)) 945 return -E2BIG; 946 emit_b(b_off, ctx); 947 emit_nop(ctx); 948 break; 949 case BPF_JMP | BPF_JEQ | BPF_K: 950 /* pc += ( A == K ) ? pc->jt : pc->jf */ 951 condt = MIPS_COND_EQ | MIPS_COND_K; 952 goto jmp_cmp; 953 case BPF_JMP | BPF_JEQ | BPF_X: 954 ctx->flags |= SEEN_X; 955 /* pc += ( A == X ) ? pc->jt : pc->jf */ 956 condt = MIPS_COND_EQ | MIPS_COND_X; 957 goto jmp_cmp; 958 case BPF_JMP | BPF_JGE | BPF_K: 959 /* pc += ( A >= K ) ? pc->jt : pc->jf */ 960 condt = MIPS_COND_GE | MIPS_COND_K; 961 goto jmp_cmp; 962 case BPF_JMP | BPF_JGE | BPF_X: 963 ctx->flags |= SEEN_X; 964 /* pc += ( A >= X ) ? pc->jt : pc->jf */ 965 condt = MIPS_COND_GE | MIPS_COND_X; 966 goto jmp_cmp; 967 case BPF_JMP | BPF_JGT | BPF_K: 968 /* pc += ( A > K ) ? pc->jt : pc->jf */ 969 condt = MIPS_COND_GT | MIPS_COND_K; 970 goto jmp_cmp; 971 case BPF_JMP | BPF_JGT | BPF_X: 972 ctx->flags |= SEEN_X; 973 /* pc += ( A > X ) ? pc->jt : pc->jf */ 974 condt = MIPS_COND_GT | MIPS_COND_X; 975jmp_cmp: 976 /* Greater or Equal */ 977 if ((condt & MIPS_COND_GE) || 978 (condt & MIPS_COND_GT)) { 979 if (condt & MIPS_COND_K) { /* K */ 980 ctx->flags |= SEEN_A; 981 emit_sltiu(r_s0, r_A, k, ctx); 982 } else { /* X */ 983 ctx->flags |= SEEN_A | 984 SEEN_X; 985 emit_sltu(r_s0, r_A, r_X, ctx); 986 } 987 /* A < (K|X) ? r_scrach = 1 */ 988 b_off = b_imm(i + inst->jf + 1, ctx); 989 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, 990 ctx); 991 emit_nop(ctx); 992 /* A > (K|X) ? scratch = 0 */ 993 if (condt & MIPS_COND_GT) { 994 /* Checking for equality */ 995 ctx->flags |= SEEN_A | SEEN_X; 996 if (condt & MIPS_COND_K) 997 emit_load_imm(r_s0, k, ctx); 998 else 999 emit_jit_reg_move(r_s0, r_X, 1000 ctx); 1001 b_off = b_imm(i + inst->jf + 1, ctx); 1002 emit_bcond(MIPS_COND_EQ, r_A, r_s0, 1003 b_off, ctx); 1004 emit_nop(ctx); 1005 /* Finally, A > K|X */ 1006 b_off = b_imm(i + inst->jt + 1, ctx); 1007 emit_b(b_off, ctx); 1008 emit_nop(ctx); 1009 } else { 1010 /* A >= (K|X) so jump */ 1011 b_off = b_imm(i + inst->jt + 1, ctx); 1012 emit_b(b_off, ctx); 1013 emit_nop(ctx); 1014 } 1015 } else { 1016 /* A == K|X */ 1017 if (condt & MIPS_COND_K) { /* K */ 1018 ctx->flags |= SEEN_A; 1019 emit_load_imm(r_s0, k, ctx); 1020 /* jump true */ 1021 b_off = b_imm(i + inst->jt + 1, ctx); 1022 emit_bcond(MIPS_COND_EQ, r_A, r_s0, 1023 b_off, ctx); 1024 emit_nop(ctx); 1025 /* jump false */ 1026 b_off = b_imm(i + inst->jf + 1, 1027 ctx); 1028 emit_bcond(MIPS_COND_NE, r_A, r_s0, 1029 b_off, ctx); 1030 emit_nop(ctx); 1031 } else { /* X */ 1032 /* jump true */ 1033 ctx->flags |= SEEN_A | SEEN_X; 1034 b_off = b_imm(i + inst->jt + 1, 1035 ctx); 1036 emit_bcond(MIPS_COND_EQ, r_A, r_X, 1037 b_off, ctx); 1038 emit_nop(ctx); 1039 /* jump false */ 1040 b_off = b_imm(i + inst->jf + 1, ctx); 1041 emit_bcond(MIPS_COND_NE, r_A, r_X, 1042 b_off, ctx); 1043 emit_nop(ctx); 1044 } 1045 } 1046 break; 1047 case BPF_JMP | BPF_JSET | BPF_K: 1048 ctx->flags |= SEEN_A; 1049 /* pc += (A & K) ? pc -> jt : pc -> jf */ 1050 emit_load_imm(r_s1, k, ctx); 1051 emit_and(r_s0, r_A, r_s1, ctx); 1052 /* jump true */ 1053 b_off = b_imm(i + inst->jt + 1, ctx); 1054 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); 1055 emit_nop(ctx); 1056 /* jump false */ 1057 b_off = b_imm(i + inst->jf + 1, ctx); 1058 emit_b(b_off, ctx); 1059 emit_nop(ctx); 1060 break; 1061 case BPF_JMP | BPF_JSET | BPF_X: 1062 ctx->flags |= SEEN_X | SEEN_A; 1063 /* pc += (A & X) ? pc -> jt : pc -> jf */ 1064 emit_and(r_s0, r_A, r_X, ctx); 1065 /* jump true */ 1066 b_off = b_imm(i + inst->jt + 1, ctx); 1067 emit_bcond(MIPS_COND_NE, r_s0, r_zero, b_off, ctx); 1068 emit_nop(ctx); 1069 /* jump false */ 1070 b_off = b_imm(i + inst->jf + 1, ctx); 1071 emit_b(b_off, ctx); 1072 emit_nop(ctx); 1073 break; 1074 case BPF_RET | BPF_A: 1075 ctx->flags |= SEEN_A; 1076 if (i != prog->len - 1) { 1077 /* 1078 * If this is not the last instruction 1079 * then jump to the epilogue 1080 */ 1081 b_off = b_imm(prog->len, ctx); 1082 if (is_bad_offset(b_off)) 1083 return -E2BIG; 1084 emit_b(b_off, ctx); 1085 } 1086 emit_reg_move(r_ret, r_A, ctx); /* delay slot */ 1087 break; 1088 case BPF_RET | BPF_K: 1089 /* 1090 * It can emit two instructions so it does not fit on 1091 * the delay slot. 1092 */ 1093 emit_load_imm(r_ret, k, ctx); 1094 if (i != prog->len - 1) { 1095 /* 1096 * If this is not the last instruction 1097 * then jump to the epilogue 1098 */ 1099 b_off = b_imm(prog->len, ctx); 1100 if (is_bad_offset(b_off)) 1101 return -E2BIG; 1102 emit_b(b_off, ctx); 1103 emit_nop(ctx); 1104 } 1105 break; 1106 case BPF_MISC | BPF_TAX: 1107 /* X = A */ 1108 ctx->flags |= SEEN_X | SEEN_A; 1109 emit_jit_reg_move(r_X, r_A, ctx); 1110 break; 1111 case BPF_MISC | BPF_TXA: 1112 /* A = X */ 1113 ctx->flags |= SEEN_A | SEEN_X; 1114 emit_jit_reg_move(r_A, r_X, ctx); 1115 break; 1116 /* AUX */ 1117 case BPF_ANC | SKF_AD_PROTOCOL: 1118 /* A = ntohs(skb->protocol */ 1119 ctx->flags |= SEEN_SKB | SEEN_OFF | SEEN_A; 1120 BUILD_BUG_ON(sizeof_field(struct sk_buff, 1121 protocol) != 2); 1122 off = offsetof(struct sk_buff, protocol); 1123 emit_half_load(r_A, r_skb, off, ctx); 1124#ifdef CONFIG_CPU_LITTLE_ENDIAN 1125 /* This needs little endian fixup */ 1126 if (cpu_has_wsbh) { 1127 /* R2 and later have the wsbh instruction */ 1128 emit_wsbh(r_A, r_A, ctx); 1129 } else { 1130 /* Get first byte */ 1131 emit_andi(r_tmp_imm, r_A, 0xff, ctx); 1132 /* Shift it */ 1133 emit_sll(r_tmp, r_tmp_imm, 8, ctx); 1134 /* Get second byte */ 1135 emit_srl(r_tmp_imm, r_A, 8, ctx); 1136 emit_andi(r_tmp_imm, r_tmp_imm, 0xff, ctx); 1137 /* Put everyting together in r_A */ 1138 emit_or(r_A, r_tmp, r_tmp_imm, ctx); 1139 } 1140#endif 1141 break; 1142 case BPF_ANC | SKF_AD_CPU: 1143 ctx->flags |= SEEN_A | SEEN_OFF; 1144 /* A = current_thread_info()->cpu */ 1145 BUILD_BUG_ON(sizeof_field(struct thread_info, 1146 cpu) != 4); 1147 off = offsetof(struct thread_info, cpu); 1148 /* $28/gp points to the thread_info struct */ 1149 emit_load(r_A, 28, off, ctx); 1150 break; 1151 case BPF_ANC | SKF_AD_IFINDEX: 1152 /* A = skb->dev->ifindex */ 1153 case BPF_ANC | SKF_AD_HATYPE: 1154 /* A = skb->dev->type */ 1155 ctx->flags |= SEEN_SKB | SEEN_A; 1156 off = offsetof(struct sk_buff, dev); 1157 /* Load *dev pointer */ 1158 emit_load_ptr(r_s0, r_skb, off, ctx); 1159 /* error (0) in the delay slot */ 1160 b_off = b_imm(prog->len, ctx); 1161 if (is_bad_offset(b_off)) 1162 return -E2BIG; 1163 emit_bcond(MIPS_COND_EQ, r_s0, r_zero, b_off, ctx); 1164 emit_reg_move(r_ret, r_zero, ctx); 1165 if (code == (BPF_ANC | SKF_AD_IFINDEX)) { 1166 BUILD_BUG_ON(sizeof_field(struct net_device, ifindex) != 4); 1167 off = offsetof(struct net_device, ifindex); 1168 emit_load(r_A, r_s0, off, ctx); 1169 } else { /* (code == (BPF_ANC | SKF_AD_HATYPE) */ 1170 BUILD_BUG_ON(sizeof_field(struct net_device, type) != 2); 1171 off = offsetof(struct net_device, type); 1172 emit_half_load_unsigned(r_A, r_s0, off, ctx); 1173 } 1174 break; 1175 case BPF_ANC | SKF_AD_MARK: 1176 ctx->flags |= SEEN_SKB | SEEN_A; 1177 BUILD_BUG_ON(sizeof_field(struct sk_buff, mark) != 4); 1178 off = offsetof(struct sk_buff, mark); 1179 emit_load(r_A, r_skb, off, ctx); 1180 break; 1181 case BPF_ANC | SKF_AD_RXHASH: 1182 ctx->flags |= SEEN_SKB | SEEN_A; 1183 BUILD_BUG_ON(sizeof_field(struct sk_buff, hash) != 4); 1184 off = offsetof(struct sk_buff, hash); 1185 emit_load(r_A, r_skb, off, ctx); 1186 break; 1187 case BPF_ANC | SKF_AD_VLAN_TAG: 1188 ctx->flags |= SEEN_SKB | SEEN_A; 1189 BUILD_BUG_ON(sizeof_field(struct sk_buff, 1190 vlan_tci) != 2); 1191 off = offsetof(struct sk_buff, vlan_tci); 1192 emit_half_load_unsigned(r_A, r_skb, off, ctx); 1193 break; 1194 case BPF_ANC | SKF_AD_VLAN_TAG_PRESENT: 1195 ctx->flags |= SEEN_SKB | SEEN_A; 1196 emit_load_byte(r_A, r_skb, PKT_VLAN_PRESENT_OFFSET(), ctx); 1197 if (PKT_VLAN_PRESENT_BIT) 1198 emit_srl(r_A, r_A, PKT_VLAN_PRESENT_BIT, ctx); 1199 if (PKT_VLAN_PRESENT_BIT < 7) 1200 emit_andi(r_A, r_A, 1, ctx); 1201 break; 1202 case BPF_ANC | SKF_AD_PKTTYPE: 1203 ctx->flags |= SEEN_SKB; 1204 1205 emit_load_byte(r_tmp, r_skb, PKT_TYPE_OFFSET(), ctx); 1206 /* Keep only the last 3 bits */ 1207 emit_andi(r_A, r_tmp, PKT_TYPE_MAX, ctx); 1208#ifdef __BIG_ENDIAN_BITFIELD 1209 /* Get the actual packet type to the lower 3 bits */ 1210 emit_srl(r_A, r_A, 5, ctx); 1211#endif 1212 break; 1213 case BPF_ANC | SKF_AD_QUEUE: 1214 ctx->flags |= SEEN_SKB | SEEN_A; 1215 BUILD_BUG_ON(sizeof_field(struct sk_buff, 1216 queue_mapping) != 2); 1217 BUILD_BUG_ON(offsetof(struct sk_buff, 1218 queue_mapping) > 0xff); 1219 off = offsetof(struct sk_buff, queue_mapping); 1220 emit_half_load_unsigned(r_A, r_skb, off, ctx); 1221 break; 1222 default: 1223 pr_debug("%s: Unhandled opcode: 0x%02x\n", __FILE__, 1224 inst->code); 1225 return -1; 1226 } 1227 } 1228 1229 /* compute offsets only during the first pass */ 1230 if (ctx->target == NULL) 1231 ctx->offsets[i] = ctx->idx * 4; 1232 1233 return 0; 1234} 1235 1236void bpf_jit_compile(struct bpf_prog *fp) 1237{ 1238 struct jit_ctx ctx; 1239 unsigned int alloc_size, tmp_idx; 1240 1241 if (!bpf_jit_enable) 1242 return; 1243 1244 memset(&ctx, 0, sizeof(ctx)); 1245 1246 ctx.offsets = kcalloc(fp->len + 1, sizeof(*ctx.offsets), GFP_KERNEL); 1247 if (ctx.offsets == NULL) 1248 return; 1249 1250 ctx.skf = fp; 1251 1252 if (build_body(&ctx)) 1253 goto out; 1254 1255 tmp_idx = ctx.idx; 1256 build_prologue(&ctx); 1257 ctx.prologue_bytes = (ctx.idx - tmp_idx) * 4; 1258 /* just to complete the ctx.idx count */ 1259 build_epilogue(&ctx); 1260 1261 alloc_size = 4 * ctx.idx; 1262 ctx.target = module_alloc(alloc_size); 1263 if (ctx.target == NULL) 1264 goto out; 1265 1266 /* Clean it */ 1267 memset(ctx.target, 0, alloc_size); 1268 1269 ctx.idx = 0; 1270 1271 /* Generate the actual JIT code */ 1272 build_prologue(&ctx); 1273 if (build_body(&ctx)) { 1274 module_memfree(ctx.target); 1275 goto out; 1276 } 1277 build_epilogue(&ctx); 1278 1279 /* Update the icache */ 1280 flush_icache_range((ptr)ctx.target, (ptr)(ctx.target + ctx.idx)); 1281 1282 if (bpf_jit_enable > 1) 1283 /* Dump JIT code */ 1284 bpf_jit_dump(fp->len, alloc_size, 2, ctx.target); 1285 1286 fp->bpf_func = (void *)ctx.target; 1287 fp->jited = 1; 1288 1289out: 1290 kfree(ctx.offsets); 1291} 1292 1293void bpf_jit_free(struct bpf_prog *fp) 1294{ 1295 if (fp->jited) 1296 module_memfree(fp->bpf_func); 1297 1298 bpf_prog_unlock_free(fp); 1299} 1300