1/* SPDX-License-Identifier: GPL-2.0 */ 2/* 3 * Common functionality for RV32 and RV64 BPF JIT compilers 4 * 5 * Copyright (c) 2019 Björn Töpel <bjorn.topel@gmail.com> 6 * 7 */ 8 9#ifndef _BPF_JIT_H 10#define _BPF_JIT_H 11 12#include <linux/bpf.h> 13#include <linux/filter.h> 14#include <asm/cacheflush.h> 15 16static inline bool rvc_enabled(void) 17{ 18 return IS_ENABLED(CONFIG_RISCV_ISA_C); 19} 20 21enum { 22 RV_REG_ZERO = 0, /* The constant value 0 */ 23 RV_REG_RA = 1, /* Return address */ 24 RV_REG_SP = 2, /* Stack pointer */ 25 RV_REG_GP = 3, /* Global pointer */ 26 RV_REG_TP = 4, /* Thread pointer */ 27 RV_REG_T0 = 5, /* Temporaries */ 28 RV_REG_T1 = 6, 29 RV_REG_T2 = 7, 30 RV_REG_FP = 8, /* Saved register/frame pointer */ 31 RV_REG_S1 = 9, /* Saved register */ 32 RV_REG_A0 = 10, /* Function argument/return values */ 33 RV_REG_A1 = 11, /* Function arguments */ 34 RV_REG_A2 = 12, 35 RV_REG_A3 = 13, 36 RV_REG_A4 = 14, 37 RV_REG_A5 = 15, 38 RV_REG_A6 = 16, 39 RV_REG_A7 = 17, 40 RV_REG_S2 = 18, /* Saved registers */ 41 RV_REG_S3 = 19, 42 RV_REG_S4 = 20, 43 RV_REG_S5 = 21, 44 RV_REG_S6 = 22, 45 RV_REG_S7 = 23, 46 RV_REG_S8 = 24, 47 RV_REG_S9 = 25, 48 RV_REG_S10 = 26, 49 RV_REG_S11 = 27, 50 RV_REG_T3 = 28, /* Temporaries */ 51 RV_REG_T4 = 29, 52 RV_REG_T5 = 30, 53 RV_REG_T6 = 31, 54}; 55 56static inline bool is_creg(u8 reg) 57{ 58 return (1 << reg) & (BIT(RV_REG_FP) | 59 BIT(RV_REG_S1) | 60 BIT(RV_REG_A0) | 61 BIT(RV_REG_A1) | 62 BIT(RV_REG_A2) | 63 BIT(RV_REG_A3) | 64 BIT(RV_REG_A4) | 65 BIT(RV_REG_A5)); 66} 67 68struct rv_jit_context { 69 struct bpf_prog *prog; 70 u16 *insns; /* RV insns */ 71 u16 *ro_insns; 72 int ninsns; 73 int prologue_len; 74 int epilogue_offset; 75 int *offset; /* BPF to RV */ 76 int nexentries; 77 unsigned long flags; 78 int stack_size; 79}; 80 81/* Convert from ninsns to bytes. */ 82static inline int ninsns_rvoff(int ninsns) 83{ 84 return ninsns << 1; 85} 86 87struct rv_jit_data { 88 struct bpf_binary_header *header; 89 struct bpf_binary_header *ro_header; 90 u8 *image; 91 u8 *ro_image; 92 struct rv_jit_context ctx; 93}; 94 95static inline void bpf_fill_ill_insns(void *area, unsigned int size) 96{ 97 memset(area, 0, size); 98} 99 100static inline void bpf_flush_icache(void *start, void *end) 101{ 102 flush_icache_range((unsigned long)start, (unsigned long)end); 103} 104 105/* Emit a 4-byte riscv instruction. */ 106static inline void emit(const u32 insn, struct rv_jit_context *ctx) 107{ 108 if (ctx->insns) { 109 ctx->insns[ctx->ninsns] = insn; 110 ctx->insns[ctx->ninsns + 1] = (insn >> 16); 111 } 112 113 ctx->ninsns += 2; 114} 115 116/* Emit a 2-byte riscv compressed instruction. */ 117static inline void emitc(const u16 insn, struct rv_jit_context *ctx) 118{ 119 BUILD_BUG_ON(!rvc_enabled()); 120 121 if (ctx->insns) 122 ctx->insns[ctx->ninsns] = insn; 123 124 ctx->ninsns++; 125} 126 127static inline int epilogue_offset(struct rv_jit_context *ctx) 128{ 129 int to = ctx->epilogue_offset, from = ctx->ninsns; 130 131 return ninsns_rvoff(to - from); 132} 133 134/* Return -1 or inverted cond. */ 135static inline int invert_bpf_cond(u8 cond) 136{ 137 switch (cond) { 138 case BPF_JEQ: 139 return BPF_JNE; 140 case BPF_JGT: 141 return BPF_JLE; 142 case BPF_JLT: 143 return BPF_JGE; 144 case BPF_JGE: 145 return BPF_JLT; 146 case BPF_JLE: 147 return BPF_JGT; 148 case BPF_JNE: 149 return BPF_JEQ; 150 case BPF_JSGT: 151 return BPF_JSLE; 152 case BPF_JSLT: 153 return BPF_JSGE; 154 case BPF_JSGE: 155 return BPF_JSLT; 156 case BPF_JSLE: 157 return BPF_JSGT; 158 } 159 return -1; 160} 161 162static inline bool is_6b_int(long val) 163{ 164 return -(1L << 5) <= val && val < (1L << 5); 165} 166 167static inline bool is_7b_uint(unsigned long val) 168{ 169 return val < (1UL << 7); 170} 171 172static inline bool is_8b_uint(unsigned long val) 173{ 174 return val < (1UL << 8); 175} 176 177static inline bool is_9b_uint(unsigned long val) 178{ 179 return val < (1UL << 9); 180} 181 182static inline bool is_10b_int(long val) 183{ 184 return -(1L << 9) <= val && val < (1L << 9); 185} 186 187static inline bool is_10b_uint(unsigned long val) 188{ 189 return val < (1UL << 10); 190} 191 192static inline bool is_12b_int(long val) 193{ 194 return -(1L << 11) <= val && val < (1L << 11); 195} 196 197static inline int is_12b_check(int off, int insn) 198{ 199 if (!is_12b_int(off)) { 200 pr_err("bpf-jit: insn=%d 12b < offset=%d not supported yet!\n", 201 insn, (int)off); 202 return -1; 203 } 204 return 0; 205} 206 207static inline bool is_13b_int(long val) 208{ 209 return -(1L << 12) <= val && val < (1L << 12); 210} 211 212static inline bool is_21b_int(long val) 213{ 214 return -(1L << 20) <= val && val < (1L << 20); 215} 216 217static inline int rv_offset(int insn, int off, struct rv_jit_context *ctx) 218{ 219 int from, to; 220 221 off++; /* BPF branch is from PC+1, RV is from PC */ 222 from = (insn > 0) ? ctx->offset[insn - 1] : ctx->prologue_len; 223 to = (insn + off > 0) ? ctx->offset[insn + off - 1] : ctx->prologue_len; 224 return ninsns_rvoff(to - from); 225} 226 227/* Instruction formats. */ 228 229static inline u32 rv_r_insn(u8 funct7, u8 rs2, u8 rs1, u8 funct3, u8 rd, 230 u8 opcode) 231{ 232 return (funct7 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 233 (rd << 7) | opcode; 234} 235 236static inline u32 rv_i_insn(u16 imm11_0, u8 rs1, u8 funct3, u8 rd, u8 opcode) 237{ 238 return (imm11_0 << 20) | (rs1 << 15) | (funct3 << 12) | (rd << 7) | 239 opcode; 240} 241 242static inline u32 rv_s_insn(u16 imm11_0, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 243{ 244 u8 imm11_5 = imm11_0 >> 5, imm4_0 = imm11_0 & 0x1f; 245 246 return (imm11_5 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 247 (imm4_0 << 7) | opcode; 248} 249 250static inline u32 rv_b_insn(u16 imm12_1, u8 rs2, u8 rs1, u8 funct3, u8 opcode) 251{ 252 u8 imm12 = ((imm12_1 & 0x800) >> 5) | ((imm12_1 & 0x3f0) >> 4); 253 u8 imm4_1 = ((imm12_1 & 0xf) << 1) | ((imm12_1 & 0x400) >> 10); 254 255 return (imm12 << 25) | (rs2 << 20) | (rs1 << 15) | (funct3 << 12) | 256 (imm4_1 << 7) | opcode; 257} 258 259static inline u32 rv_u_insn(u32 imm31_12, u8 rd, u8 opcode) 260{ 261 return (imm31_12 << 12) | (rd << 7) | opcode; 262} 263 264static inline u32 rv_j_insn(u32 imm20_1, u8 rd, u8 opcode) 265{ 266 u32 imm; 267 268 imm = (imm20_1 & 0x80000) | ((imm20_1 & 0x3ff) << 9) | 269 ((imm20_1 & 0x400) >> 2) | ((imm20_1 & 0x7f800) >> 11); 270 271 return (imm << 12) | (rd << 7) | opcode; 272} 273 274static inline u32 rv_amo_insn(u8 funct5, u8 aq, u8 rl, u8 rs2, u8 rs1, 275 u8 funct3, u8 rd, u8 opcode) 276{ 277 u8 funct7 = (funct5 << 2) | (aq << 1) | rl; 278 279 return rv_r_insn(funct7, rs2, rs1, funct3, rd, opcode); 280} 281 282/* RISC-V compressed instruction formats. */ 283 284static inline u16 rv_cr_insn(u8 funct4, u8 rd, u8 rs2, u8 op) 285{ 286 return (funct4 << 12) | (rd << 7) | (rs2 << 2) | op; 287} 288 289static inline u16 rv_ci_insn(u8 funct3, u32 imm6, u8 rd, u8 op) 290{ 291 u32 imm; 292 293 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 294 return (funct3 << 13) | (rd << 7) | op | imm; 295} 296 297static inline u16 rv_css_insn(u8 funct3, u32 uimm, u8 rs2, u8 op) 298{ 299 return (funct3 << 13) | (uimm << 7) | (rs2 << 2) | op; 300} 301 302static inline u16 rv_ciw_insn(u8 funct3, u32 uimm, u8 rd, u8 op) 303{ 304 return (funct3 << 13) | (uimm << 5) | ((rd & 0x7) << 2) | op; 305} 306 307static inline u16 rv_cl_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rd, 308 u8 op) 309{ 310 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 311 (imm_lo << 5) | ((rd & 0x7) << 2) | op; 312} 313 314static inline u16 rv_cs_insn(u8 funct3, u32 imm_hi, u8 rs1, u32 imm_lo, u8 rs2, 315 u8 op) 316{ 317 return (funct3 << 13) | (imm_hi << 10) | ((rs1 & 0x7) << 7) | 318 (imm_lo << 5) | ((rs2 & 0x7) << 2) | op; 319} 320 321static inline u16 rv_ca_insn(u8 funct6, u8 rd, u8 funct2, u8 rs2, u8 op) 322{ 323 return (funct6 << 10) | ((rd & 0x7) << 7) | (funct2 << 5) | 324 ((rs2 & 0x7) << 2) | op; 325} 326 327static inline u16 rv_cb_insn(u8 funct3, u32 imm6, u8 funct2, u8 rd, u8 op) 328{ 329 u32 imm; 330 331 imm = ((imm6 & 0x20) << 7) | ((imm6 & 0x1f) << 2); 332 return (funct3 << 13) | (funct2 << 10) | ((rd & 0x7) << 7) | op | imm; 333} 334 335/* Instructions shared by both RV32 and RV64. */ 336 337static inline u32 rv_addi(u8 rd, u8 rs1, u16 imm11_0) 338{ 339 return rv_i_insn(imm11_0, rs1, 0, rd, 0x13); 340} 341 342static inline u32 rv_andi(u8 rd, u8 rs1, u16 imm11_0) 343{ 344 return rv_i_insn(imm11_0, rs1, 7, rd, 0x13); 345} 346 347static inline u32 rv_ori(u8 rd, u8 rs1, u16 imm11_0) 348{ 349 return rv_i_insn(imm11_0, rs1, 6, rd, 0x13); 350} 351 352static inline u32 rv_xori(u8 rd, u8 rs1, u16 imm11_0) 353{ 354 return rv_i_insn(imm11_0, rs1, 4, rd, 0x13); 355} 356 357static inline u32 rv_slli(u8 rd, u8 rs1, u16 imm11_0) 358{ 359 return rv_i_insn(imm11_0, rs1, 1, rd, 0x13); 360} 361 362static inline u32 rv_srli(u8 rd, u8 rs1, u16 imm11_0) 363{ 364 return rv_i_insn(imm11_0, rs1, 5, rd, 0x13); 365} 366 367static inline u32 rv_srai(u8 rd, u8 rs1, u16 imm11_0) 368{ 369 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x13); 370} 371 372static inline u32 rv_lui(u8 rd, u32 imm31_12) 373{ 374 return rv_u_insn(imm31_12, rd, 0x37); 375} 376 377static inline u32 rv_auipc(u8 rd, u32 imm31_12) 378{ 379 return rv_u_insn(imm31_12, rd, 0x17); 380} 381 382static inline u32 rv_add(u8 rd, u8 rs1, u8 rs2) 383{ 384 return rv_r_insn(0, rs2, rs1, 0, rd, 0x33); 385} 386 387static inline u32 rv_sub(u8 rd, u8 rs1, u8 rs2) 388{ 389 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x33); 390} 391 392static inline u32 rv_sltu(u8 rd, u8 rs1, u8 rs2) 393{ 394 return rv_r_insn(0, rs2, rs1, 3, rd, 0x33); 395} 396 397static inline u32 rv_and(u8 rd, u8 rs1, u8 rs2) 398{ 399 return rv_r_insn(0, rs2, rs1, 7, rd, 0x33); 400} 401 402static inline u32 rv_or(u8 rd, u8 rs1, u8 rs2) 403{ 404 return rv_r_insn(0, rs2, rs1, 6, rd, 0x33); 405} 406 407static inline u32 rv_xor(u8 rd, u8 rs1, u8 rs2) 408{ 409 return rv_r_insn(0, rs2, rs1, 4, rd, 0x33); 410} 411 412static inline u32 rv_sll(u8 rd, u8 rs1, u8 rs2) 413{ 414 return rv_r_insn(0, rs2, rs1, 1, rd, 0x33); 415} 416 417static inline u32 rv_srl(u8 rd, u8 rs1, u8 rs2) 418{ 419 return rv_r_insn(0, rs2, rs1, 5, rd, 0x33); 420} 421 422static inline u32 rv_sra(u8 rd, u8 rs1, u8 rs2) 423{ 424 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x33); 425} 426 427static inline u32 rv_mul(u8 rd, u8 rs1, u8 rs2) 428{ 429 return rv_r_insn(1, rs2, rs1, 0, rd, 0x33); 430} 431 432static inline u32 rv_mulhu(u8 rd, u8 rs1, u8 rs2) 433{ 434 return rv_r_insn(1, rs2, rs1, 3, rd, 0x33); 435} 436 437static inline u32 rv_div(u8 rd, u8 rs1, u8 rs2) 438{ 439 return rv_r_insn(1, rs2, rs1, 4, rd, 0x33); 440} 441 442static inline u32 rv_divu(u8 rd, u8 rs1, u8 rs2) 443{ 444 return rv_r_insn(1, rs2, rs1, 5, rd, 0x33); 445} 446 447static inline u32 rv_rem(u8 rd, u8 rs1, u8 rs2) 448{ 449 return rv_r_insn(1, rs2, rs1, 6, rd, 0x33); 450} 451 452static inline u32 rv_remu(u8 rd, u8 rs1, u8 rs2) 453{ 454 return rv_r_insn(1, rs2, rs1, 7, rd, 0x33); 455} 456 457static inline u32 rv_jal(u8 rd, u32 imm20_1) 458{ 459 return rv_j_insn(imm20_1, rd, 0x6f); 460} 461 462static inline u32 rv_jalr(u8 rd, u8 rs1, u16 imm11_0) 463{ 464 return rv_i_insn(imm11_0, rs1, 0, rd, 0x67); 465} 466 467static inline u32 rv_beq(u8 rs1, u8 rs2, u16 imm12_1) 468{ 469 return rv_b_insn(imm12_1, rs2, rs1, 0, 0x63); 470} 471 472static inline u32 rv_bne(u8 rs1, u8 rs2, u16 imm12_1) 473{ 474 return rv_b_insn(imm12_1, rs2, rs1, 1, 0x63); 475} 476 477static inline u32 rv_bltu(u8 rs1, u8 rs2, u16 imm12_1) 478{ 479 return rv_b_insn(imm12_1, rs2, rs1, 6, 0x63); 480} 481 482static inline u32 rv_bgtu(u8 rs1, u8 rs2, u16 imm12_1) 483{ 484 return rv_bltu(rs2, rs1, imm12_1); 485} 486 487static inline u32 rv_bgeu(u8 rs1, u8 rs2, u16 imm12_1) 488{ 489 return rv_b_insn(imm12_1, rs2, rs1, 7, 0x63); 490} 491 492static inline u32 rv_bleu(u8 rs1, u8 rs2, u16 imm12_1) 493{ 494 return rv_bgeu(rs2, rs1, imm12_1); 495} 496 497static inline u32 rv_blt(u8 rs1, u8 rs2, u16 imm12_1) 498{ 499 return rv_b_insn(imm12_1, rs2, rs1, 4, 0x63); 500} 501 502static inline u32 rv_bgt(u8 rs1, u8 rs2, u16 imm12_1) 503{ 504 return rv_blt(rs2, rs1, imm12_1); 505} 506 507static inline u32 rv_bge(u8 rs1, u8 rs2, u16 imm12_1) 508{ 509 return rv_b_insn(imm12_1, rs2, rs1, 5, 0x63); 510} 511 512static inline u32 rv_ble(u8 rs1, u8 rs2, u16 imm12_1) 513{ 514 return rv_bge(rs2, rs1, imm12_1); 515} 516 517static inline u32 rv_lb(u8 rd, u16 imm11_0, u8 rs1) 518{ 519 return rv_i_insn(imm11_0, rs1, 0, rd, 0x03); 520} 521 522static inline u32 rv_lh(u8 rd, u16 imm11_0, u8 rs1) 523{ 524 return rv_i_insn(imm11_0, rs1, 1, rd, 0x03); 525} 526 527static inline u32 rv_lw(u8 rd, u16 imm11_0, u8 rs1) 528{ 529 return rv_i_insn(imm11_0, rs1, 2, rd, 0x03); 530} 531 532static inline u32 rv_lbu(u8 rd, u16 imm11_0, u8 rs1) 533{ 534 return rv_i_insn(imm11_0, rs1, 4, rd, 0x03); 535} 536 537static inline u32 rv_lhu(u8 rd, u16 imm11_0, u8 rs1) 538{ 539 return rv_i_insn(imm11_0, rs1, 5, rd, 0x03); 540} 541 542static inline u32 rv_sb(u8 rs1, u16 imm11_0, u8 rs2) 543{ 544 return rv_s_insn(imm11_0, rs2, rs1, 0, 0x23); 545} 546 547static inline u32 rv_sh(u8 rs1, u16 imm11_0, u8 rs2) 548{ 549 return rv_s_insn(imm11_0, rs2, rs1, 1, 0x23); 550} 551 552static inline u32 rv_sw(u8 rs1, u16 imm11_0, u8 rs2) 553{ 554 return rv_s_insn(imm11_0, rs2, rs1, 2, 0x23); 555} 556 557static inline u32 rv_amoadd_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 558{ 559 return rv_amo_insn(0, aq, rl, rs2, rs1, 2, rd, 0x2f); 560} 561 562static inline u32 rv_amoand_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 563{ 564 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 2, rd, 0x2f); 565} 566 567static inline u32 rv_amoor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 568{ 569 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 2, rd, 0x2f); 570} 571 572static inline u32 rv_amoxor_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 573{ 574 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 2, rd, 0x2f); 575} 576 577static inline u32 rv_amoswap_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 578{ 579 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 2, rd, 0x2f); 580} 581 582static inline u32 rv_lr_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 583{ 584 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 2, rd, 0x2f); 585} 586 587static inline u32 rv_sc_w(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 588{ 589 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 2, rd, 0x2f); 590} 591 592static inline u32 rv_fence(u8 pred, u8 succ) 593{ 594 u16 imm11_0 = pred << 4 | succ; 595 596 return rv_i_insn(imm11_0, 0, 0, 0, 0xf); 597} 598 599static inline u32 rv_nop(void) 600{ 601 return rv_i_insn(0, 0, 0, 0, 0x13); 602} 603 604/* RVC instrutions. */ 605 606static inline u16 rvc_addi4spn(u8 rd, u32 imm10) 607{ 608 u32 imm; 609 610 imm = ((imm10 & 0x30) << 2) | ((imm10 & 0x3c0) >> 4) | 611 ((imm10 & 0x4) >> 1) | ((imm10 & 0x8) >> 3); 612 return rv_ciw_insn(0x0, imm, rd, 0x0); 613} 614 615static inline u16 rvc_lw(u8 rd, u32 imm7, u8 rs1) 616{ 617 u32 imm_hi, imm_lo; 618 619 imm_hi = (imm7 & 0x38) >> 3; 620 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 621 return rv_cl_insn(0x2, imm_hi, rs1, imm_lo, rd, 0x0); 622} 623 624static inline u16 rvc_sw(u8 rs1, u32 imm7, u8 rs2) 625{ 626 u32 imm_hi, imm_lo; 627 628 imm_hi = (imm7 & 0x38) >> 3; 629 imm_lo = ((imm7 & 0x4) >> 1) | ((imm7 & 0x40) >> 6); 630 return rv_cs_insn(0x6, imm_hi, rs1, imm_lo, rs2, 0x0); 631} 632 633static inline u16 rvc_addi(u8 rd, u32 imm6) 634{ 635 return rv_ci_insn(0, imm6, rd, 0x1); 636} 637 638static inline u16 rvc_li(u8 rd, u32 imm6) 639{ 640 return rv_ci_insn(0x2, imm6, rd, 0x1); 641} 642 643static inline u16 rvc_addi16sp(u32 imm10) 644{ 645 u32 imm; 646 647 imm = ((imm10 & 0x200) >> 4) | (imm10 & 0x10) | ((imm10 & 0x40) >> 3) | 648 ((imm10 & 0x180) >> 6) | ((imm10 & 0x20) >> 5); 649 return rv_ci_insn(0x3, imm, RV_REG_SP, 0x1); 650} 651 652static inline u16 rvc_lui(u8 rd, u32 imm6) 653{ 654 return rv_ci_insn(0x3, imm6, rd, 0x1); 655} 656 657static inline u16 rvc_srli(u8 rd, u32 imm6) 658{ 659 return rv_cb_insn(0x4, imm6, 0, rd, 0x1); 660} 661 662static inline u16 rvc_srai(u8 rd, u32 imm6) 663{ 664 return rv_cb_insn(0x4, imm6, 0x1, rd, 0x1); 665} 666 667static inline u16 rvc_andi(u8 rd, u32 imm6) 668{ 669 return rv_cb_insn(0x4, imm6, 0x2, rd, 0x1); 670} 671 672static inline u16 rvc_sub(u8 rd, u8 rs) 673{ 674 return rv_ca_insn(0x23, rd, 0, rs, 0x1); 675} 676 677static inline u16 rvc_xor(u8 rd, u8 rs) 678{ 679 return rv_ca_insn(0x23, rd, 0x1, rs, 0x1); 680} 681 682static inline u16 rvc_or(u8 rd, u8 rs) 683{ 684 return rv_ca_insn(0x23, rd, 0x2, rs, 0x1); 685} 686 687static inline u16 rvc_and(u8 rd, u8 rs) 688{ 689 return rv_ca_insn(0x23, rd, 0x3, rs, 0x1); 690} 691 692static inline u16 rvc_slli(u8 rd, u32 imm6) 693{ 694 return rv_ci_insn(0, imm6, rd, 0x2); 695} 696 697static inline u16 rvc_lwsp(u8 rd, u32 imm8) 698{ 699 u32 imm; 700 701 imm = ((imm8 & 0xc0) >> 6) | (imm8 & 0x3c); 702 return rv_ci_insn(0x2, imm, rd, 0x2); 703} 704 705static inline u16 rvc_jr(u8 rs1) 706{ 707 return rv_cr_insn(0x8, rs1, RV_REG_ZERO, 0x2); 708} 709 710static inline u16 rvc_mv(u8 rd, u8 rs) 711{ 712 return rv_cr_insn(0x8, rd, rs, 0x2); 713} 714 715static inline u16 rvc_jalr(u8 rs1) 716{ 717 return rv_cr_insn(0x9, rs1, RV_REG_ZERO, 0x2); 718} 719 720static inline u16 rvc_add(u8 rd, u8 rs) 721{ 722 return rv_cr_insn(0x9, rd, rs, 0x2); 723} 724 725static inline u16 rvc_swsp(u32 imm8, u8 rs2) 726{ 727 u32 imm; 728 729 imm = (imm8 & 0x3c) | ((imm8 & 0xc0) >> 6); 730 return rv_css_insn(0x6, imm, rs2, 0x2); 731} 732 733/* 734 * RV64-only instructions. 735 * 736 * These instructions are not available on RV32. Wrap them below a #if to 737 * ensure that the RV32 JIT doesn't emit any of these instructions. 738 */ 739 740#if __riscv_xlen == 64 741 742static inline u32 rv_addiw(u8 rd, u8 rs1, u16 imm11_0) 743{ 744 return rv_i_insn(imm11_0, rs1, 0, rd, 0x1b); 745} 746 747static inline u32 rv_slliw(u8 rd, u8 rs1, u16 imm11_0) 748{ 749 return rv_i_insn(imm11_0, rs1, 1, rd, 0x1b); 750} 751 752static inline u32 rv_srliw(u8 rd, u8 rs1, u16 imm11_0) 753{ 754 return rv_i_insn(imm11_0, rs1, 5, rd, 0x1b); 755} 756 757static inline u32 rv_sraiw(u8 rd, u8 rs1, u16 imm11_0) 758{ 759 return rv_i_insn(0x400 | imm11_0, rs1, 5, rd, 0x1b); 760} 761 762static inline u32 rv_addw(u8 rd, u8 rs1, u8 rs2) 763{ 764 return rv_r_insn(0, rs2, rs1, 0, rd, 0x3b); 765} 766 767static inline u32 rv_subw(u8 rd, u8 rs1, u8 rs2) 768{ 769 return rv_r_insn(0x20, rs2, rs1, 0, rd, 0x3b); 770} 771 772static inline u32 rv_sllw(u8 rd, u8 rs1, u8 rs2) 773{ 774 return rv_r_insn(0, rs2, rs1, 1, rd, 0x3b); 775} 776 777static inline u32 rv_srlw(u8 rd, u8 rs1, u8 rs2) 778{ 779 return rv_r_insn(0, rs2, rs1, 5, rd, 0x3b); 780} 781 782static inline u32 rv_sraw(u8 rd, u8 rs1, u8 rs2) 783{ 784 return rv_r_insn(0x20, rs2, rs1, 5, rd, 0x3b); 785} 786 787static inline u32 rv_mulw(u8 rd, u8 rs1, u8 rs2) 788{ 789 return rv_r_insn(1, rs2, rs1, 0, rd, 0x3b); 790} 791 792static inline u32 rv_divw(u8 rd, u8 rs1, u8 rs2) 793{ 794 return rv_r_insn(1, rs2, rs1, 4, rd, 0x3b); 795} 796 797static inline u32 rv_divuw(u8 rd, u8 rs1, u8 rs2) 798{ 799 return rv_r_insn(1, rs2, rs1, 5, rd, 0x3b); 800} 801 802static inline u32 rv_remw(u8 rd, u8 rs1, u8 rs2) 803{ 804 return rv_r_insn(1, rs2, rs1, 6, rd, 0x3b); 805} 806 807static inline u32 rv_remuw(u8 rd, u8 rs1, u8 rs2) 808{ 809 return rv_r_insn(1, rs2, rs1, 7, rd, 0x3b); 810} 811 812static inline u32 rv_ld(u8 rd, u16 imm11_0, u8 rs1) 813{ 814 return rv_i_insn(imm11_0, rs1, 3, rd, 0x03); 815} 816 817static inline u32 rv_lwu(u8 rd, u16 imm11_0, u8 rs1) 818{ 819 return rv_i_insn(imm11_0, rs1, 6, rd, 0x03); 820} 821 822static inline u32 rv_sd(u8 rs1, u16 imm11_0, u8 rs2) 823{ 824 return rv_s_insn(imm11_0, rs2, rs1, 3, 0x23); 825} 826 827static inline u32 rv_amoadd_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 828{ 829 return rv_amo_insn(0, aq, rl, rs2, rs1, 3, rd, 0x2f); 830} 831 832static inline u32 rv_amoand_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 833{ 834 return rv_amo_insn(0xc, aq, rl, rs2, rs1, 3, rd, 0x2f); 835} 836 837static inline u32 rv_amoor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 838{ 839 return rv_amo_insn(0x8, aq, rl, rs2, rs1, 3, rd, 0x2f); 840} 841 842static inline u32 rv_amoxor_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 843{ 844 return rv_amo_insn(0x4, aq, rl, rs2, rs1, 3, rd, 0x2f); 845} 846 847static inline u32 rv_amoswap_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 848{ 849 return rv_amo_insn(0x1, aq, rl, rs2, rs1, 3, rd, 0x2f); 850} 851 852static inline u32 rv_lr_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 853{ 854 return rv_amo_insn(0x2, aq, rl, rs2, rs1, 3, rd, 0x2f); 855} 856 857static inline u32 rv_sc_d(u8 rd, u8 rs2, u8 rs1, u8 aq, u8 rl) 858{ 859 return rv_amo_insn(0x3, aq, rl, rs2, rs1, 3, rd, 0x2f); 860} 861 862/* RV64-only RVC instructions. */ 863 864static inline u16 rvc_ld(u8 rd, u32 imm8, u8 rs1) 865{ 866 u32 imm_hi, imm_lo; 867 868 imm_hi = (imm8 & 0x38) >> 3; 869 imm_lo = (imm8 & 0xc0) >> 6; 870 return rv_cl_insn(0x3, imm_hi, rs1, imm_lo, rd, 0x0); 871} 872 873static inline u16 rvc_sd(u8 rs1, u32 imm8, u8 rs2) 874{ 875 u32 imm_hi, imm_lo; 876 877 imm_hi = (imm8 & 0x38) >> 3; 878 imm_lo = (imm8 & 0xc0) >> 6; 879 return rv_cs_insn(0x7, imm_hi, rs1, imm_lo, rs2, 0x0); 880} 881 882static inline u16 rvc_subw(u8 rd, u8 rs) 883{ 884 return rv_ca_insn(0x27, rd, 0, rs, 0x1); 885} 886 887static inline u16 rvc_addiw(u8 rd, u32 imm6) 888{ 889 return rv_ci_insn(0x1, imm6, rd, 0x1); 890} 891 892static inline u16 rvc_ldsp(u8 rd, u32 imm9) 893{ 894 u32 imm; 895 896 imm = ((imm9 & 0x1c0) >> 6) | (imm9 & 0x38); 897 return rv_ci_insn(0x3, imm, rd, 0x2); 898} 899 900static inline u16 rvc_sdsp(u32 imm9, u8 rs2) 901{ 902 u32 imm; 903 904 imm = (imm9 & 0x38) | ((imm9 & 0x1c0) >> 6); 905 return rv_css_insn(0x7, imm, rs2, 0x2); 906} 907 908#endif /* __riscv_xlen == 64 */ 909 910/* Helper functions that emit RVC instructions when possible. */ 911 912static inline void emit_jalr(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 913{ 914 if (rvc_enabled() && rd == RV_REG_RA && rs && !imm) 915 emitc(rvc_jalr(rs), ctx); 916 else if (rvc_enabled() && !rd && rs && !imm) 917 emitc(rvc_jr(rs), ctx); 918 else 919 emit(rv_jalr(rd, rs, imm), ctx); 920} 921 922static inline void emit_mv(u8 rd, u8 rs, struct rv_jit_context *ctx) 923{ 924 if (rvc_enabled() && rd && rs) 925 emitc(rvc_mv(rd, rs), ctx); 926 else 927 emit(rv_addi(rd, rs, 0), ctx); 928} 929 930static inline void emit_add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 931{ 932 if (rvc_enabled() && rd && rd == rs1 && rs2) 933 emitc(rvc_add(rd, rs2), ctx); 934 else 935 emit(rv_add(rd, rs1, rs2), ctx); 936} 937 938static inline void emit_addi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 939{ 940 if (rvc_enabled() && rd == RV_REG_SP && rd == rs && is_10b_int(imm) && imm && !(imm & 0xf)) 941 emitc(rvc_addi16sp(imm), ctx); 942 else if (rvc_enabled() && is_creg(rd) && rs == RV_REG_SP && is_10b_uint(imm) && 943 !(imm & 0x3) && imm) 944 emitc(rvc_addi4spn(rd, imm), ctx); 945 else if (rvc_enabled() && rd && rd == rs && imm && is_6b_int(imm)) 946 emitc(rvc_addi(rd, imm), ctx); 947 else 948 emit(rv_addi(rd, rs, imm), ctx); 949} 950 951static inline void emit_li(u8 rd, s32 imm, struct rv_jit_context *ctx) 952{ 953 if (rvc_enabled() && rd && is_6b_int(imm)) 954 emitc(rvc_li(rd, imm), ctx); 955 else 956 emit(rv_addi(rd, RV_REG_ZERO, imm), ctx); 957} 958 959static inline void emit_lui(u8 rd, s32 imm, struct rv_jit_context *ctx) 960{ 961 if (rvc_enabled() && rd && rd != RV_REG_SP && is_6b_int(imm) && imm) 962 emitc(rvc_lui(rd, imm), ctx); 963 else 964 emit(rv_lui(rd, imm), ctx); 965} 966 967static inline void emit_slli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 968{ 969 if (rvc_enabled() && rd && rd == rs && imm && (u32)imm < __riscv_xlen) 970 emitc(rvc_slli(rd, imm), ctx); 971 else 972 emit(rv_slli(rd, rs, imm), ctx); 973} 974 975static inline void emit_andi(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 976{ 977 if (rvc_enabled() && is_creg(rd) && rd == rs && is_6b_int(imm)) 978 emitc(rvc_andi(rd, imm), ctx); 979 else 980 emit(rv_andi(rd, rs, imm), ctx); 981} 982 983static inline void emit_srli(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 984{ 985 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 986 emitc(rvc_srli(rd, imm), ctx); 987 else 988 emit(rv_srli(rd, rs, imm), ctx); 989} 990 991static inline void emit_srai(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 992{ 993 if (rvc_enabled() && is_creg(rd) && rd == rs && imm && (u32)imm < __riscv_xlen) 994 emitc(rvc_srai(rd, imm), ctx); 995 else 996 emit(rv_srai(rd, rs, imm), ctx); 997} 998 999static inline void emit_sub(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1000{ 1001 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1002 emitc(rvc_sub(rd, rs2), ctx); 1003 else 1004 emit(rv_sub(rd, rs1, rs2), ctx); 1005} 1006 1007static inline void emit_or(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1008{ 1009 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1010 emitc(rvc_or(rd, rs2), ctx); 1011 else 1012 emit(rv_or(rd, rs1, rs2), ctx); 1013} 1014 1015static inline void emit_and(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1016{ 1017 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1018 emitc(rvc_and(rd, rs2), ctx); 1019 else 1020 emit(rv_and(rd, rs1, rs2), ctx); 1021} 1022 1023static inline void emit_xor(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1024{ 1025 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1026 emitc(rvc_xor(rd, rs2), ctx); 1027 else 1028 emit(rv_xor(rd, rs1, rs2), ctx); 1029} 1030 1031static inline void emit_lw(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1032{ 1033 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_8b_uint(off) && !(off & 0x3)) 1034 emitc(rvc_lwsp(rd, off), ctx); 1035 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_7b_uint(off) && !(off & 0x3)) 1036 emitc(rvc_lw(rd, off, rs1), ctx); 1037 else 1038 emit(rv_lw(rd, off, rs1), ctx); 1039} 1040 1041static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1042{ 1043 if (rvc_enabled() && rs1 == RV_REG_SP && is_8b_uint(off) && !(off & 0x3)) 1044 emitc(rvc_swsp(off, rs2), ctx); 1045 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_7b_uint(off) && !(off & 0x3)) 1046 emitc(rvc_sw(rs1, off, rs2), ctx); 1047 else 1048 emit(rv_sw(rs1, off, rs2), ctx); 1049} 1050 1051/* RV64-only helper functions. */ 1052#if __riscv_xlen == 64 1053 1054static inline void emit_addiw(u8 rd, u8 rs, s32 imm, struct rv_jit_context *ctx) 1055{ 1056 if (rvc_enabled() && rd && rd == rs && is_6b_int(imm)) 1057 emitc(rvc_addiw(rd, imm), ctx); 1058 else 1059 emit(rv_addiw(rd, rs, imm), ctx); 1060} 1061 1062static inline void emit_ld(u8 rd, s32 off, u8 rs1, struct rv_jit_context *ctx) 1063{ 1064 if (rvc_enabled() && rs1 == RV_REG_SP && rd && is_9b_uint(off) && !(off & 0x7)) 1065 emitc(rvc_ldsp(rd, off), ctx); 1066 else if (rvc_enabled() && is_creg(rd) && is_creg(rs1) && is_8b_uint(off) && !(off & 0x7)) 1067 emitc(rvc_ld(rd, off, rs1), ctx); 1068 else 1069 emit(rv_ld(rd, off, rs1), ctx); 1070} 1071 1072static inline void emit_sd(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx) 1073{ 1074 if (rvc_enabled() && rs1 == RV_REG_SP && is_9b_uint(off) && !(off & 0x7)) 1075 emitc(rvc_sdsp(off, rs2), ctx); 1076 else if (rvc_enabled() && is_creg(rs1) && is_creg(rs2) && is_8b_uint(off) && !(off & 0x7)) 1077 emitc(rvc_sd(rs1, off, rs2), ctx); 1078 else 1079 emit(rv_sd(rs1, off, rs2), ctx); 1080} 1081 1082static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) 1083{ 1084 if (rvc_enabled() && is_creg(rd) && rd == rs1 && is_creg(rs2)) 1085 emitc(rvc_subw(rd, rs2), ctx); 1086 else 1087 emit(rv_subw(rd, rs1, rs2), ctx); 1088} 1089 1090#endif /* __riscv_xlen == 64 */ 1091 1092void bpf_jit_build_prologue(struct rv_jit_context *ctx); 1093void bpf_jit_build_epilogue(struct rv_jit_context *ctx); 1094 1095int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, 1096 bool extra_pass); 1097 1098#endif /* _BPF_JIT_H */ 1099