1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * eBPF JIT compiler 4 * 5 * Copyright 2016 Naveen N. Rao <naveen.n.rao@linux.vnet.ibm.com> 6 * IBM Corporation 7 * 8 * Based on the powerpc classic BPF JIT compiler by Matt Evans 9 */ 10#include <linux/moduleloader.h> 11#include <asm/cacheflush.h> 12#include <asm/asm-compat.h> 13#include <linux/netdevice.h> 14#include <linux/filter.h> 15#include <linux/if_vlan.h> 16#include <asm/kprobes.h> 17#include <linux/bpf.h> 18 19#include "bpf_jit.h" 20 21static void bpf_jit_fill_ill_insns(void *area, unsigned int size) 22{ 23 memset32(area, BREAKPOINT_INSTRUCTION, size / 4); 24} 25 26int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr) 27{ 28 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) { 29 PPC_JMP(exit_addr); 30 } else if (ctx->alt_exit_addr) { 31 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4)))) 32 return -1; 33 PPC_JMP(ctx->alt_exit_addr); 34 } else { 35 ctx->alt_exit_addr = ctx->idx * 4; 36 bpf_jit_build_epilogue(image, ctx); 37 } 38 39 return 0; 40} 41 42struct powerpc64_jit_data { 43 struct bpf_binary_header *header; 44 u32 *addrs; 45 u8 *image; 46 u32 proglen; 47 struct codegen_context ctx; 48}; 49 50bool bpf_jit_needs_zext(void) 51{ 52 return true; 53} 54 55struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) 56{ 57 u32 proglen; 58 u32 alloclen; 59 u8 *image = NULL; 60 u32 *code_base; 61 u32 *addrs; 62 struct powerpc64_jit_data *jit_data; 63 struct codegen_context cgctx; 64 int pass; 65 int flen; 66 struct bpf_binary_header *bpf_hdr; 67 struct bpf_prog *org_fp = fp; 68 struct bpf_prog *tmp_fp; 69 bool bpf_blinded = false; 70 bool extra_pass = false; 71 u32 extable_len; 72 u32 fixup_len; 73 74 if (!fp->jit_requested) 75 return org_fp; 76 77 tmp_fp = bpf_jit_blind_constants(org_fp); 78 if (IS_ERR(tmp_fp)) 79 return org_fp; 80 81 if (tmp_fp != org_fp) { 82 bpf_blinded = true; 83 fp = tmp_fp; 84 } 85 86 jit_data = fp->aux->jit_data; 87 if (!jit_data) { 88 jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); 89 if (!jit_data) { 90 fp = org_fp; 91 goto out; 92 } 93 fp->aux->jit_data = jit_data; 94 } 95 96 flen = fp->len; 97 addrs = jit_data->addrs; 98 if (addrs) { 99 cgctx = jit_data->ctx; 100 image = jit_data->image; 101 bpf_hdr = jit_data->header; 102 proglen = jit_data->proglen; 103 extra_pass = true; 104 /* During extra pass, ensure index is reset before repopulating extable entries */ 105 cgctx.exentry_idx = 0; 106 goto skip_init_ctx; 107 } 108 109 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); 110 if (addrs == NULL) { 111 fp = org_fp; 112 goto out_addrs; 113 } 114 115 memset(&cgctx, 0, sizeof(struct codegen_context)); 116 bpf_jit_init_reg_mapping(&cgctx); 117 118 /* Make sure that the stack is quadword aligned. */ 119 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); 120 121 /* Scouting faux-generate pass 0 */ 122 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0, false)) { 123 /* We hit something illegal or unsupported. */ 124 fp = org_fp; 125 goto out_addrs; 126 } 127 128 /* 129 * If we have seen a tail call, we need a second pass. 130 * This is because bpf_jit_emit_common_epilogue() is called 131 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. 132 * We also need a second pass if we ended up with too large 133 * a program so as to ensure BPF_EXIT branches are in range. 134 */ 135 if (cgctx.seen & SEEN_TAILCALL || !is_offset_in_branch_range((long)cgctx.idx * 4)) { 136 cgctx.idx = 0; 137 if (bpf_jit_build_body(fp, 0, &cgctx, addrs, 0, false)) { 138 fp = org_fp; 139 goto out_addrs; 140 } 141 } 142 143 bpf_jit_realloc_regs(&cgctx); 144 /* 145 * Pretend to build prologue, given the features we've seen. This will 146 * update ctgtx.idx as it pretends to output instructions, then we can 147 * calculate total size from idx. 148 */ 149 bpf_jit_build_prologue(0, &cgctx); 150 addrs[fp->len] = cgctx.idx * 4; 151 bpf_jit_build_epilogue(0, &cgctx); 152 153 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; 154 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); 155 156 proglen = cgctx.idx * 4; 157 alloclen = proglen + FUNCTION_DESCR_SIZE + fixup_len + extable_len; 158 159 bpf_hdr = bpf_jit_binary_alloc(alloclen, &image, 4, bpf_jit_fill_ill_insns); 160 if (!bpf_hdr) { 161 fp = org_fp; 162 goto out_addrs; 163 } 164 165 if (extable_len) 166 fp->aux->extable = (void *)image + FUNCTION_DESCR_SIZE + proglen + fixup_len; 167 168skip_init_ctx: 169 code_base = (u32 *)(image + FUNCTION_DESCR_SIZE); 170 171 /* Code generation passes 1-2 */ 172 for (pass = 1; pass < 3; pass++) { 173 /* Now build the prologue, body code & epilogue for real. */ 174 cgctx.idx = 0; 175 cgctx.alt_exit_addr = 0; 176 bpf_jit_build_prologue(code_base, &cgctx); 177 if (bpf_jit_build_body(fp, code_base, &cgctx, addrs, pass, extra_pass)) { 178 bpf_jit_binary_free(bpf_hdr); 179 fp = org_fp; 180 goto out_addrs; 181 } 182 bpf_jit_build_epilogue(code_base, &cgctx); 183 184 if (bpf_jit_enable > 1) 185 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, 186 proglen - (cgctx.idx * 4), cgctx.seen); 187 } 188 189 if (bpf_jit_enable > 1) 190 /* 191 * Note that we output the base address of the code_base 192 * rather than image, since opcodes are in code_base. 193 */ 194 bpf_jit_dump(flen, proglen, pass, code_base); 195 196#ifdef CONFIG_PPC64_ELF_ABI_V1 197 /* Function descriptor nastiness: Address + TOC */ 198 ((u64 *)image)[0] = (u64)code_base; 199 ((u64 *)image)[1] = local_paca->kernel_toc; 200#endif 201 202 fp->bpf_func = (void *)image; 203 fp->jited = 1; 204 fp->jited_len = proglen + FUNCTION_DESCR_SIZE; 205 206 bpf_flush_icache(bpf_hdr, (u8 *)bpf_hdr + bpf_hdr->size); 207 if (!fp->is_func || extra_pass) { 208 bpf_jit_binary_lock_ro(bpf_hdr); 209 bpf_prog_fill_jited_linfo(fp, addrs); 210out_addrs: 211 kfree(addrs); 212 kfree(jit_data); 213 fp->aux->jit_data = NULL; 214 } else { 215 jit_data->addrs = addrs; 216 jit_data->ctx = cgctx; 217 jit_data->proglen = proglen; 218 jit_data->image = image; 219 jit_data->header = bpf_hdr; 220 } 221 222out: 223 if (bpf_blinded) 224 bpf_jit_prog_release_other(fp, fp == org_fp ? tmp_fp : org_fp); 225 226 return fp; 227} 228 229/* 230 * The caller should check for (BPF_MODE(code) == BPF_PROBE_MEM) before calling 231 * this function, as this only applies to BPF_PROBE_MEM, for now. 232 */ 233int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, int pass, struct codegen_context *ctx, 234 int insn_idx, int jmp_off, int dst_reg) 235{ 236 off_t offset; 237 unsigned long pc; 238 struct exception_table_entry *ex; 239 u32 *fixup; 240 241 /* Populate extable entries only in the last pass */ 242 if (pass != 2) 243 return 0; 244 245 if (!fp->aux->extable || 246 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) 247 return -EINVAL; 248 249 pc = (unsigned long)&image[insn_idx]; 250 251 fixup = (void *)fp->aux->extable - 252 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + 253 (ctx->exentry_idx * BPF_FIXUP_LEN * 4); 254 255 fixup[0] = PPC_RAW_LI(dst_reg, 0); 256 if (IS_ENABLED(CONFIG_PPC32)) 257 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ 258 259 fixup[BPF_FIXUP_LEN - 1] = 260 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); 261 262 ex = &fp->aux->extable[ctx->exentry_idx]; 263 264 offset = pc - (long)&ex->insn; 265 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 266 return -ERANGE; 267 ex->insn = offset; 268 269 offset = (long)fixup - (long)&ex->fixup; 270 if (WARN_ON_ONCE(offset >= 0 || offset < INT_MIN)) 271 return -ERANGE; 272 ex->fixup = offset; 273 274 ctx->exentry_idx++; 275 return 0; 276} 277