1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* bpf_jit.S: Packet/header access helper functions 3 * for PPC64 BPF compiler. 4 * 5 * Copyright 2011 Matt Evans <matt@ozlabs.org>, IBM Corporation 6 */ 7 8 #include <asm/ppc_asm.h> 9 #include <asm/asm-compat.h> 10 #include "bpf_jit32.h" 11 12 /* 13 * All of these routines are called directly from generated code, 14 * whose register usage is: 15 * 16 * r3 skb 17 * r4,r5 A,X 18 * r6 *** address parameter to helper *** 19 * r7-r10 scratch 20 * r14 skb->data 21 * r15 skb headlen 22 * r16-31 M[] 23 */ 24 25 /* 26 * To consider: These helpers are so small it could be better to just 27 * generate them inline. Inline code can do the simple headlen check 28 * then branch directly to slow_path_XXX if required. (In fact, could 29 * load a spare GPR with the address of slow_path_generic and pass size 30 * as an argument, making the call site a mtlr, li and bllr.) 31 */ 32 .globl sk_load_word 33 sk_load_word: 34 PPC_LCMPI r_addr, 0 35 blt bpf_slow_path_word_neg 36 .globl sk_load_word_positive_offset 37 sk_load_word_positive_offset: 38 /* Are we accessing past headlen? */ 39 subi r_scratch1, r_HL, 4 40 PPC_LCMP r_scratch1, r_addr 41 blt bpf_slow_path_word 42 /* Nope, just hitting the header. cr0 here is eq or gt! */ 43 #ifdef __LITTLE_ENDIAN__ 44 lwbrx r_A, r_D, r_addr 45 #else 46 lwzx r_A, r_D, r_addr 47 #endif 48 blr /* Return success, cr0 != LT */ 49 50 .globl sk_load_half 51 sk_load_half: 52 PPC_LCMPI r_addr, 0 53 blt bpf_slow_path_half_neg 54 .globl sk_load_half_positive_offset 55 sk_load_half_positive_offset: 56 subi r_scratch1, r_HL, 2 57 PPC_LCMP r_scratch1, r_addr 58 blt bpf_slow_path_half 59 #ifdef __LITTLE_ENDIAN__ 60 lhbrx r_A, r_D, r_addr 61 #else 62 lhzx r_A, r_D, r_addr 63 #endif 64 blr 65 66 .globl sk_load_byte 67 sk_load_byte: 68 PPC_LCMPI r_addr, 0 69 blt bpf_slow_path_byte_neg 70 .globl sk_load_byte_positive_offset 71 sk_load_byte_positive_offset: 72 PPC_LCMP r_HL, r_addr 73 ble bpf_slow_path_byte 74 lbzx r_A, r_D, r_addr 75 blr 76 77 /* 78 * BPF_LDX | BPF_B | BPF_MSH: ldxb 4*([offset]&0xf) 79 * r_addr is the offset value 80 */ 81 .globl sk_load_byte_msh 82 sk_load_byte_msh: 83 PPC_LCMPI r_addr, 0 84 blt bpf_slow_path_byte_msh_neg 85 .globl sk_load_byte_msh_positive_offset 86 sk_load_byte_msh_positive_offset: 87 PPC_LCMP r_HL, r_addr 88 ble bpf_slow_path_byte_msh 89 lbzx r_X, r_D, r_addr 90 rlwinm r_X, r_X, 2, 32-4-2, 31-2 91 blr 92 93 /* Call out to skb_copy_bits: 94 * We'll need to back up our volatile regs first; we have 95 * local variable space at r1+(BPF_PPC_STACK_BASIC). 96 * Allocate a new stack frame here to remain ABI-compliant in 97 * stashing LR. 98 */ 99 #define bpf_slow_path_common(SIZE) \ 100 mflr r0; \ 101 PPC_STL r0, PPC_LR_STKOFF(r1); \ 102 /* R3 goes in parameter space of caller's frame */ \ 103 PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ 104 PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ 105 PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ 106 addi r5, r1, BPF_PPC_STACK_BASIC+(2*REG_SZ); \ 107 PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ 108 /* R3 = r_skb, as passed */ \ 109 mr r4, r_addr; \ 110 li r6, SIZE; \ 111 bl skb_copy_bits; \ 112 nop; \ 113 /* R3 = 0 on success */ \ 114 addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ 115 PPC_LL r0, PPC_LR_STKOFF(r1); \ 116 PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ 117 PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ 118 mtlr r0; \ 119 PPC_LCMPI r3, 0; \ 120 blt bpf_error; /* cr0 = LT */ \ 121 PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ 122 /* Great success! */ 123 124 bpf_slow_path_word: 125 bpf_slow_path_common(4) 126 /* Data value is on stack, and cr0 != LT */ 127 lwz r_A, BPF_PPC_STACK_BASIC+(2*REG_SZ)(r1) 128 blr 129 130 bpf_slow_path_half: 131 bpf_slow_path_common(2) 132 lhz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) 133 blr 134 135 bpf_slow_path_byte: 136 bpf_slow_path_common(1) 137 lbz r_A, BPF_PPC_STACK_BASIC+(2*8)(r1) 138 blr 139 140 bpf_slow_path_byte_msh: 141 bpf_slow_path_common(1) 142 lbz r_X, BPF_PPC_STACK_BASIC+(2*8)(r1) 143 rlwinm r_X, r_X, 2, 32-4-2, 31-2 144 blr 145 146 /* Call out to bpf_internal_load_pointer_neg_helper: 147 * We'll need to back up our volatile regs first; we have 148 * local variable space at r1+(BPF_PPC_STACK_BASIC). 149 * Allocate a new stack frame here to remain ABI-compliant in 150 * stashing LR. 151 */ 152 #define sk_negative_common(SIZE) \ 153 mflr r0; \ 154 PPC_STL r0, PPC_LR_STKOFF(r1); \ 155 /* R3 goes in parameter space of caller's frame */ \ 156 PPC_STL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ 157 PPC_STL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ 158 PPC_STL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ 159 PPC_STLU r1, -BPF_PPC_SLOWPATH_FRAME(r1); \ 160 /* R3 = r_skb, as passed */ \ 161 mr r4, r_addr; \ 162 li r5, SIZE; \ 163 bl bpf_internal_load_pointer_neg_helper; \ 164 nop; \ 165 /* R3 != 0 on success */ \ 166 addi r1, r1, BPF_PPC_SLOWPATH_FRAME; \ 167 PPC_LL r0, PPC_LR_STKOFF(r1); \ 168 PPC_LL r_A, (BPF_PPC_STACK_BASIC+(0*REG_SZ))(r1); \ 169 PPC_LL r_X, (BPF_PPC_STACK_BASIC+(1*REG_SZ))(r1); \ 170 mtlr r0; \ 171 PPC_LCMPLI r3, 0; \ 172 beq bpf_error_slow; /* cr0 = EQ */ \ 173 mr r_addr, r3; \ 174 PPC_LL r_skb, (BPF_PPC_STACKFRAME+BPF_PPC_STACK_R3_OFF)(r1); \ 175 /* Great success! */ 176 177 bpf_slow_path_word_neg: 178 lis r_scratch1,-32 /* SKF_LL_OFF */ 179 PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ 180 blt bpf_error /* cr0 = LT */ 181 .globl sk_load_word_negative_offset 182 sk_load_word_negative_offset: 183 sk_negative_common(4) 184 lwz r_A, 0(r_addr) 185 blr 186 187 bpf_slow_path_half_neg: 188 lis r_scratch1,-32 /* SKF_LL_OFF */ 189 PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ 190 blt bpf_error /* cr0 = LT */ 191 .globl sk_load_half_negative_offset 192 sk_load_half_negative_offset: 193 sk_negative_common(2) 194 lhz r_A, 0(r_addr) 195 blr 196 197 bpf_slow_path_byte_neg: 198 lis r_scratch1,-32 /* SKF_LL_OFF */ 199 PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ 200 blt bpf_error /* cr0 = LT */ 201 .globl sk_load_byte_negative_offset 202 sk_load_byte_negative_offset: 203 sk_negative_common(1) 204 lbz r_A, 0(r_addr) 205 blr 206 207 bpf_slow_path_byte_msh_neg: 208 lis r_scratch1,-32 /* SKF_LL_OFF */ 209 PPC_LCMP r_addr, r_scratch1 /* addr < SKF_* */ 210 blt bpf_error /* cr0 = LT */ 211 .globl sk_load_byte_msh_negative_offset 212 sk_load_byte_msh_negative_offset: 213 sk_negative_common(1) 214 lbz r_X, 0(r_addr) 215 rlwinm r_X, r_X, 2, 32-4-2, 31-2 216 blr 217 218 bpf_error_slow: 219 /* fabricate a cr0 = lt */ 220 li r_scratch1, -1 221 PPC_LCMPI r_scratch1, 0 222 bpf_error: 223 /* Entered with cr0 = lt */ 224 li r3, 0 225 /* Generated code will 'blt epilogue', returning 0. */ 226 blr 227