1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #include <asm/ptrace.h> 3 4 #include "bpf_jit_32.h" 5 6 #define SAVE_SZ 96 7 #define SCRATCH_OFF 72 8 #define BE_PTR(label) be label 9 #define SIGN_EXTEND(reg) 10 11 #define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ 12 13 .text 14 .globl bpf_jit_load_word 15 bpf_jit_load_word: 16 cmp r_OFF, 0 17 bl bpf_slow_path_word_neg 18 nop 19 .globl bpf_jit_load_word_positive_offset 20 bpf_jit_load_word_positive_offset: 21 sub r_HEADLEN, r_OFF, r_TMP 22 cmp r_TMP, 3 23 ble bpf_slow_path_word 24 add r_SKB_DATA, r_OFF, r_TMP 25 andcc r_TMP, 3, %g0 26 bne load_word_unaligned 27 nop 28 retl 29 ld [r_TMP], r_A 30 load_word_unaligned: 31 ldub [r_TMP + 0x0], r_OFF 32 ldub [r_TMP + 0x1], r_TMP2 33 sll r_OFF, 8, r_OFF 34 or r_OFF, r_TMP2, r_OFF 35 ldub [r_TMP + 0x2], r_TMP2 36 sll r_OFF, 8, r_OFF 37 or r_OFF, r_TMP2, r_OFF 38 ldub [r_TMP + 0x3], r_TMP2 39 sll r_OFF, 8, r_OFF 40 retl 41 or r_OFF, r_TMP2, r_A 42 43 .globl bpf_jit_load_half 44 bpf_jit_load_half: 45 cmp r_OFF, 0 46 bl bpf_slow_path_half_neg 47 nop 48 .globl bpf_jit_load_half_positive_offset 49 bpf_jit_load_half_positive_offset: 50 sub r_HEADLEN, r_OFF, r_TMP 51 cmp r_TMP, 1 52 ble bpf_slow_path_half 53 add r_SKB_DATA, r_OFF, r_TMP 54 andcc r_TMP, 1, %g0 55 bne load_half_unaligned 56 nop 57 retl 58 lduh [r_TMP], r_A 59 load_half_unaligned: 60 ldub [r_TMP + 0x0], r_OFF 61 ldub [r_TMP + 0x1], r_TMP2 62 sll r_OFF, 8, r_OFF 63 retl 64 or r_OFF, r_TMP2, r_A 65 66 .globl bpf_jit_load_byte 67 bpf_jit_load_byte: 68 cmp r_OFF, 0 69 bl bpf_slow_path_byte_neg 70 nop 71 .globl bpf_jit_load_byte_positive_offset 72 bpf_jit_load_byte_positive_offset: 73 cmp r_OFF, r_HEADLEN 74 bge bpf_slow_path_byte 75 nop 76 retl 77 ldub [r_SKB_DATA + r_OFF], r_A 78 79 .globl bpf_jit_load_byte_msh 80 bpf_jit_load_byte_msh: 81 cmp r_OFF, 0 82 bl bpf_slow_path_byte_msh_neg 83 nop 84 .globl bpf_jit_load_byte_msh_positive_offset 85 bpf_jit_load_byte_msh_positive_offset: 86 cmp r_OFF, r_HEADLEN 87 bge bpf_slow_path_byte_msh 88 nop 89 ldub [r_SKB_DATA + r_OFF], r_OFF 90 and r_OFF, 0xf, r_OFF 91 retl 92 sll r_OFF, 2, r_X 93 94 #define bpf_slow_path_common(LEN) \ 95 save %sp, -SAVE_SZ, %sp; \ 96 mov %i0, %o0; \ 97 mov r_OFF, %o1; \ 98 add %fp, SCRATCH_OFF, %o2; \ 99 call skb_copy_bits; \ 100 mov (LEN), %o3; \ 101 cmp %o0, 0; \ 102 restore; 103 104 bpf_slow_path_word: 105 bpf_slow_path_common(4) 106 bl bpf_error 107 ld [%sp + SCRATCH_OFF], r_A 108 retl 109 nop 110 bpf_slow_path_half: 111 bpf_slow_path_common(2) 112 bl bpf_error 113 lduh [%sp + SCRATCH_OFF], r_A 114 retl 115 nop 116 bpf_slow_path_byte: 117 bpf_slow_path_common(1) 118 bl bpf_error 119 ldub [%sp + SCRATCH_OFF], r_A 120 retl 121 nop 122 bpf_slow_path_byte_msh: 123 bpf_slow_path_common(1) 124 bl bpf_error 125 ldub [%sp + SCRATCH_OFF], r_A 126 and r_OFF, 0xf, r_OFF 127 retl 128 sll r_OFF, 2, r_X 129 130 #define bpf_negative_common(LEN) \ 131 save %sp, -SAVE_SZ, %sp; \ 132 mov %i0, %o0; \ 133 mov r_OFF, %o1; \ 134 SIGN_EXTEND(%o1); \ 135 call bpf_internal_load_pointer_neg_helper; \ 136 mov (LEN), %o2; \ 137 mov %o0, r_TMP; \ 138 cmp %o0, 0; \ 139 BE_PTR(bpf_error); \ 140 restore; 141 142 bpf_slow_path_word_neg: 143 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 144 cmp r_OFF, r_TMP 145 bl bpf_error 146 nop 147 .globl bpf_jit_load_word_negative_offset 148 bpf_jit_load_word_negative_offset: 149 bpf_negative_common(4) 150 andcc r_TMP, 3, %g0 151 bne load_word_unaligned 152 nop 153 retl 154 ld [r_TMP], r_A 155 156 bpf_slow_path_half_neg: 157 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 158 cmp r_OFF, r_TMP 159 bl bpf_error 160 nop 161 .globl bpf_jit_load_half_negative_offset 162 bpf_jit_load_half_negative_offset: 163 bpf_negative_common(2) 164 andcc r_TMP, 1, %g0 165 bne load_half_unaligned 166 nop 167 retl 168 lduh [r_TMP], r_A 169 170 bpf_slow_path_byte_neg: 171 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 172 cmp r_OFF, r_TMP 173 bl bpf_error 174 nop 175 .globl bpf_jit_load_byte_negative_offset 176 bpf_jit_load_byte_negative_offset: 177 bpf_negative_common(1) 178 retl 179 ldub [r_TMP], r_A 180 181 bpf_slow_path_byte_msh_neg: 182 sethi %hi(SKF_MAX_NEG_OFF), r_TMP 183 cmp r_OFF, r_TMP 184 bl bpf_error 185 nop 186 .globl bpf_jit_load_byte_msh_negative_offset 187 bpf_jit_load_byte_msh_negative_offset: 188 bpf_negative_common(1) 189 ldub [r_TMP], r_OFF 190 and r_OFF, 0xf, r_OFF 191 retl 192 sll r_OFF, 2, r_X 193 194 bpf_error: 195 /* Make the JIT program return zero. The JIT epilogue 196 * stores away the original %o7 into r_saved_O7. The 197 * normal leaf function return is to use "retl" which 198 * would evalute to "jmpl %o7 + 8, %g0" but we want to 199 * use the saved value thus the sequence you see here. 200 */ 201 jmpl r_saved_O7 + 8, %g0 202 clr %o0 203