1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * BPF JIT compiler for LoongArch
4 *
5 * Copyright (C) 2021 Loongson Technology Corporation Limited
6 */
7 #include <linux/bpf.h>
8 #include <linux/filter.h>
9 #include <asm/cacheflush.h>
10 #include <asm/inst.h>
11
12 struct jit_ctx {
13 const struct bpf_prog *prog;
14 unsigned int idx;
15 unsigned int flags;
16 unsigned int epilogue_offset;
17 u32 *offset;
18 union loongarch_instruction *image;
19 u32 stack_size;
20 };
21
22 struct jit_data {
23 struct bpf_binary_header *header;
24 u8 *image;
25 struct jit_ctx ctx;
26 };
27
28 #define emit_insn(ctx, func, ...) \
29 do { \
30 if (ctx->image != NULL) { \
31 union loongarch_instruction *insn = &ctx->image[ctx->idx]; \
32 emit_##func(insn, ##__VA_ARGS__); \
33 } \
34 ctx->idx++; \
35 } while (0)
36
is_unsigned_imm(unsigned long val, unsigned int bit)37 static inline bool is_unsigned_imm(unsigned long val, unsigned int bit)
38 {
39 return val < (1UL << bit);
40 }
41
is_signed_imm(long val, unsigned int bit)42 static inline bool is_signed_imm(long val, unsigned int bit)
43 {
44 return -(1L << (bit - 1)) <= val && val < (1L << (bit - 1));
45 }
46
47 #define is_signed_imm12(val) is_signed_imm(val, 12)
48 #define is_signed_imm16(val) is_signed_imm(val, 16)
49 #define is_signed_imm26(val) is_signed_imm(val, 26)
50 #define is_signed_imm32(val) is_signed_imm(val, 32)
51 #define is_signed_imm52(val) is_signed_imm(val, 52)
52 #define is_unsigned_imm12(val) is_unsigned_imm(val, 12)
53 #define is_unsigned_imm32(val) is_unsigned_imm(val, 32)
54
bpf2la_offset(int bpf_insn, int off, const struct jit_ctx *ctx)55 static inline int bpf2la_offset(int bpf_insn, int off, const struct jit_ctx *ctx)
56 {
57 /* BPF JMP offset is relative to the next instruction */
58 bpf_insn++;
59 /*
60 * Whereas la64 branch instructions encode the offset
61 * from the branch itself, so we must subtract 1 from the
62 * instruction offset.
63 */
64 return (ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1));
65 }
66
epilogue_offset(const struct jit_ctx *ctx)67 static inline int epilogue_offset(const struct jit_ctx *ctx)
68 {
69 int to = ctx->epilogue_offset;
70 int from = ctx->idx;
71
72 return (to - from);
73 }
74
emit_ldbu(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)75 static inline void emit_ldbu(union loongarch_instruction *insn,
76 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
77 {
78 insn->reg2i12_format.opcode = ldbu_op;
79 insn->reg2i12_format.simmediate = imm;
80 insn->reg2i12_format.rd = rd;
81 insn->reg2i12_format.rj = rj;
82 }
83
emit_ldhu(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)84 static inline void emit_ldhu(union loongarch_instruction *insn,
85 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
86 {
87 insn->reg2i12_format.opcode = ldhu_op;
88 insn->reg2i12_format.simmediate = imm;
89 insn->reg2i12_format.rd = rd;
90 insn->reg2i12_format.rj = rj;
91 }
92
emit_ldwu(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)93 static inline void emit_ldwu(union loongarch_instruction *insn,
94 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
95 {
96 insn->reg2i12_format.opcode = ldwu_op;
97 insn->reg2i12_format.simmediate = imm;
98 insn->reg2i12_format.rd = rd;
99 insn->reg2i12_format.rj = rj;
100 }
101
emit_ldd(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)102 static inline void emit_ldd(union loongarch_instruction *insn,
103 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
104 {
105 insn->reg2i12_format.opcode = ldd_op;
106 insn->reg2i12_format.simmediate = imm;
107 insn->reg2i12_format.rd = rd;
108 insn->reg2i12_format.rj = rj;
109 }
110
emit_stb(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)111 static inline void emit_stb(union loongarch_instruction *insn,
112 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
113 {
114 insn->reg2i12_format.opcode = stb_op;
115 insn->reg2i12_format.simmediate = imm;
116 insn->reg2i12_format.rd = rd;
117 insn->reg2i12_format.rj = rj;
118 }
119
emit_sth(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)120 static inline void emit_sth(union loongarch_instruction *insn,
121 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
122 {
123 insn->reg2i12_format.opcode = sth_op;
124 insn->reg2i12_format.simmediate = imm;
125 insn->reg2i12_format.rd = rd;
126 insn->reg2i12_format.rj = rj;
127 }
128
emit_stw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)129 static inline void emit_stw(union loongarch_instruction *insn,
130 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
131 {
132 insn->reg2i12_format.opcode = stw_op;
133 insn->reg2i12_format.simmediate = imm;
134 insn->reg2i12_format.rd = rd;
135 insn->reg2i12_format.rj = rj;
136 }
137
emit_std(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)138 static inline void emit_std(union loongarch_instruction *insn,
139 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
140 {
141 insn->reg2i12_format.opcode = std_op;
142 insn->reg2i12_format.simmediate = imm;
143 insn->reg2i12_format.rd = rd;
144 insn->reg2i12_format.rj = rj;
145 }
146
emit_ldxbu(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)147 static inline void emit_ldxbu(union loongarch_instruction *insn, enum loongarch_gpr rd,
148 enum loongarch_gpr rj, enum loongarch_gpr rk)
149 {
150 insn->reg3_format.opcode = ldxbu_op;
151 insn->reg3_format.rd = rd;
152 insn->reg3_format.rj = rj;
153 insn->reg3_format.rk = rk;
154 }
155
emit_ldxhu(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)156 static inline void emit_ldxhu(union loongarch_instruction *insn, enum loongarch_gpr rd,
157 enum loongarch_gpr rj, enum loongarch_gpr rk)
158 {
159 insn->reg3_format.opcode = ldxhu_op;
160 insn->reg3_format.rd = rd;
161 insn->reg3_format.rj = rj;
162 insn->reg3_format.rk = rk;
163 }
164
emit_ldxwu(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)165 static inline void emit_ldxwu(union loongarch_instruction *insn, enum loongarch_gpr rd,
166 enum loongarch_gpr rj, enum loongarch_gpr rk)
167 {
168 insn->reg3_format.opcode = ldxwu_op;
169 insn->reg3_format.rd = rd;
170 insn->reg3_format.rj = rj;
171 insn->reg3_format.rk = rk;
172 }
173
emit_ldxd(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)174 static inline void emit_ldxd(union loongarch_instruction *insn, enum loongarch_gpr rd,
175 enum loongarch_gpr rj, enum loongarch_gpr rk)
176 {
177 insn->reg3_format.opcode = ldxd_op;
178 insn->reg3_format.rd = rd;
179 insn->reg3_format.rj = rj;
180 insn->reg3_format.rk = rk;
181 }
182
emit_stxb(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)183 static inline void emit_stxb(union loongarch_instruction *insn, enum loongarch_gpr rd,
184 enum loongarch_gpr rj, enum loongarch_gpr rk)
185 {
186 insn->reg3_format.opcode = stxb_op;
187 insn->reg3_format.rd = rd;
188 insn->reg3_format.rj = rj;
189 insn->reg3_format.rk = rk;
190 }
191
emit_stxh(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)192 static inline void emit_stxh(union loongarch_instruction *insn, enum loongarch_gpr rd,
193 enum loongarch_gpr rj, enum loongarch_gpr rk)
194 {
195 insn->reg3_format.opcode = stxh_op;
196 insn->reg3_format.rd = rd;
197 insn->reg3_format.rj = rj;
198 insn->reg3_format.rk = rk;
199 }
200
emit_stxw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)201 static inline void emit_stxw(union loongarch_instruction *insn, enum loongarch_gpr rd,
202 enum loongarch_gpr rj, enum loongarch_gpr rk)
203 {
204 insn->reg3_format.opcode = stxw_op;
205 insn->reg3_format.rd = rd;
206 insn->reg3_format.rj = rj;
207 insn->reg3_format.rk = rk;
208 }
209
emit_stxd(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)210 static inline void emit_stxd(union loongarch_instruction *insn, enum loongarch_gpr rd,
211 enum loongarch_gpr rj, enum loongarch_gpr rk)
212 {
213 insn->reg3_format.opcode = stxd_op;
214 insn->reg3_format.rd = rd;
215 insn->reg3_format.rj = rj;
216 insn->reg3_format.rk = rk;
217 }
218
emit_amaddw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rk, enum loongarch_gpr rj)219 static inline void emit_amaddw(union loongarch_instruction *insn, enum loongarch_gpr rd,
220 enum loongarch_gpr rk, enum loongarch_gpr rj)
221 {
222 insn->reg3_format.opcode = amaddw_op;
223 insn->reg3_format.rd = rd;
224 insn->reg3_format.rk = rk;
225 insn->reg3_format.rj = rj;
226 }
227
emit_amaddd(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rk, enum loongarch_gpr rj)228 static inline void emit_amaddd(union loongarch_instruction *insn, enum loongarch_gpr rd,
229 enum loongarch_gpr rk, enum loongarch_gpr rj)
230 {
231 insn->reg3_format.opcode = amaddd_op;
232 insn->reg3_format.rd = rd;
233 insn->reg3_format.rk = rk;
234 insn->reg3_format.rj = rj;
235 }
236
emit_addd(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)237 static inline void emit_addd(union loongarch_instruction *insn, enum loongarch_gpr rd,
238 enum loongarch_gpr rj, enum loongarch_gpr rk)
239 {
240 insn->reg3_format.opcode = addd_op;
241 insn->reg3_format.rd = rd;
242 insn->reg3_format.rj = rj;
243 insn->reg3_format.rk = rk;
244 }
245
emit_addiw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)246 static inline void emit_addiw(union loongarch_instruction *insn,
247 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
248 {
249 insn->reg2i12_format.opcode = addiw_op;
250 insn->reg2i12_format.simmediate = imm;
251 insn->reg2i12_format.rd = rd;
252 insn->reg2i12_format.rj = rj;
253 }
254
emit_addid(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)255 static inline void emit_addid(union loongarch_instruction *insn,
256 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
257 {
258 insn->reg2i12_format.opcode = addid_op;
259 insn->reg2i12_format.simmediate = imm;
260 insn->reg2i12_format.rd = rd;
261 insn->reg2i12_format.rj = rj;
262 }
263
emit_subd(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)264 static inline void emit_subd(union loongarch_instruction *insn, enum loongarch_gpr rd,
265 enum loongarch_gpr rj, enum loongarch_gpr rk)
266 {
267 insn->reg3_format.opcode = subd_op;
268 insn->reg3_format.rd = rd;
269 insn->reg3_format.rj = rj;
270 insn->reg3_format.rk = rk;
271 }
272
emit_muld(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)273 static inline void emit_muld(union loongarch_instruction *insn, enum loongarch_gpr rd,
274 enum loongarch_gpr rj, enum loongarch_gpr rk)
275 {
276 insn->reg3_format.opcode = muld_op;
277 insn->reg3_format.rd = rd;
278 insn->reg3_format.rj = rj;
279 insn->reg3_format.rk = rk;
280 }
281
emit_divdu(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)282 static inline void emit_divdu(union loongarch_instruction *insn, enum loongarch_gpr rd,
283 enum loongarch_gpr rj, enum loongarch_gpr rk)
284 {
285 insn->reg3_format.opcode = divdu_op;
286 insn->reg3_format.rd = rd;
287 insn->reg3_format.rj = rj;
288 insn->reg3_format.rk = rk;
289 }
290
emit_moddu(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)291 static inline void emit_moddu(union loongarch_instruction *insn, enum loongarch_gpr rd,
292 enum loongarch_gpr rj, enum loongarch_gpr rk)
293 {
294 insn->reg3_format.opcode = moddu_op;
295 insn->reg3_format.rd = rd;
296 insn->reg3_format.rj = rj;
297 insn->reg3_format.rk = rk;
298 }
299
emit_and(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)300 static inline void emit_and(union loongarch_instruction *insn, enum loongarch_gpr rd,
301 enum loongarch_gpr rj, enum loongarch_gpr rk)
302 {
303 insn->reg3_format.opcode = and_op;
304 insn->reg3_format.rd = rd;
305 insn->reg3_format.rj = rj;
306 insn->reg3_format.rk = rk;
307 }
308
emit_andi(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)309 static inline void emit_andi(union loongarch_instruction *insn,
310 enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)
311 {
312 insn->reg2ui12_format.opcode = andi_op;
313 insn->reg2ui12_format.simmediate = imm;
314 insn->reg2ui12_format.rd = rd;
315 insn->reg2ui12_format.rj = rj;
316 }
317
emit_or(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)318 static inline void emit_or(union loongarch_instruction *insn, enum loongarch_gpr rd,
319 enum loongarch_gpr rj, enum loongarch_gpr rk)
320 {
321 insn->reg3_format.opcode = or_op;
322 insn->reg3_format.rd = rd;
323 insn->reg3_format.rj = rj;
324 insn->reg3_format.rk = rk;
325 }
326
emit_ori(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)327 static inline void emit_ori(union loongarch_instruction *insn,
328 enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)
329 {
330 insn->reg2ui12_format.opcode = ori_op;
331 insn->reg2ui12_format.simmediate = imm;
332 insn->reg2ui12_format.rd = rd;
333 insn->reg2ui12_format.rj = rj;
334 }
335
emit_xor(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)336 static inline void emit_xor(union loongarch_instruction *insn, enum loongarch_gpr rd,
337 enum loongarch_gpr rj, enum loongarch_gpr rk)
338 {
339 insn->reg3_format.opcode = xor_op;
340 insn->reg3_format.rd = rd;
341 insn->reg3_format.rj = rj;
342 insn->reg3_format.rk = rk;
343 }
344
emit_xori(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)345 static inline void emit_xori(union loongarch_instruction *insn,
346 enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)
347 {
348 insn->reg2ui12_format.opcode = xori_op;
349 insn->reg2ui12_format.simmediate = imm;
350 insn->reg2ui12_format.rd = rd;
351 insn->reg2ui12_format.rj = rj;
352 }
353
emit_lu12iw(union loongarch_instruction *insn, enum loongarch_gpr rd, int imm)354 static inline void emit_lu12iw(union loongarch_instruction *insn,
355 enum loongarch_gpr rd, int imm)
356 {
357 insn->reg1i20_format.opcode = lu12iw_op;
358 insn->reg1i20_format.simmediate = imm;
359 insn->reg1i20_format.rd = rd;
360 }
361
emit_lu32id(union loongarch_instruction *insn, enum loongarch_gpr rd, int imm)362 static inline void emit_lu32id(union loongarch_instruction *insn,
363 enum loongarch_gpr rd, int imm)
364 {
365 insn->reg1i20_format.opcode = lu32id_op;
366 insn->reg1i20_format.simmediate = imm;
367 insn->reg1i20_format.rd = rd;
368 }
369
emit_lu52id(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)370 static inline void emit_lu52id(union loongarch_instruction *insn,
371 enum loongarch_gpr rd, enum loongarch_gpr rj, int imm)
372 {
373 insn->reg2i12_format.opcode = lu52id_op;
374 insn->reg2i12_format.simmediate = imm;
375 insn->reg2i12_format.rd = rd;
376 insn->reg2i12_format.rj = rj;
377 }
378
emit_sllw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)379 static inline void emit_sllw(union loongarch_instruction *insn, enum loongarch_gpr rd,
380 enum loongarch_gpr rj, enum loongarch_gpr rk)
381 {
382 insn->reg3_format.opcode = sllw_op;
383 insn->reg3_format.rd = rd;
384 insn->reg3_format.rj = rj;
385 insn->reg3_format.rk = rk;
386 }
387
emit_slliw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)388 static inline void emit_slliw(union loongarch_instruction *insn,
389 enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)
390 {
391 insn->reg2ui5_format.opcode = slliw_op;
392 insn->reg2ui5_format.simmediate = imm;
393 insn->reg2ui5_format.rd = rd;
394 insn->reg2ui5_format.rj = rj;
395 }
396
emit_slld(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)397 static inline void emit_slld(union loongarch_instruction *insn, enum loongarch_gpr rd,
398 enum loongarch_gpr rj, enum loongarch_gpr rk)
399 {
400 insn->reg3_format.opcode = slld_op;
401 insn->reg3_format.rd = rd;
402 insn->reg3_format.rj = rj;
403 insn->reg3_format.rk = rk;
404 }
405
emit_sllid(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)406 static inline void emit_sllid(union loongarch_instruction *insn,
407 enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)
408 {
409 insn->reg2ui6_format.opcode = sllid_op;
410 insn->reg2ui6_format.simmediate = imm;
411 insn->reg2ui6_format.rd = rd;
412 insn->reg2ui6_format.rj = rj;
413 }
414
emit_srlw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)415 static inline void emit_srlw(union loongarch_instruction *insn, enum loongarch_gpr rd,
416 enum loongarch_gpr rj, enum loongarch_gpr rk)
417 {
418 insn->reg3_format.opcode = srlw_op;
419 insn->reg3_format.rd = rd;
420 insn->reg3_format.rj = rj;
421 insn->reg3_format.rk = rk;
422 }
423
emit_srliw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)424 static inline void emit_srliw(union loongarch_instruction *insn,
425 enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)
426 {
427 insn->reg2ui5_format.opcode = srliw_op;
428 insn->reg2ui5_format.simmediate = imm;
429 insn->reg2ui5_format.rd = rd;
430 insn->reg2ui5_format.rj = rj;
431 }
432
emit_srld(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)433 static inline void emit_srld(union loongarch_instruction *insn, enum loongarch_gpr rd,
434 enum loongarch_gpr rj, enum loongarch_gpr rk)
435 {
436 insn->reg3_format.opcode = srld_op;
437 insn->reg3_format.rd = rd;
438 insn->reg3_format.rj = rj;
439 insn->reg3_format.rk = rk;
440 }
441
emit_srlid(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)442 static inline void emit_srlid(union loongarch_instruction *insn,
443 enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)
444 {
445 insn->reg2ui6_format.opcode = srlid_op;
446 insn->reg2ui6_format.simmediate = imm;
447 insn->reg2ui6_format.rd = rd;
448 insn->reg2ui6_format.rj = rj;
449 }
450
emit_sraw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)451 static inline void emit_sraw(union loongarch_instruction *insn, enum loongarch_gpr rd,
452 enum loongarch_gpr rj, enum loongarch_gpr rk)
453 {
454 insn->reg3_format.opcode = sraw_op;
455 insn->reg3_format.rd = rd;
456 insn->reg3_format.rj = rj;
457 insn->reg3_format.rk = rk;
458 }
459
emit_sraiw(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)460 static inline void emit_sraiw(union loongarch_instruction *insn,
461 enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)
462 {
463 insn->reg2ui5_format.opcode = sraid_op;
464 insn->reg2ui5_format.simmediate = imm;
465 insn->reg2ui5_format.rd = rd;
466 insn->reg2ui5_format.rj = rj;
467 }
468
emit_srad(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, enum loongarch_gpr rk)469 static inline void emit_srad(union loongarch_instruction *insn, enum loongarch_gpr rd,
470 enum loongarch_gpr rj, enum loongarch_gpr rk)
471 {
472 insn->reg3_format.opcode = srad_op;
473 insn->reg3_format.rd = rd;
474 insn->reg3_format.rj = rj;
475 insn->reg3_format.rk = rk;
476 }
477
emit_sraid(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)478 static inline void emit_sraid(union loongarch_instruction *insn,
479 enum loongarch_gpr rd, enum loongarch_gpr rj, u32 imm)
480 {
481 insn->reg2ui6_format.opcode = sraid_op;
482 insn->reg2ui6_format.simmediate = imm;
483 insn->reg2ui6_format.rd = rd;
484 insn->reg2ui6_format.rj = rj;
485 }
486
emit_beq(union loongarch_instruction *insn, enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)487 static inline void emit_beq(union loongarch_instruction *insn,
488 enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)
489 {
490 insn->reg2i16_format.opcode = beq_op;
491 insn->reg2i16_format.simmediate = offset;
492 insn->reg2i16_format.rj = rj;
493 insn->reg2i16_format.rd = rd;
494 }
495
emit_bne(union loongarch_instruction *insn, enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)496 static inline void emit_bne(union loongarch_instruction *insn,
497 enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)
498 {
499 insn->reg2i16_format.opcode = bne_op;
500 insn->reg2i16_format.simmediate = offset;
501 insn->reg2i16_format.rj = rj;
502 insn->reg2i16_format.rd = rd;
503 }
504
emit_blt(union loongarch_instruction *insn, enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)505 static inline void emit_blt(union loongarch_instruction *insn,
506 enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)
507 {
508 insn->reg2i16_format.opcode = blt_op;
509 insn->reg2i16_format.simmediate = offset;
510 insn->reg2i16_format.rj = rj;
511 insn->reg2i16_format.rd = rd;
512 }
513
emit_bge(union loongarch_instruction *insn, enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)514 static inline void emit_bge(union loongarch_instruction *insn,
515 enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)
516 {
517 insn->reg2i16_format.opcode = bge_op;
518 insn->reg2i16_format.simmediate = offset;
519 insn->reg2i16_format.rj = rj;
520 insn->reg2i16_format.rd = rd;
521 }
522
emit_bltu(union loongarch_instruction *insn, enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)523 static inline void emit_bltu(union loongarch_instruction *insn,
524 enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)
525 {
526 insn->reg2i16_format.opcode = bltu_op;
527 insn->reg2i16_format.simmediate = offset;
528 insn->reg2i16_format.rj = rj;
529 insn->reg2i16_format.rd = rd;
530 }
531
emit_bgeu(union loongarch_instruction *insn, enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)532 static inline void emit_bgeu(union loongarch_instruction *insn,
533 enum loongarch_gpr rj, enum loongarch_gpr rd, int offset)
534 {
535 insn->reg2i16_format.opcode = bgeu_op;
536 insn->reg2i16_format.simmediate = offset;
537 insn->reg2i16_format.rj = rj;
538 insn->reg2i16_format.rd = rd;
539 }
540
emit_b(union loongarch_instruction *insn, int offset)541 static inline void emit_b(union loongarch_instruction *insn, int offset)
542 {
543 unsigned int simmediate_l, simmediate_h;
544
545 simmediate_l = offset & 0xffff;
546 offset >>= 16;
547 simmediate_h = offset & 0x3ff;
548
549 insn->reg0i26_format.opcode = b_op;
550 insn->reg0i26_format.simmediate_l = simmediate_l;
551 insn->reg0i26_format.simmediate_h = simmediate_h;
552 }
553
emit_jirl(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj, int offset)554 static inline void emit_jirl(union loongarch_instruction *insn,
555 enum loongarch_gpr rd, enum loongarch_gpr rj, int offset)
556 {
557 insn->reg2i16_format.opcode = jirl_op;
558 insn->reg2i16_format.simmediate = offset;
559 insn->reg2i16_format.rd = rd;
560 insn->reg2i16_format.rj = rj;
561 }
562
emit_pcaddu18i(union loongarch_instruction *insn, enum loongarch_gpr rd, int imm)563 static inline void emit_pcaddu18i(union loongarch_instruction *insn,
564 enum loongarch_gpr rd, int imm)
565 {
566 insn->reg1i20_format.opcode = pcaddu18i_op;
567 insn->reg1i20_format.simmediate = imm;
568 insn->reg1i20_format.rd = rd;
569 }
570
emit_revb2h(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj)571 static inline void emit_revb2h(union loongarch_instruction *insn,
572 enum loongarch_gpr rd, enum loongarch_gpr rj)
573 {
574 insn->reg2_format.opcode = revb2h_op;
575 insn->reg2_format.rd = rd;
576 insn->reg2_format.rj = rj;
577 }
578
emit_revb2w(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj)579 static inline void emit_revb2w(union loongarch_instruction *insn,
580 enum loongarch_gpr rd, enum loongarch_gpr rj)
581 {
582 insn->reg2_format.opcode = revb2w_op;
583 insn->reg2_format.rd = rd;
584 insn->reg2_format.rj = rj;
585 }
586
emit_revbd(union loongarch_instruction *insn, enum loongarch_gpr rd, enum loongarch_gpr rj)587 static inline void emit_revbd(union loongarch_instruction *insn,
588 enum loongarch_gpr rd, enum loongarch_gpr rj)
589 {
590 insn->reg2_format.opcode = revbd_op;
591 insn->reg2_format.rd = rd;
592 insn->reg2_format.rj = rj;
593 }
594
595 /* Zero-extend 32 bits into 64 bits */
emit_zext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32)596 static inline void emit_zext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32)
597 {
598 if (!is32)
599 return;
600
601 /* Clear the upper 32 bits */
602 emit_insn(ctx, lu32id, reg, 0);
603 }
604
605 /* Signed-extend 32 bits into 64 bits */
emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg)606 static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg)
607 {
608 emit_insn(ctx, addiw, reg, reg, 0);
609 }
610
move_imm32(struct jit_ctx *ctx, enum loongarch_gpr rd, int imm32, bool is32)611 static inline void move_imm32(struct jit_ctx *ctx, enum loongarch_gpr rd,
612 int imm32, bool is32)
613 {
614 int si20;
615 u32 ui12;
616
617 /* or rd, $zero, $zero */
618 if (imm32 == 0) {
619 emit_insn(ctx, or, rd, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_ZERO);
620 return;
621 }
622
623 /* addiw rd, $zero, imm_11_0(signed) */
624 if (is_signed_imm12(imm32)) {
625 emit_insn(ctx, addiw, rd, LOONGARCH_GPR_ZERO, imm32);
626 goto zext;
627 }
628
629 /* ori rd, $zero, imm_11_0(unsigned) */
630 if (is_unsigned_imm12(imm32)) {
631 emit_insn(ctx, ori, rd, LOONGARCH_GPR_ZERO, imm32);
632 goto zext;
633 }
634
635 /* lu12iw rd, imm_31_12(signed) */
636 si20 = (imm32 >> 12) & 0xfffff;
637 emit_insn(ctx, lu12iw, rd, si20);
638
639 /* ori rd, rd, imm_11_0(unsigned) */
640 ui12 = imm32 & 0xfff;
641 if (ui12 != 0)
642 emit_insn(ctx, ori, rd, rd, ui12);
643
644 zext:
645 emit_zext_32(ctx, rd, is32);
646 }
647
move_imm64(struct jit_ctx *ctx, enum loongarch_gpr rd, long imm64, bool is32)648 static inline void move_imm64(struct jit_ctx *ctx, enum loongarch_gpr rd,
649 long imm64, bool is32)
650 {
651 int imm32, si20, si12;
652 long imm52;
653
654 si12 = (imm64 >> 52) & 0xfff;
655 imm52 = imm64 & 0xfffffffffffff;
656 /* lu52id rd, $zero, imm_63_52(signed) */
657 if (si12 != 0 && imm52 == 0) {
658 emit_insn(ctx, lu52id, rd, LOONGARCH_GPR_ZERO, si12);
659 return;
660 }
661
662 imm32 = imm64 & 0xffffffff;
663 move_imm32(ctx, rd, imm32, is32);
664
665 if (!is_signed_imm32(imm64)) {
666 if (imm52 != 0) {
667 /* lu32id rd, imm_51_32(signed) */
668 si20 = (imm64 >> 32) & 0xfffff;
669 emit_insn(ctx, lu32id, rd, si20);
670 }
671
672 /* lu52id rd, rd, imm_63_52(signed) */
673 if (!is_signed_imm52(imm64))
674 emit_insn(ctx, lu52id, rd, rd, si12);
675 }
676 }
677
move_reg(struct jit_ctx *ctx, enum loongarch_gpr rd, enum loongarch_gpr rj)678 static inline void move_reg(struct jit_ctx *ctx, enum loongarch_gpr rd,
679 enum loongarch_gpr rj)
680 {
681 emit_insn(ctx, or, rd, rj, LOONGARCH_GPR_ZERO);
682 }
683
invert_jump_cond(u8 cond)684 static inline int invert_jump_cond(u8 cond)
685 {
686 switch (cond) {
687 case BPF_JEQ:
688 return BPF_JNE;
689 case BPF_JNE:
690 case BPF_JSET:
691 return BPF_JEQ;
692 case BPF_JGT:
693 return BPF_JLE;
694 case BPF_JGE:
695 return BPF_JLT;
696 case BPF_JLT:
697 return BPF_JGE;
698 case BPF_JLE:
699 return BPF_JGT;
700 case BPF_JSGT:
701 return BPF_JSLE;
702 case BPF_JSGE:
703 return BPF_JSLT;
704 case BPF_JSLT:
705 return BPF_JSGE;
706 case BPF_JSLE:
707 return BPF_JSGT;
708 }
709 return -1;
710 }
711
cond_jump_offs16(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, enum loongarch_gpr rd, int jmp_offset)712 static inline void cond_jump_offs16(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
713 enum loongarch_gpr rd, int jmp_offset)
714 {
715 switch (cond) {
716 case BPF_JEQ:
717 /* PC += jmp_offset if rj == rd */
718 emit_insn(ctx, beq, rj, rd, jmp_offset);
719 return;
720 case BPF_JNE:
721 case BPF_JSET:
722 /* PC += jmp_offset if rj != rd */
723 emit_insn(ctx, bne, rj, rd, jmp_offset);
724 return;
725 case BPF_JGT:
726 /* PC += jmp_offset if rj > rd (unsigned) */
727 emit_insn(ctx, bltu, rd, rj, jmp_offset);
728 return;
729 case BPF_JLT:
730 /* PC += jmp_offset if rj < rd (unsigned) */
731 emit_insn(ctx, bltu, rj, rd, jmp_offset);
732 return;
733 case BPF_JGE:
734 /* PC += jmp_offset if rj >= rd (unsigned) */
735 emit_insn(ctx, bgeu, rj, rd, jmp_offset);
736 return;
737 case BPF_JLE:
738 /* PC += jmp_offset if rj <= rd (unsigned) */
739 emit_insn(ctx, bgeu, rd, rj, jmp_offset);
740 return;
741 case BPF_JSGT:
742 /* PC += jmp_offset if rj > rd (signed) */
743 emit_insn(ctx, blt, rd, rj, jmp_offset);
744 return;
745 case BPF_JSLT:
746 /* PC += jmp_offset if rj < rd (signed) */
747 emit_insn(ctx, blt, rj, rd, jmp_offset);
748 return;
749 case BPF_JSGE:
750 /* PC += jmp_offset if rj >= rd (signed) */
751 emit_insn(ctx, bge, rj, rd, jmp_offset);
752 return;
753 case BPF_JSLE:
754 /* PC += jmp_offset if rj <= rd (signed) */
755 emit_insn(ctx, bge, rd, rj, jmp_offset);
756 return;
757 }
758 }
759
cond_jump_offs26(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, enum loongarch_gpr rd, int jmp_offset)760 static inline void cond_jump_offs26(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
761 enum loongarch_gpr rd, int jmp_offset)
762 {
763 cond = invert_jump_cond(cond);
764 cond_jump_offs16(ctx, cond, rj, rd, 2);
765 emit_insn(ctx, b, jmp_offset);
766 }
767
cond_jump_offs32(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, enum loongarch_gpr rd, int jmp_offset)768 static inline void cond_jump_offs32(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
769 enum loongarch_gpr rd, int jmp_offset)
770 {
771 s64 upper, lower;
772
773 upper = (jmp_offset + (1 << 15)) >> 16;
774 lower = jmp_offset & 0xffff;
775
776 cond = invert_jump_cond(cond);
777 cond_jump_offs16(ctx, cond, rj, rd, 3);
778
779 /*
780 * jmp_addr = jmp_offset << 2
781 * tmp2 = PC + jmp_addr[31, 18] + 18'b0
782 */
783 emit_insn(ctx, pcaddu18i, LOONGARCH_GPR_T2, upper << 2);
784
785 /* jump to (tmp2 + jmp_addr[17, 2] + 2'b0) */
786 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T2, lower + 1);
787 }
788
uncond_jump_offs26(struct jit_ctx *ctx, int jmp_offset)789 static inline void uncond_jump_offs26(struct jit_ctx *ctx, int jmp_offset)
790 {
791 emit_insn(ctx, b, jmp_offset);
792 }
793
uncond_jump_offs32(struct jit_ctx *ctx, int jmp_offset, bool is_exit)794 static inline void uncond_jump_offs32(struct jit_ctx *ctx, int jmp_offset, bool is_exit)
795 {
796 s64 upper, lower;
797
798 upper = (jmp_offset + (1 << 15)) >> 16;
799 lower = jmp_offset & 0xffff;
800
801 if (is_exit)
802 lower -= 1;
803
804 /*
805 * jmp_addr = jmp_offset << 2;
806 * tmp1 = PC + jmp_addr[31, 18] + 18'b0
807 */
808 emit_insn(ctx, pcaddu18i, LOONGARCH_GPR_T1, upper << 2);
809
810 /* jump to (tmp1 + jmp_addr[17, 2] + 2'b0) */
811 emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T1, lower + 1);
812 }
813
emit_cond_jump(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, enum loongarch_gpr rd, int jmp_offset)814 static inline void emit_cond_jump(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
815 enum loongarch_gpr rd, int jmp_offset)
816 {
817 if (is_signed_imm16(jmp_offset))
818 cond_jump_offs16(ctx, cond, rj, rd, jmp_offset);
819 else if (is_signed_imm26(jmp_offset))
820 cond_jump_offs26(ctx, cond, rj, rd, jmp_offset);
821 else
822 cond_jump_offs32(ctx, cond, rj, rd, jmp_offset);
823 }
824
emit_uncond_jump(struct jit_ctx *ctx, int jmp_offset, bool is_exit)825 static inline void emit_uncond_jump(struct jit_ctx *ctx, int jmp_offset, bool is_exit)
826 {
827 if (is_signed_imm26(jmp_offset))
828 uncond_jump_offs26(ctx, jmp_offset);
829 else
830 uncond_jump_offs32(ctx, jmp_offset, is_exit);
831 }
832
emit_tailcall_jump(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, enum loongarch_gpr rd, int jmp_offset)833 static inline void emit_tailcall_jump(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj,
834 enum loongarch_gpr rd, int jmp_offset)
835 {
836 if (is_signed_imm16(jmp_offset))
837 cond_jump_offs16(ctx, cond, rj, rd, jmp_offset);
838 else if (is_signed_imm26(jmp_offset))
839 cond_jump_offs26(ctx, cond, rj, rd, jmp_offset - 1);
840 else
841 cond_jump_offs32(ctx, cond, rj, rd, jmp_offset - 2);
842 }
843