| /kernel/linux/linux-6.6/arch/x86/net/ |
| H A D | bpf_jit_comp.c | 258 static void push_callee_regs(u8 **pprog, bool *callee_regs_used) in push_callee_regs() argument
260 u8 *prog = *pprog; in push_callee_regs()
270 *pprog = prog; in push_callee_regs()
273 static void pop_callee_regs(u8 **pprog, bool *callee_regs_used) in pop_callee_regs() argument
275 u8 *prog = *pprog; in pop_callee_regs()
285 *pprog = prog; in pop_callee_regs()
293 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, in emit_prologue() argument
296 u8 *prog = *pprog; in emit_prologue()
321 *pprog = prog; in emit_prologue()
324 static int emit_patch(u8 **pprog, voi argument
339 emit_call(u8 **pprog, void *func, void *ip) emit_call() argument
344 emit_rsb_call(u8 **pprog, void *func, void *ip) emit_rsb_call() argument
351 emit_jump(u8 **pprog, void *func, void *ip) emit_jump() argument
419 emit_indirect_jump(u8 **pprog, int reg, u8 *ip) emit_indirect_jump() argument
441 emit_return(u8 **pprog, u8 *ip) emit_return() argument
470 emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, u32 stack_depth, u8 *ip, struct jit_context *ctx) emit_bpf_tail_call_indirect() argument
544 emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, u8 **pprog, u8 *ip, bool *callee_regs_used, u32 stack_depth, struct jit_context *ctx) emit_bpf_tail_call_direct() argument
624 emit_mov_imm32(u8 **pprog, bool sign_propagate, u32 dst_reg, const u32 imm32) emit_mov_imm32() argument
664 emit_mov_imm64(u8 **pprog, u32 dst_reg, const u32 imm32_hi, const u32 imm32_lo) emit_mov_imm64() argument
687 emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg) emit_mov_reg() argument
704 emit_movsx_reg(u8 **pprog, int num_bits, bool is64, u32 dst_reg, u32 src_reg) emit_movsx_reg() argument
737 emit_insn_suffix(u8 **pprog, u32 ptr_reg, u32 val_reg, int off) emit_insn_suffix() argument
759 maybe_emit_mod(u8 **pprog, u32 dst_reg, u32 src_reg, bool is64) maybe_emit_mod() argument
773 maybe_emit_1mod(u8 **pprog, u32 reg, bool is64) maybe_emit_1mod() argument
785 emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) emit_ldx() argument
815 emit_ldsx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) emit_ldsx() argument
838 emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) emit_stx() argument
871 emit_atomic(u8 **pprog, u8 atomic_op, u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size) emit_atomic() argument
941 emit_nops(u8 **pprog, int len) emit_nops() argument
971 emit_3vex(u8 **pprog, bool r, bool x, bool b, u8 m, bool w, u8 src_reg2, bool l, u8 pp) emit_3vex() argument
1008 emit_shiftx(u8 **pprog, u32 dst_reg, u8 src_reg, bool is64, u8 op) emit_shiftx() argument
1955 clean_stack_garbage(const struct btf_func_model *m, u8 **pprog, int nr_stack_slots, int stack_size) clean_stack_garbage() argument
2129 invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_link *l, int stack_size, int run_ctx_off, bool save_ret) invoke_bpf_prog() argument
2217 emit_align(u8 **pprog, u32 align) emit_align() argument
2228 emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) emit_cond_near_jump() argument
2243 invoke_bpf(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_links *tl, int stack_size, int run_ctx_off, bool save_ret) invoke_bpf() argument
2259 invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_links *tl, int stack_size, int run_ctx_off, u8 **branches) invoke_bpf_mod_ret() argument
2611 emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs, u8 *image, u8 *buf) emit_bpf_dispatcher() argument
[all...] |
| H A D | bpf_jit_comp32.c | 207 u8 **pprog) in emit_ia32_mov_i()
209 u8 *prog = *pprog; in emit_ia32_mov_i()
230 *pprog = prog; in emit_ia32_mov_i()
235 bool sstk, u8 **pprog) in emit_ia32_mov_r()
237 u8 *prog = *pprog; in emit_ia32_mov_r()
251 *pprog = prog; in emit_ia32_mov_r()
257 bool sstk, u8 **pprog, in emit_ia32_mov_r64()
260 emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog); in emit_ia32_mov_r64()
263 emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog); in emit_ia32_mov_r64()
266 emit_ia32_mov_i(dst_hi, 0, dstk, pprog); in emit_ia32_mov_r64()
206 emit_ia32_mov_i(const u8 dst, const u32 val, bool dstk, u8 **pprog) emit_ia32_mov_i() argument
234 emit_ia32_mov_r(const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_mov_r() argument
255 emit_ia32_mov_r64(const bool is64, const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_mov_r64() argument
270 emit_ia32_mov_i64(const bool is64, const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_mov_i64() argument
285 emit_ia32_mul_r(const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_mul_r() argument
317 emit_ia32_to_le_r64(const u8 dst[], s32 val, bool dstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_to_le_r64() argument
365 emit_ia32_to_be_r64(const u8 dst[], s32 val, bool dstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_to_be_r64() argument
435 emit_ia32_div_mod_r(const u8 op, const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_div_mod_r() argument
482 emit_ia32_shift_r(const u8 op, const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_shift_r() argument
523 emit_ia32_alu_r(const bool is64, const bool hi, const u8 op, const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_alu_r() argument
577 emit_ia32_alu_r64(const bool is64, const u8 op, const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_alu_r64() argument
597 emit_ia32_alu_i(const bool is64, const bool hi, const u8 op, const u8 dst, const s32 val, bool dstk, u8 **pprog) emit_ia32_alu_i() argument
677 emit_ia32_alu_i64(const bool is64, const u8 op, const u8 dst[], const u32 val, bool dstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_alu_i64() argument
698 emit_ia32_neg64(const u8 dst[], bool dstk, u8 **pprog) emit_ia32_neg64() argument
731 emit_ia32_lsh_r64(const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog) emit_ia32_lsh_r64() argument
784 emit_ia32_arsh_r64(const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog) emit_ia32_arsh_r64() argument
837 emit_ia32_rsh_r64(const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog) emit_ia32_rsh_r64() argument
890 emit_ia32_lsh_i64(const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_lsh_i64() argument
938 emit_ia32_rsh_i64(const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_rsh_i64() argument
987 emit_ia32_arsh_i64(const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_arsh_i64() argument
1035 emit_ia32_mul_r64(const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog) emit_ia32_mul_r64() argument
1112 emit_ia32_mul_i64(const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_mul_i64() argument
1200 emit_prologue(u8 **pprog, u32 stack_depth) emit_prologue() argument
1245 emit_epilogue(u8 **pprog, u32 stack_depth) emit_epilogue() argument
1271 emit_jmp_edx(u8 **pprog, u8 *ip) emit_jmp_edx() argument
1299 emit_bpf_tail_call(u8 **pprog, u8 *ip) emit_bpf_tail_call() argument
1391 emit_push_r64(const u8 src[], u8 **pprog) emit_push_r64() argument
1409 emit_push_r32(const u8 src[], u8 **pprog) emit_push_r32() argument
1569 emit_kfunc_call(const struct bpf_prog *bpf_prog, u8 *end_addr, const struct bpf_insn *insn, u8 **pprog) emit_kfunc_call() argument
[all...] |
| /kernel/linux/linux-5.10/arch/x86/net/ |
| H A D | bpf_jit_comp.c | 234 static void push_callee_regs(u8 **pprog, bool *callee_regs_used) in push_callee_regs() argument
236 u8 *prog = *pprog; in push_callee_regs()
247 *pprog = prog; in push_callee_regs()
250 static void pop_callee_regs(u8 **pprog, bool *callee_regs_used) in pop_callee_regs() argument
252 u8 *prog = *pprog; in pop_callee_regs()
263 *pprog = prog; in pop_callee_regs()
271 static void emit_prologue(u8 **pprog, u32 stack_depth, bool ebpf_from_cbpf, in emit_prologue() argument
274 u8 *prog = *pprog; in emit_prologue()
295 *pprog = prog; in emit_prologue()
298 static int emit_patch(u8 **pprog, voi argument
314 emit_call(u8 **pprog, void *func, void *ip) emit_call() argument
319 emit_jump(u8 **pprog, void *func, void *ip) emit_jump() argument
384 emit_indirect_jump(u8 **pprog, int reg, u8 *ip) emit_indirect_jump() argument
402 emit_return(u8 **pprog, u8 *ip) emit_return() argument
432 emit_bpf_tail_call_indirect(u8 **pprog, bool *callee_regs_used, u32 stack_depth, u8 *ip, struct jit_context *ctx) emit_bpf_tail_call_indirect() argument
506 emit_bpf_tail_call_direct(struct bpf_jit_poke_descriptor *poke, u8 **pprog, u8 *ip, bool *callee_regs_used, u32 stack_depth, struct jit_context *ctx) emit_bpf_tail_call_direct() argument
595 emit_mov_imm32(u8 **pprog, bool sign_propagate, u32 dst_reg, const u32 imm32) emit_mov_imm32() argument
636 emit_mov_imm64(u8 **pprog, u32 dst_reg, const u32 imm32_hi, const u32 imm32_lo) emit_mov_imm64() argument
660 emit_mov_reg(u8 **pprog, bool is64, u32 dst_reg, u32 src_reg) emit_mov_reg() argument
679 emit_ldx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) emit_ldx() argument
718 emit_stx(u8 **pprog, u32 size, u32 dst_reg, u32 src_reg, int off) emit_stx() argument
1543 invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog, struct bpf_prog *p, int stack_size, bool save_ret) invoke_bpf_prog() argument
1598 emit_nops(u8 **pprog, unsigned int len) emit_nops() argument
1618 emit_align(u8 **pprog, u32 align) emit_align() argument
1629 emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond) emit_cond_near_jump() argument
1645 invoke_bpf(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_progs *tp, int stack_size, bool save_ret) invoke_bpf() argument
1661 invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog, struct bpf_tramp_progs *tp, int stack_size, u8 **branches) invoke_bpf_mod_ret() argument
1916 emit_bpf_dispatcher(u8 **pprog, int a, int b, s64 *progs) emit_bpf_dispatcher() argument
[all...] |
| H A D | bpf_jit_comp32.c | 207 u8 **pprog) in emit_ia32_mov_i()
209 u8 *prog = *pprog; in emit_ia32_mov_i()
230 *pprog = prog; in emit_ia32_mov_i()
235 bool sstk, u8 **pprog) in emit_ia32_mov_r()
237 u8 *prog = *pprog; in emit_ia32_mov_r()
251 *pprog = prog; in emit_ia32_mov_r()
257 bool sstk, u8 **pprog, in emit_ia32_mov_r64()
260 emit_ia32_mov_r(dst_lo, src_lo, dstk, sstk, pprog); in emit_ia32_mov_r64()
263 emit_ia32_mov_r(dst_hi, src_hi, dstk, sstk, pprog); in emit_ia32_mov_r64()
266 emit_ia32_mov_i(dst_hi, 0, dstk, pprog); in emit_ia32_mov_r64()
206 emit_ia32_mov_i(const u8 dst, const u32 val, bool dstk, u8 **pprog) emit_ia32_mov_i() argument
234 emit_ia32_mov_r(const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_mov_r() argument
255 emit_ia32_mov_r64(const bool is64, const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_mov_r64() argument
270 emit_ia32_mov_i64(const bool is64, const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_mov_i64() argument
285 emit_ia32_mul_r(const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_mul_r() argument
317 emit_ia32_to_le_r64(const u8 dst[], s32 val, bool dstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_to_le_r64() argument
365 emit_ia32_to_be_r64(const u8 dst[], s32 val, bool dstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_to_be_r64() argument
435 emit_ia32_div_mod_r(const u8 op, const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_div_mod_r() argument
482 emit_ia32_shift_r(const u8 op, const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_shift_r() argument
523 emit_ia32_alu_r(const bool is64, const bool hi, const u8 op, const u8 dst, const u8 src, bool dstk, bool sstk, u8 **pprog) emit_ia32_alu_r() argument
577 emit_ia32_alu_r64(const bool is64, const u8 op, const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_alu_r64() argument
597 emit_ia32_alu_i(const bool is64, const bool hi, const u8 op, const u8 dst, const s32 val, bool dstk, u8 **pprog) emit_ia32_alu_i() argument
677 emit_ia32_alu_i64(const bool is64, const u8 op, const u8 dst[], const u32 val, bool dstk, u8 **pprog, const struct bpf_prog_aux *aux) emit_ia32_alu_i64() argument
698 emit_ia32_neg64(const u8 dst[], bool dstk, u8 **pprog) emit_ia32_neg64() argument
731 emit_ia32_lsh_r64(const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog) emit_ia32_lsh_r64() argument
784 emit_ia32_arsh_r64(const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog) emit_ia32_arsh_r64() argument
837 emit_ia32_rsh_r64(const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog) emit_ia32_rsh_r64() argument
890 emit_ia32_lsh_i64(const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_lsh_i64() argument
938 emit_ia32_rsh_i64(const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_rsh_i64() argument
987 emit_ia32_arsh_i64(const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_arsh_i64() argument
1035 emit_ia32_mul_r64(const u8 dst[], const u8 src[], bool dstk, bool sstk, u8 **pprog) emit_ia32_mul_r64() argument
1112 emit_ia32_mul_i64(const u8 dst[], const u32 val, bool dstk, u8 **pprog) emit_ia32_mul_i64() argument
1200 emit_prologue(u8 **pprog, u32 stack_depth) emit_prologue() argument
1245 emit_epilogue(u8 **pprog, u32 stack_depth) emit_epilogue() argument
1271 emit_jmp_edx(u8 **pprog, u8 *ip) emit_jmp_edx() argument
1299 emit_bpf_tail_call(u8 **pprog, u8 *ip) emit_bpf_tail_call() argument
1391 emit_push_r64(const u8 src[], u8 **pprog) emit_push_r64() argument
[all...] |
| /kernel/linux/linux-6.6/arch/x86/kernel/ |
| H A D | callthunks.c | 309 int x86_call_depth_emit_accounting(u8 **pprog, void *func) in x86_call_depth_emit_accounting() argument
321 memcpy(*pprog, tmpl, tmpl_size); in x86_call_depth_emit_accounting()
322 *pprog += tmpl_size; in x86_call_depth_emit_accounting()
|
| /kernel/linux/linux-5.10/include/linux/ |
| H A D | skmsg.h | 417 static inline void psock_set_prog(struct bpf_prog **pprog, in psock_set_prog() argument
420 prog = xchg(pprog, prog); in psock_set_prog()
425 static inline int psock_replace_prog(struct bpf_prog **pprog, in psock_replace_prog() argument
429 if (cmpxchg(pprog, old, prog) != old) in psock_replace_prog()
|
| /kernel/linux/linux-6.6/include/linux/ |
| H A D | skmsg.h | 465 static inline void psock_set_prog(struct bpf_prog **pprog, in psock_set_prog() argument
468 prog = xchg(pprog, prog); in psock_set_prog()
473 static inline int psock_replace_prog(struct bpf_prog **pprog, in psock_replace_prog() argument
477 if (cmpxchg(pprog, old, prog) != old) in psock_replace_prog()
|
| /kernel/linux/linux-6.6/net/core/ |
| H A D | sock_map.c | 1457 static int sock_map_prog_lookup(struct bpf_map *map, struct bpf_prog ***pprog, in sock_map_prog_lookup() argument
1467 *pprog = &progs->msg_parser; in sock_map_prog_lookup()
1471 *pprog = &progs->stream_parser; in sock_map_prog_lookup()
1477 *pprog = &progs->stream_verdict; in sock_map_prog_lookup()
1482 *pprog = &progs->skb_verdict; in sock_map_prog_lookup()
1494 struct bpf_prog **pprog; in sock_map_prog_update() local
1497 ret = sock_map_prog_lookup(map, &pprog, which); in sock_map_prog_update()
1502 return psock_replace_prog(pprog, prog, old); in sock_map_prog_update()
1504 psock_set_prog(pprog, prog); in sock_map_prog_update()
1513 struct bpf_prog **pprog; in sock_map_bpf_prog_query() local
[all...] |
| /kernel/linux/linux-6.6/arch/x86/include/asm/ |
| H A D | alternative.h | 116 extern int x86_call_depth_emit_accounting(u8 **pprog, void *func);
126 static __always_inline int x86_call_depth_emit_accounting(u8 **pprog, in x86_call_depth_emit_accounting() argument
|
| /kernel/linux/linux-5.10/net/core/ |
| H A D | sock_map.c | 1493 struct bpf_prog **pprog; in sock_map_prog_update() local
1500 pprog = &progs->msg_parser; in sock_map_prog_update()
1503 pprog = &progs->skb_parser; in sock_map_prog_update()
1506 pprog = &progs->skb_verdict; in sock_map_prog_update()
1513 return psock_replace_prog(pprog, prog, old); in sock_map_prog_update()
1515 psock_set_prog(pprog, prog); in sock_map_prog_update()
|