1/* SPDX-License-Identifier: GPL-2.0
2 *  Copyright(c) 2017-2018 Jesper Dangaard Brouer, Red Hat Inc.
3 *
4 * XDP monitor tool, based on tracepoints
5 */
6#include <uapi/linux/bpf.h>
7#include <bpf/bpf_helpers.h>
8
9struct {
10	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
11	__type(key, u32);
12	__type(value, u64);
13	__uint(max_entries, 2);
14	/* TODO: have entries for all possible errno's */
15} redirect_err_cnt SEC(".maps");
16
17#define XDP_UNKNOWN	XDP_REDIRECT + 1
18struct {
19	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
20	__type(key, u32);
21	__type(value, u64);
22	__uint(max_entries, XDP_UNKNOWN + 1);
23} exception_cnt SEC(".maps");
24
25/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_redirect/format
26 * Code in:                kernel/include/trace/events/xdp.h
27 */
28struct xdp_redirect_ctx {
29	u64 __pad;		// First 8 bytes are not accessible by bpf code
30	int prog_id;		//	offset:8;  size:4; signed:1;
31	u32 act;		//	offset:12  size:4; signed:0;
32	int ifindex;		//	offset:16  size:4; signed:1;
33	int err;		//	offset:20  size:4; signed:1;
34	int to_ifindex;		//	offset:24  size:4; signed:1;
35	u32 map_id;		//	offset:28  size:4; signed:0;
36	int map_index;		//	offset:32  size:4; signed:1;
37};				//	offset:36
38
39enum {
40	XDP_REDIRECT_SUCCESS = 0,
41	XDP_REDIRECT_ERROR = 1
42};
43
44static __always_inline
45int xdp_redirect_collect_stat(struct xdp_redirect_ctx *ctx)
46{
47	u32 key = XDP_REDIRECT_ERROR;
48	int err = ctx->err;
49	u64 *cnt;
50
51	if (!err)
52		key = XDP_REDIRECT_SUCCESS;
53
54	cnt  = bpf_map_lookup_elem(&redirect_err_cnt, &key);
55	if (!cnt)
56		return 1;
57	*cnt += 1;
58
59	return 0; /* Indicate event was filtered (no further processing)*/
60	/*
61	 * Returning 1 here would allow e.g. a perf-record tracepoint
62	 * to see and record these events, but it doesn't work well
63	 * in-practice as stopping perf-record also unload this
64	 * bpf_prog.  Plus, there is additional overhead of doing so.
65	 */
66}
67
68SEC("tracepoint/xdp/xdp_redirect_err")
69int trace_xdp_redirect_err(struct xdp_redirect_ctx *ctx)
70{
71	return xdp_redirect_collect_stat(ctx);
72}
73
74
75SEC("tracepoint/xdp/xdp_redirect_map_err")
76int trace_xdp_redirect_map_err(struct xdp_redirect_ctx *ctx)
77{
78	return xdp_redirect_collect_stat(ctx);
79}
80
81/* Likely unloaded when prog starts */
82SEC("tracepoint/xdp/xdp_redirect")
83int trace_xdp_redirect(struct xdp_redirect_ctx *ctx)
84{
85	return xdp_redirect_collect_stat(ctx);
86}
87
88/* Likely unloaded when prog starts */
89SEC("tracepoint/xdp/xdp_redirect_map")
90int trace_xdp_redirect_map(struct xdp_redirect_ctx *ctx)
91{
92	return xdp_redirect_collect_stat(ctx);
93}
94
95/* Tracepoint format: /sys/kernel/debug/tracing/events/xdp/xdp_exception/format
96 * Code in:                kernel/include/trace/events/xdp.h
97 */
98struct xdp_exception_ctx {
99	u64 __pad;	// First 8 bytes are not accessible by bpf code
100	int prog_id;	//	offset:8;  size:4; signed:1;
101	u32 act;	//	offset:12; size:4; signed:0;
102	int ifindex;	//	offset:16; size:4; signed:1;
103};
104
105SEC("tracepoint/xdp/xdp_exception")
106int trace_xdp_exception(struct xdp_exception_ctx *ctx)
107{
108	u64 *cnt;
109	u32 key;
110
111	key = ctx->act;
112	if (key > XDP_REDIRECT)
113		key = XDP_UNKNOWN;
114
115	cnt = bpf_map_lookup_elem(&exception_cnt, &key);
116	if (!cnt)
117		return 1;
118	*cnt += 1;
119
120	return 0;
121}
122
123/* Common stats data record shared with _user.c */
124struct datarec {
125	u64 processed;
126	u64 dropped;
127	u64 info;
128	u64 err;
129};
130#define MAX_CPUS 64
131
132struct {
133	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
134	__type(key, u32);
135	__type(value, struct datarec);
136	__uint(max_entries, MAX_CPUS);
137} cpumap_enqueue_cnt SEC(".maps");
138
139struct {
140	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
141	__type(key, u32);
142	__type(value, struct datarec);
143	__uint(max_entries, 1);
144} cpumap_kthread_cnt SEC(".maps");
145
146/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_enqueue/format
147 * Code in:         kernel/include/trace/events/xdp.h
148 */
149struct cpumap_enqueue_ctx {
150	u64 __pad;		// First 8 bytes are not accessible by bpf code
151	int map_id;		//	offset:8;  size:4; signed:1;
152	u32 act;		//	offset:12; size:4; signed:0;
153	int cpu;		//	offset:16; size:4; signed:1;
154	unsigned int drops;	//	offset:20; size:4; signed:0;
155	unsigned int processed;	//	offset:24; size:4; signed:0;
156	int to_cpu;		//	offset:28; size:4; signed:1;
157};
158
159SEC("tracepoint/xdp/xdp_cpumap_enqueue")
160int trace_xdp_cpumap_enqueue(struct cpumap_enqueue_ctx *ctx)
161{
162	u32 to_cpu = ctx->to_cpu;
163	struct datarec *rec;
164
165	if (to_cpu >= MAX_CPUS)
166		return 1;
167
168	rec = bpf_map_lookup_elem(&cpumap_enqueue_cnt, &to_cpu);
169	if (!rec)
170		return 0;
171	rec->processed += ctx->processed;
172	rec->dropped   += ctx->drops;
173
174	/* Record bulk events, then userspace can calc average bulk size */
175	if (ctx->processed > 0)
176		rec->info += 1;
177
178	return 0;
179}
180
181/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_cpumap_kthread/format
182 * Code in:         kernel/include/trace/events/xdp.h
183 */
184struct cpumap_kthread_ctx {
185	u64 __pad;		// First 8 bytes are not accessible by bpf code
186	int map_id;		//	offset:8;  size:4; signed:1;
187	u32 act;		//	offset:12; size:4; signed:0;
188	int cpu;		//	offset:16; size:4; signed:1;
189	unsigned int drops;	//	offset:20; size:4; signed:0;
190	unsigned int processed;	//	offset:24; size:4; signed:0;
191	int sched;		//	offset:28; size:4; signed:1;
192};
193
194SEC("tracepoint/xdp/xdp_cpumap_kthread")
195int trace_xdp_cpumap_kthread(struct cpumap_kthread_ctx *ctx)
196{
197	struct datarec *rec;
198	u32 key = 0;
199
200	rec = bpf_map_lookup_elem(&cpumap_kthread_cnt, &key);
201	if (!rec)
202		return 0;
203	rec->processed += ctx->processed;
204	rec->dropped   += ctx->drops;
205
206	/* Count times kthread yielded CPU via schedule call */
207	if (ctx->sched)
208		rec->info++;
209
210	return 0;
211}
212
213struct {
214	__uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
215	__type(key, u32);
216	__type(value, struct datarec);
217	__uint(max_entries, 1);
218} devmap_xmit_cnt SEC(".maps");
219
220/* Tracepoint: /sys/kernel/debug/tracing/events/xdp/xdp_devmap_xmit/format
221 * Code in:         kernel/include/trace/events/xdp.h
222 */
223struct devmap_xmit_ctx {
224	u64 __pad;		// First 8 bytes are not accessible by bpf code
225	int from_ifindex;	//	offset:8;  size:4; signed:1;
226	u32 act;		//	offset:12; size:4; signed:0;
227	int to_ifindex; 	//	offset:16; size:4; signed:1;
228	int drops;		//	offset:20; size:4; signed:1;
229	int sent;		//	offset:24; size:4; signed:1;
230	int err;		//	offset:28; size:4; signed:1;
231};
232
233SEC("tracepoint/xdp/xdp_devmap_xmit")
234int trace_xdp_devmap_xmit(struct devmap_xmit_ctx *ctx)
235{
236	struct datarec *rec;
237	u32 key = 0;
238
239	rec = bpf_map_lookup_elem(&devmap_xmit_cnt, &key);
240	if (!rec)
241		return 0;
242	rec->processed += ctx->sent;
243	rec->dropped   += ctx->drops;
244
245	/* Record bulk events, then userspace can calc average bulk size */
246	rec->info += 1;
247
248	/* Record error cases, where no frame were sent */
249	if (ctx->err)
250		rec->err++;
251
252	/* Catch API error of drv ndo_xdp_xmit sent more than count */
253	if (ctx->drops < 0)
254		rec->err++;
255
256	return 1;
257}
258