1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2019 Facebook
3#include "vmlinux.h"
4#include <bpf/bpf_helpers.h>
5#include "runqslower.h"
6
7#define TASK_RUNNING 0
8#define BPF_F_CURRENT_CPU 0xffffffffULL
9
10const volatile __u64 min_us = 0;
11const volatile pid_t targ_pid = 0;
12
13struct {
14	__uint(type, BPF_MAP_TYPE_HASH);
15	__uint(max_entries, 10240);
16	__type(key, u32);
17	__type(value, u64);
18} start SEC(".maps");
19
20struct {
21	__uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY);
22	__uint(key_size, sizeof(u32));
23	__uint(value_size, sizeof(u32));
24} events SEC(".maps");
25
26/* record enqueue timestamp */
27__always_inline
28static int trace_enqueue(u32 tgid, u32 pid)
29{
30	u64 ts;
31
32	if (!pid || (targ_pid && targ_pid != pid))
33		return 0;
34
35	ts = bpf_ktime_get_ns();
36	bpf_map_update_elem(&start, &pid, &ts, 0);
37	return 0;
38}
39
40SEC("tp_btf/sched_wakeup")
41int handle__sched_wakeup(u64 *ctx)
42{
43	/* TP_PROTO(struct task_struct *p) */
44	struct task_struct *p = (void *)ctx[0];
45
46	return trace_enqueue(p->tgid, p->pid);
47}
48
49SEC("tp_btf/sched_wakeup_new")
50int handle__sched_wakeup_new(u64 *ctx)
51{
52	/* TP_PROTO(struct task_struct *p) */
53	struct task_struct *p = (void *)ctx[0];
54
55	return trace_enqueue(p->tgid, p->pid);
56}
57
58SEC("tp_btf/sched_switch")
59int handle__sched_switch(u64 *ctx)
60{
61	/* TP_PROTO(bool preempt, struct task_struct *prev,
62	 *	    struct task_struct *next)
63	 */
64	struct task_struct *prev = (struct task_struct *)ctx[1];
65	struct task_struct *next = (struct task_struct *)ctx[2];
66	struct event event = {};
67	u64 *tsp, delta_us;
68	long state;
69	u32 pid;
70
71	/* ivcsw: treat like an enqueue event and store timestamp */
72	if (prev->state == TASK_RUNNING)
73		trace_enqueue(prev->tgid, prev->pid);
74
75	pid = next->pid;
76
77	/* fetch timestamp and calculate delta */
78	tsp = bpf_map_lookup_elem(&start, &pid);
79	if (!tsp)
80		return 0;   /* missed enqueue */
81
82	delta_us = (bpf_ktime_get_ns() - *tsp) / 1000;
83	if (min_us && delta_us <= min_us)
84		return 0;
85
86	event.pid = pid;
87	event.delta_us = delta_us;
88	bpf_get_current_comm(&event.task, sizeof(event.task));
89
90	/* output */
91	bpf_perf_event_output(ctx, &events, BPF_F_CURRENT_CPU,
92			      &event, sizeof(event));
93
94	bpf_map_delete_elem(&start, &pid);
95	return 0;
96}
97
98char LICENSE[] SEC("license") = "GPL";
99