1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * KVM nVHE hypervisor stack tracing support.
4 *
5 * The unwinder implementation depends on the nVHE mode:
6 *
7 *   1) Non-protected nVHE mode - the host can directly access the
8 *      HYP stack pages and unwind the HYP stack in EL1. This saves having
9 *      to allocate shared buffers for the host to read the unwinded
10 *      stacktrace.
11 *
12 *   2) pKVM (protected nVHE) mode - the host cannot directly access
13 *      the HYP memory. The stack is unwinded in EL2 and dumped to a shared
14 *      buffer where the host can read and print the stacktrace.
15 *
16 * Copyright (C) 2022 Google LLC
17 */
18
19#include <linux/kvm.h>
20#include <linux/kvm_host.h>
21
22#include <asm/stacktrace/nvhe.h>
23
24static struct stack_info stackinfo_get_overflow(void)
25{
26	struct kvm_nvhe_stacktrace_info *stacktrace_info
27				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
28	unsigned long low = (unsigned long)stacktrace_info->overflow_stack_base;
29	unsigned long high = low + OVERFLOW_STACK_SIZE;
30
31	return (struct stack_info) {
32		.low = low,
33		.high = high,
34	};
35}
36
37static struct stack_info stackinfo_get_overflow_kern_va(void)
38{
39	unsigned long low = (unsigned long)this_cpu_ptr_nvhe_sym(overflow_stack);
40	unsigned long high = low + OVERFLOW_STACK_SIZE;
41
42	return (struct stack_info) {
43		.low = low,
44		.high = high,
45	};
46}
47
48static struct stack_info stackinfo_get_hyp(void)
49{
50	struct kvm_nvhe_stacktrace_info *stacktrace_info
51				= this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
52	unsigned long low = (unsigned long)stacktrace_info->stack_base;
53	unsigned long high = low + PAGE_SIZE;
54
55	return (struct stack_info) {
56		.low = low,
57		.high = high,
58	};
59}
60
61static struct stack_info stackinfo_get_hyp_kern_va(void)
62{
63	unsigned long low = (unsigned long)*this_cpu_ptr(&kvm_arm_hyp_stack_page);
64	unsigned long high = low + PAGE_SIZE;
65
66	return (struct stack_info) {
67		.low = low,
68		.high = high,
69	};
70}
71
72/*
73 * kvm_nvhe_stack_kern_va - Convert KVM nVHE HYP stack addresses to a kernel VAs
74 *
75 * The nVHE hypervisor stack is mapped in the flexible 'private' VA range, to
76 * allow for guard pages below the stack. Consequently, the fixed offset address
77 * translation macros won't work here.
78 *
79 * The kernel VA is calculated as an offset from the kernel VA of the hypervisor
80 * stack base.
81 *
82 * Returns true on success and updates @addr to its corresponding kernel VA;
83 * otherwise returns false.
84 */
85static bool kvm_nvhe_stack_kern_va(unsigned long *addr, unsigned long size)
86{
87	struct stack_info stack_hyp, stack_kern;
88
89	stack_hyp = stackinfo_get_hyp();
90	stack_kern = stackinfo_get_hyp_kern_va();
91	if (stackinfo_on_stack(&stack_hyp, *addr, size))
92		goto found;
93
94	stack_hyp = stackinfo_get_overflow();
95	stack_kern = stackinfo_get_overflow_kern_va();
96	if (stackinfo_on_stack(&stack_hyp, *addr, size))
97		goto found;
98
99	return false;
100
101found:
102	*addr = *addr - stack_hyp.low + stack_kern.low;
103	return true;
104}
105
106/*
107 * Convert a KVN nVHE HYP frame record address to a kernel VA
108 */
109static bool kvm_nvhe_stack_kern_record_va(unsigned long *addr)
110{
111	return kvm_nvhe_stack_kern_va(addr, 16);
112}
113
114static int unwind_next(struct unwind_state *state)
115{
116	/*
117	 * The FP is in the hypervisor VA space. Convert it to the kernel VA
118	 * space so it can be unwound by the regular unwind functions.
119	 */
120	if (!kvm_nvhe_stack_kern_record_va(&state->fp))
121		return -EINVAL;
122
123	return unwind_next_frame_record(state);
124}
125
126static void unwind(struct unwind_state *state,
127		   stack_trace_consume_fn consume_entry, void *cookie)
128{
129	while (1) {
130		int ret;
131
132		if (!consume_entry(cookie, state->pc))
133			break;
134		ret = unwind_next(state);
135		if (ret < 0)
136			break;
137	}
138}
139
140/*
141 * kvm_nvhe_dump_backtrace_entry - Symbolize and print an nVHE backtrace entry
142 *
143 * @arg    : the hypervisor offset, used for address translation
144 * @where  : the program counter corresponding to the stack frame
145 */
146static bool kvm_nvhe_dump_backtrace_entry(void *arg, unsigned long where)
147{
148	unsigned long va_mask = GENMASK_ULL(vabits_actual - 1, 0);
149	unsigned long hyp_offset = (unsigned long)arg;
150
151	/* Mask tags and convert to kern addr */
152	where = (where & va_mask) + hyp_offset;
153	kvm_err(" [<%016lx>] %pB\n", where, (void *)(where + kaslr_offset()));
154
155	return true;
156}
157
158static void kvm_nvhe_dump_backtrace_start(void)
159{
160	kvm_err("nVHE call trace:\n");
161}
162
163static void kvm_nvhe_dump_backtrace_end(void)
164{
165	kvm_err("---[ end nVHE call trace ]---\n");
166}
167
168/*
169 * hyp_dump_backtrace - Dump the non-protected nVHE backtrace.
170 *
171 * @hyp_offset: hypervisor offset, used for address translation.
172 *
173 * The host can directly access HYP stack pages in non-protected
174 * mode, so the unwinding is done directly from EL1. This removes
175 * the need for shared buffers between host and hypervisor for
176 * the stacktrace.
177 */
178static void hyp_dump_backtrace(unsigned long hyp_offset)
179{
180	struct kvm_nvhe_stacktrace_info *stacktrace_info;
181	struct stack_info stacks[] = {
182		stackinfo_get_overflow_kern_va(),
183		stackinfo_get_hyp_kern_va(),
184	};
185	struct unwind_state state = {
186		.stacks = stacks,
187		.nr_stacks = ARRAY_SIZE(stacks),
188	};
189
190	stacktrace_info = this_cpu_ptr_nvhe_sym(kvm_stacktrace_info);
191
192	kvm_nvhe_unwind_init(&state, stacktrace_info->fp, stacktrace_info->pc);
193
194	kvm_nvhe_dump_backtrace_start();
195	unwind(&state, kvm_nvhe_dump_backtrace_entry, (void *)hyp_offset);
196	kvm_nvhe_dump_backtrace_end();
197}
198
199#ifdef CONFIG_PROTECTED_NVHE_STACKTRACE
200DECLARE_KVM_NVHE_PER_CPU(unsigned long [NVHE_STACKTRACE_SIZE/sizeof(long)],
201			 pkvm_stacktrace);
202
203/*
204 * pkvm_dump_backtrace - Dump the protected nVHE HYP backtrace.
205 *
206 * @hyp_offset: hypervisor offset, used for address translation.
207 *
208 * Dumping of the pKVM HYP backtrace is done by reading the
209 * stack addresses from the shared stacktrace buffer, since the
210 * host cannot directly access hypervisor memory in protected
211 * mode.
212 */
213static void pkvm_dump_backtrace(unsigned long hyp_offset)
214{
215	unsigned long *stacktrace
216		= (unsigned long *) this_cpu_ptr_nvhe_sym(pkvm_stacktrace);
217	int i;
218
219	kvm_nvhe_dump_backtrace_start();
220	/* The saved stacktrace is terminated by a null entry */
221	for (i = 0;
222	     i < ARRAY_SIZE(kvm_nvhe_sym(pkvm_stacktrace)) && stacktrace[i];
223	     i++)
224		kvm_nvhe_dump_backtrace_entry((void *)hyp_offset, stacktrace[i]);
225	kvm_nvhe_dump_backtrace_end();
226}
227#else	/* !CONFIG_PROTECTED_NVHE_STACKTRACE */
228static void pkvm_dump_backtrace(unsigned long hyp_offset)
229{
230	kvm_err("Cannot dump pKVM nVHE stacktrace: !CONFIG_PROTECTED_NVHE_STACKTRACE\n");
231}
232#endif /* CONFIG_PROTECTED_NVHE_STACKTRACE */
233
234/*
235 * kvm_nvhe_dump_backtrace - Dump KVM nVHE hypervisor backtrace.
236 *
237 * @hyp_offset: hypervisor offset, used for address translation.
238 */
239void kvm_nvhe_dump_backtrace(unsigned long hyp_offset)
240{
241	if (is_protected_kvm_enabled())
242		pkvm_dump_backtrace(hyp_offset);
243	else
244		hyp_dump_backtrace(hyp_offset);
245}
246