1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2012-2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#include <hyp/sysreg-sr.h>
8
9#include <linux/compiler.h>
10#include <linux/kvm_host.h>
11
12#include <asm/kprobes.h>
13#include <asm/kvm_asm.h>
14#include <asm/kvm_emulate.h>
15#include <asm/kvm_hyp.h>
16
17/*
18 * VHE: Host and guest must save mdscr_el1 and sp_el0 (and the PC and
19 * pstate, which are handled as part of the el2 return state) on every
20 * switch (sp_el0 is being dealt with in the assembly code).
21 * tpidr_el0 and tpidrro_el0 only need to be switched when going
22 * to host userspace or a different VCPU.  EL1 registers only need to be
23 * switched when potentially going to run a different VCPU.  The latter two
24 * classes are handled as part of kvm_arch_vcpu_load and kvm_arch_vcpu_put.
25 */
26
27void sysreg_save_host_state_vhe(struct kvm_cpu_context *ctxt)
28{
29	__sysreg_save_common_state(ctxt);
30}
31NOKPROBE_SYMBOL(sysreg_save_host_state_vhe);
32
33void sysreg_save_guest_state_vhe(struct kvm_cpu_context *ctxt)
34{
35	__sysreg_save_common_state(ctxt);
36	__sysreg_save_el2_return_state(ctxt);
37}
38NOKPROBE_SYMBOL(sysreg_save_guest_state_vhe);
39
40void sysreg_restore_host_state_vhe(struct kvm_cpu_context *ctxt)
41{
42	__sysreg_restore_common_state(ctxt);
43}
44NOKPROBE_SYMBOL(sysreg_restore_host_state_vhe);
45
46void sysreg_restore_guest_state_vhe(struct kvm_cpu_context *ctxt)
47{
48	__sysreg_restore_common_state(ctxt);
49	__sysreg_restore_el2_return_state(ctxt);
50}
51NOKPROBE_SYMBOL(sysreg_restore_guest_state_vhe);
52
53/**
54 * kvm_vcpu_load_sysregs_vhe - Load guest system registers to the physical CPU
55 *
56 * @vcpu: The VCPU pointer
57 *
58 * Load system registers that do not affect the host's execution, for
59 * example EL1 system registers on a VHE system where the host kernel
60 * runs at EL2.  This function is called from KVM's vcpu_load() function
61 * and loading system register state early avoids having to load them on
62 * every entry to the VM.
63 */
64void kvm_vcpu_load_sysregs_vhe(struct kvm_vcpu *vcpu)
65{
66	struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
67	struct kvm_cpu_context *host_ctxt;
68
69	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
70	__sysreg_save_user_state(host_ctxt);
71
72	/*
73	 * Load guest EL1 and user state
74	 *
75	 * We must restore the 32-bit state before the sysregs, thanks
76	 * to erratum #852523 (Cortex-A57) or #853709 (Cortex-A72).
77	 */
78	__sysreg32_restore_state(vcpu);
79	__sysreg_restore_user_state(guest_ctxt);
80	__sysreg_restore_el1_state(guest_ctxt);
81
82	vcpu->arch.sysregs_loaded_on_cpu = true;
83
84	activate_traps_vhe_load(vcpu);
85}
86
87/**
88 * kvm_vcpu_put_sysregs_vhe - Restore host system registers to the physical CPU
89 *
90 * @vcpu: The VCPU pointer
91 *
92 * Save guest system registers that do not affect the host's execution, for
93 * example EL1 system registers on a VHE system where the host kernel
94 * runs at EL2.  This function is called from KVM's vcpu_put() function
95 * and deferring saving system register state until we're no longer running the
96 * VCPU avoids having to save them on every exit from the VM.
97 */
98void kvm_vcpu_put_sysregs_vhe(struct kvm_vcpu *vcpu)
99{
100	struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt;
101	struct kvm_cpu_context *host_ctxt;
102
103	host_ctxt = &this_cpu_ptr(&kvm_host_data)->host_ctxt;
104	deactivate_traps_vhe_put();
105
106	__sysreg_save_el1_state(guest_ctxt);
107	__sysreg_save_user_state(guest_ctxt);
108	__sysreg32_save_state(vcpu);
109
110	/* Restore host user state */
111	__sysreg_restore_user_state(host_ctxt);
112
113	vcpu->arch.sysregs_loaded_on_cpu = false;
114}
115