1// SPDX-License-Identifier: GPL-2.0-only 2#include <linux/extable.h> 3#include <linux/uaccess.h> 4#include <linux/sched/debug.h> 5#include <xen/xen.h> 6 7#include <asm/fpu/internal.h> 8#include <asm/sev-es.h> 9#include <asm/traps.h> 10#include <asm/kdebug.h> 11 12typedef bool (*ex_handler_t)(const struct exception_table_entry *, 13 struct pt_regs *, int, unsigned long, 14 unsigned long); 15 16static inline unsigned long 17ex_fixup_addr(const struct exception_table_entry *x) 18{ 19 return (unsigned long)&x->fixup + x->fixup; 20} 21static inline ex_handler_t 22ex_fixup_handler(const struct exception_table_entry *x) 23{ 24 return (ex_handler_t)((unsigned long)&x->handler + x->handler); 25} 26 27__visible bool ex_handler_default(const struct exception_table_entry *fixup, 28 struct pt_regs *regs, int trapnr, 29 unsigned long error_code, 30 unsigned long fault_addr) 31{ 32 regs->ip = ex_fixup_addr(fixup); 33 return true; 34} 35EXPORT_SYMBOL(ex_handler_default); 36 37__visible bool ex_handler_fault(const struct exception_table_entry *fixup, 38 struct pt_regs *regs, int trapnr, 39 unsigned long error_code, 40 unsigned long fault_addr) 41{ 42 regs->ip = ex_fixup_addr(fixup); 43 regs->ax = trapnr; 44 return true; 45} 46EXPORT_SYMBOL_GPL(ex_handler_fault); 47 48/* 49 * Handler for when we fail to restore a task's FPU state. We should never get 50 * here because the FPU state of a task using the FPU (task->thread.fpu.state) 51 * should always be valid. However, past bugs have allowed userspace to set 52 * reserved bits in the XSAVE area using PTRACE_SETREGSET or sys_rt_sigreturn(). 53 * These caused XRSTOR to fail when switching to the task, leaking the FPU 54 * registers of the task previously executing on the CPU. Mitigate this class 55 * of vulnerability by restoring from the initial state (essentially, zeroing 56 * out all the FPU registers) if we can't restore from the task's FPU state. 57 */ 58__visible bool ex_handler_fprestore(const struct exception_table_entry *fixup, 59 struct pt_regs *regs, int trapnr, 60 unsigned long error_code, 61 unsigned long fault_addr) 62{ 63 regs->ip = ex_fixup_addr(fixup); 64 65 WARN_ONCE(1, "Bad FPU state detected at %pB, reinitializing FPU registers.", 66 (void *)instruction_pointer(regs)); 67 68 __copy_kernel_to_fpregs(&init_fpstate, -1); 69 return true; 70} 71EXPORT_SYMBOL_GPL(ex_handler_fprestore); 72 73__visible bool ex_handler_uaccess(const struct exception_table_entry *fixup, 74 struct pt_regs *regs, int trapnr, 75 unsigned long error_code, 76 unsigned long fault_addr) 77{ 78 WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?"); 79 regs->ip = ex_fixup_addr(fixup); 80 return true; 81} 82EXPORT_SYMBOL(ex_handler_uaccess); 83 84__visible bool ex_handler_copy(const struct exception_table_entry *fixup, 85 struct pt_regs *regs, int trapnr, 86 unsigned long error_code, 87 unsigned long fault_addr) 88{ 89 WARN_ONCE(trapnr == X86_TRAP_GP, "General protection fault in user access. Non-canonical address?"); 90 regs->ip = ex_fixup_addr(fixup); 91 regs->ax = trapnr; 92 return true; 93} 94EXPORT_SYMBOL(ex_handler_copy); 95 96__visible bool ex_handler_rdmsr_unsafe(const struct exception_table_entry *fixup, 97 struct pt_regs *regs, int trapnr, 98 unsigned long error_code, 99 unsigned long fault_addr) 100{ 101 if (pr_warn_once("unchecked MSR access error: RDMSR from 0x%x at rIP: 0x%lx (%pS)\n", 102 (unsigned int)regs->cx, regs->ip, (void *)regs->ip)) 103 show_stack_regs(regs); 104 105 /* Pretend that the read succeeded and returned 0. */ 106 regs->ip = ex_fixup_addr(fixup); 107 regs->ax = 0; 108 regs->dx = 0; 109 return true; 110} 111EXPORT_SYMBOL(ex_handler_rdmsr_unsafe); 112 113__visible bool ex_handler_wrmsr_unsafe(const struct exception_table_entry *fixup, 114 struct pt_regs *regs, int trapnr, 115 unsigned long error_code, 116 unsigned long fault_addr) 117{ 118 if (pr_warn_once("unchecked MSR access error: WRMSR to 0x%x (tried to write 0x%08x%08x) at rIP: 0x%lx (%pS)\n", 119 (unsigned int)regs->cx, (unsigned int)regs->dx, 120 (unsigned int)regs->ax, regs->ip, (void *)regs->ip)) 121 show_stack_regs(regs); 122 123 /* Pretend that the write succeeded. */ 124 regs->ip = ex_fixup_addr(fixup); 125 return true; 126} 127EXPORT_SYMBOL(ex_handler_wrmsr_unsafe); 128 129__visible bool ex_handler_clear_fs(const struct exception_table_entry *fixup, 130 struct pt_regs *regs, int trapnr, 131 unsigned long error_code, 132 unsigned long fault_addr) 133{ 134 if (static_cpu_has(X86_BUG_NULL_SEG)) 135 asm volatile ("mov %0, %%fs" : : "rm" (__USER_DS)); 136 asm volatile ("mov %0, %%fs" : : "rm" (0)); 137 return ex_handler_default(fixup, regs, trapnr, error_code, fault_addr); 138} 139EXPORT_SYMBOL(ex_handler_clear_fs); 140 141enum handler_type ex_get_fault_handler_type(unsigned long ip) 142{ 143 const struct exception_table_entry *e; 144 ex_handler_t handler; 145 146 e = search_exception_tables(ip); 147 if (!e) 148 return EX_HANDLER_NONE; 149 handler = ex_fixup_handler(e); 150 if (handler == ex_handler_fault) 151 return EX_HANDLER_FAULT; 152 else if (handler == ex_handler_uaccess || handler == ex_handler_copy) 153 return EX_HANDLER_UACCESS; 154 else 155 return EX_HANDLER_OTHER; 156} 157 158int fixup_exception(struct pt_regs *regs, int trapnr, unsigned long error_code, 159 unsigned long fault_addr) 160{ 161 const struct exception_table_entry *e; 162 ex_handler_t handler; 163 164#ifdef CONFIG_PNPBIOS 165 if (unlikely(SEGMENT_IS_PNP_CODE(regs->cs))) { 166 extern u32 pnp_bios_fault_eip, pnp_bios_fault_esp; 167 extern u32 pnp_bios_is_utter_crap; 168 pnp_bios_is_utter_crap = 1; 169 printk(KERN_CRIT "PNPBIOS fault.. attempting recovery.\n"); 170 __asm__ volatile( 171 "movl %0, %%esp\n\t" 172 "jmp *%1\n\t" 173 : : "g" (pnp_bios_fault_esp), "g" (pnp_bios_fault_eip)); 174 panic("do_trap: can't hit this"); 175 } 176#endif 177 178 e = search_exception_tables(regs->ip); 179 if (!e) 180 return 0; 181 182 handler = ex_fixup_handler(e); 183 return handler(e, regs, trapnr, error_code, fault_addr); 184} 185 186extern unsigned int early_recursion_flag; 187 188/* Restricted version used during very early boot */ 189void __init early_fixup_exception(struct pt_regs *regs, int trapnr) 190{ 191 /* Ignore early NMIs. */ 192 if (trapnr == X86_TRAP_NMI) 193 return; 194 195 if (early_recursion_flag > 2) 196 goto halt_loop; 197 198 /* 199 * Old CPUs leave the high bits of CS on the stack 200 * undefined. I'm not sure which CPUs do this, but at least 201 * the 486 DX works this way. 202 * Xen pv domains are not using the default __KERNEL_CS. 203 */ 204 if (!xen_pv_domain() && regs->cs != __KERNEL_CS) 205 goto fail; 206 207 /* 208 * The full exception fixup machinery is available as soon as 209 * the early IDT is loaded. This means that it is the 210 * responsibility of extable users to either function correctly 211 * when handlers are invoked early or to simply avoid causing 212 * exceptions before they're ready to handle them. 213 * 214 * This is better than filtering which handlers can be used, 215 * because refusing to call a handler here is guaranteed to 216 * result in a hard-to-debug panic. 217 * 218 * Keep in mind that not all vectors actually get here. Early 219 * page faults, for example, are special. 220 */ 221 if (fixup_exception(regs, trapnr, regs->orig_ax, 0)) 222 return; 223 224 if (trapnr == X86_TRAP_UD) { 225 if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) { 226 /* Skip the ud2. */ 227 regs->ip += LEN_UD2; 228 return; 229 } 230 231 /* 232 * If this was a BUG and report_bug returns or if this 233 * was just a normal #UD, we want to continue onward and 234 * crash. 235 */ 236 } 237 238fail: 239 early_printk("PANIC: early exception 0x%02x IP %lx:%lx error %lx cr2 0x%lx\n", 240 (unsigned)trapnr, (unsigned long)regs->cs, regs->ip, 241 regs->orig_ax, read_cr2()); 242 243 show_regs(regs); 244 245halt_loop: 246 while (true) 247 halt(); 248} 249