1// SPDX-License-Identifier: GPL-2.0 2 3#ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG 4#include <linux/memblock.h> 5#endif 6#include <linux/console.h> 7#include <linux/cpu.h> 8#include <linux/kexec.h> 9#include <linux/slab.h> 10#include <linux/panic_notifier.h> 11 12#include <xen/xen.h> 13#include <xen/features.h> 14#include <xen/interface/sched.h> 15#include <xen/interface/version.h> 16#include <xen/page.h> 17 18#include <asm/xen/hypercall.h> 19#include <asm/xen/hypervisor.h> 20#include <asm/cpu.h> 21#include <asm/e820/api.h> 22#include <asm/setup.h> 23 24#include "xen-ops.h" 25#include "smp.h" 26#include "pmu.h" 27 28EXPORT_SYMBOL_GPL(hypercall_page); 29 30/* 31 * Pointer to the xen_vcpu_info structure or 32 * &HYPERVISOR_shared_info->vcpu_info[cpu]. See xen_hvm_init_shared_info 33 * and xen_vcpu_setup for details. By default it points to share_info->vcpu_info 34 * but during boot it is switched to point to xen_vcpu_info. 35 * The pointer is used in xen_evtchn_do_upcall to acknowledge pending events. 36 * Make sure that xen_vcpu_info doesn't cross a page boundary by making it 37 * cache-line aligned (the struct is guaranteed to have a size of 64 bytes, 38 * which matches the cache line size of 64-bit x86 processors). 39 */ 40DEFINE_PER_CPU(struct vcpu_info *, xen_vcpu); 41DEFINE_PER_CPU_ALIGNED(struct vcpu_info, xen_vcpu_info); 42 43/* Linux <-> Xen vCPU id mapping */ 44DEFINE_PER_CPU(uint32_t, xen_vcpu_id); 45EXPORT_PER_CPU_SYMBOL(xen_vcpu_id); 46 47unsigned long *machine_to_phys_mapping = (void *)MACH2PHYS_VIRT_START; 48EXPORT_SYMBOL(machine_to_phys_mapping); 49unsigned long machine_to_phys_nr; 50EXPORT_SYMBOL(machine_to_phys_nr); 51 52struct start_info *xen_start_info; 53EXPORT_SYMBOL_GPL(xen_start_info); 54 55struct shared_info xen_dummy_shared_info; 56 57__read_mostly bool xen_have_vector_callback = true; 58EXPORT_SYMBOL_GPL(xen_have_vector_callback); 59 60/* 61 * NB: These need to live in .data or alike because they're used by 62 * xen_prepare_pvh() which runs before clearing the bss. 63 */ 64enum xen_domain_type __ro_after_init xen_domain_type = XEN_NATIVE; 65EXPORT_SYMBOL_GPL(xen_domain_type); 66uint32_t __ro_after_init xen_start_flags; 67EXPORT_SYMBOL(xen_start_flags); 68 69/* 70 * Point at some empty memory to start with. We map the real shared_info 71 * page as soon as fixmap is up and running. 72 */ 73struct shared_info *HYPERVISOR_shared_info = &xen_dummy_shared_info; 74 75static int xen_cpu_up_online(unsigned int cpu) 76{ 77 xen_init_lock_cpu(cpu); 78 return 0; 79} 80 81int xen_cpuhp_setup(int (*cpu_up_prepare_cb)(unsigned int), 82 int (*cpu_dead_cb)(unsigned int)) 83{ 84 int rc; 85 86 rc = cpuhp_setup_state_nocalls(CPUHP_XEN_PREPARE, 87 "x86/xen/guest:prepare", 88 cpu_up_prepare_cb, cpu_dead_cb); 89 if (rc >= 0) { 90 rc = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, 91 "x86/xen/guest:online", 92 xen_cpu_up_online, NULL); 93 if (rc < 0) 94 cpuhp_remove_state_nocalls(CPUHP_XEN_PREPARE); 95 } 96 97 return rc >= 0 ? 0 : rc; 98} 99 100static void xen_vcpu_setup_restore(int cpu) 101{ 102 /* Any per_cpu(xen_vcpu) is stale, so reset it */ 103 xen_vcpu_info_reset(cpu); 104 105 /* 106 * For PVH and PVHVM, setup online VCPUs only. The rest will 107 * be handled by hotplug. 108 */ 109 if (xen_pv_domain() || 110 (xen_hvm_domain() && cpu_online(cpu))) 111 xen_vcpu_setup(cpu); 112} 113 114/* 115 * On restore, set the vcpu placement up again. 116 * If it fails, then we're in a bad state, since 117 * we can't back out from using it... 118 */ 119void xen_vcpu_restore(void) 120{ 121 int cpu; 122 123 for_each_possible_cpu(cpu) { 124 bool other_cpu = (cpu != smp_processor_id()); 125 bool is_up; 126 127 if (xen_vcpu_nr(cpu) == XEN_VCPU_ID_INVALID) 128 continue; 129 130 /* Only Xen 4.5 and higher support this. */ 131 is_up = HYPERVISOR_vcpu_op(VCPUOP_is_up, 132 xen_vcpu_nr(cpu), NULL) > 0; 133 134 if (other_cpu && is_up && 135 HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL)) 136 BUG(); 137 138 if (xen_pv_domain() || xen_feature(XENFEAT_hvm_safe_pvclock)) 139 xen_setup_runstate_info(cpu); 140 141 xen_vcpu_setup_restore(cpu); 142 143 if (other_cpu && is_up && 144 HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL)) 145 BUG(); 146 } 147} 148 149void xen_vcpu_info_reset(int cpu) 150{ 151 if (xen_vcpu_nr(cpu) < MAX_VIRT_CPUS) { 152 per_cpu(xen_vcpu, cpu) = 153 &HYPERVISOR_shared_info->vcpu_info[xen_vcpu_nr(cpu)]; 154 } else { 155 /* Set to NULL so that if somebody accesses it we get an OOPS */ 156 per_cpu(xen_vcpu, cpu) = NULL; 157 } 158} 159 160void xen_vcpu_setup(int cpu) 161{ 162 struct vcpu_register_vcpu_info info; 163 int err; 164 struct vcpu_info *vcpup; 165 166 BUILD_BUG_ON(sizeof(*vcpup) > SMP_CACHE_BYTES); 167 BUG_ON(HYPERVISOR_shared_info == &xen_dummy_shared_info); 168 169 /* 170 * This path is called on PVHVM at bootup (xen_hvm_smp_prepare_boot_cpu) 171 * and at restore (xen_vcpu_restore). Also called for hotplugged 172 * VCPUs (cpu_init -> xen_hvm_cpu_prepare_hvm). 173 * However, the hypercall can only be done once (see below) so if a VCPU 174 * is offlined and comes back online then let's not redo the hypercall. 175 * 176 * For PV it is called during restore (xen_vcpu_restore) and bootup 177 * (xen_setup_vcpu_info_placement). The hotplug mechanism does not 178 * use this function. 179 */ 180 if (xen_hvm_domain()) { 181 if (per_cpu(xen_vcpu, cpu) == &per_cpu(xen_vcpu_info, cpu)) 182 return; 183 } 184 185 vcpup = &per_cpu(xen_vcpu_info, cpu); 186 info.mfn = arbitrary_virt_to_mfn(vcpup); 187 info.offset = offset_in_page(vcpup); 188 189 /* 190 * N.B. This hypercall can _only_ be called once per CPU. 191 * Subsequent calls will error out with -EINVAL. This is due to 192 * the fact that hypervisor has no unregister variant and this 193 * hypercall does not allow to over-write info.mfn and 194 * info.offset. 195 */ 196 err = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_info, xen_vcpu_nr(cpu), 197 &info); 198 if (err) 199 panic("register_vcpu_info failed: cpu=%d err=%d\n", cpu, err); 200 201 per_cpu(xen_vcpu, cpu) = vcpup; 202} 203 204void __init xen_banner(void) 205{ 206 unsigned version = HYPERVISOR_xen_version(XENVER_version, NULL); 207 struct xen_extraversion extra; 208 209 HYPERVISOR_xen_version(XENVER_extraversion, &extra); 210 211 pr_info("Booting kernel on %s\n", pv_info.name); 212 pr_info("Xen version: %u.%u%s%s\n", 213 version >> 16, version & 0xffff, extra.extraversion, 214 xen_feature(XENFEAT_mmu_pt_update_preserve_ad) 215 ? " (preserve-AD)" : ""); 216} 217 218/* Check if running on Xen version (major, minor) or later */ 219bool xen_running_on_version_or_later(unsigned int major, unsigned int minor) 220{ 221 unsigned int version; 222 223 if (!xen_domain()) 224 return false; 225 226 version = HYPERVISOR_xen_version(XENVER_version, NULL); 227 if ((((version >> 16) == major) && ((version & 0xffff) >= minor)) || 228 ((version >> 16) > major)) 229 return true; 230 return false; 231} 232 233void __init xen_add_preferred_consoles(void) 234{ 235 add_preferred_console("xenboot", 0, NULL); 236 if (!boot_params.screen_info.orig_video_isVGA) 237 add_preferred_console("tty", 0, NULL); 238 add_preferred_console("hvc", 0, NULL); 239 if (boot_params.screen_info.orig_video_isVGA) 240 add_preferred_console("tty", 0, NULL); 241} 242 243void xen_reboot(int reason) 244{ 245 struct sched_shutdown r = { .reason = reason }; 246 int cpu; 247 248 for_each_online_cpu(cpu) 249 xen_pmu_finish(cpu); 250 251 if (HYPERVISOR_sched_op(SCHEDOP_shutdown, &r)) 252 BUG(); 253} 254 255static int reboot_reason = SHUTDOWN_reboot; 256static bool xen_legacy_crash; 257void xen_emergency_restart(void) 258{ 259 xen_reboot(reboot_reason); 260} 261 262static int 263xen_panic_event(struct notifier_block *this, unsigned long event, void *ptr) 264{ 265 if (!kexec_crash_loaded()) { 266 if (xen_legacy_crash) 267 xen_reboot(SHUTDOWN_crash); 268 269 reboot_reason = SHUTDOWN_crash; 270 271 /* 272 * If panic_timeout==0 then we are supposed to wait forever. 273 * However, to preserve original dom0 behavior we have to drop 274 * into hypervisor. (domU behavior is controlled by its 275 * config file) 276 */ 277 if (panic_timeout == 0) 278 panic_timeout = -1; 279 } 280 return NOTIFY_DONE; 281} 282 283static int __init parse_xen_legacy_crash(char *arg) 284{ 285 xen_legacy_crash = true; 286 return 0; 287} 288early_param("xen_legacy_crash", parse_xen_legacy_crash); 289 290static struct notifier_block xen_panic_block = { 291 .notifier_call = xen_panic_event, 292 .priority = INT_MIN 293}; 294 295int xen_panic_handler_init(void) 296{ 297 atomic_notifier_chain_register(&panic_notifier_list, &xen_panic_block); 298 return 0; 299} 300 301void xen_pin_vcpu(int cpu) 302{ 303 static bool disable_pinning; 304 struct sched_pin_override pin_override; 305 int ret; 306 307 if (disable_pinning) 308 return; 309 310 pin_override.pcpu = cpu; 311 ret = HYPERVISOR_sched_op(SCHEDOP_pin_override, &pin_override); 312 313 /* Ignore errors when removing override. */ 314 if (cpu < 0) 315 return; 316 317 switch (ret) { 318 case -ENOSYS: 319 pr_warn("Unable to pin on physical cpu %d. In case of problems consider vcpu pinning.\n", 320 cpu); 321 disable_pinning = true; 322 break; 323 case -EPERM: 324 WARN(1, "Trying to pin vcpu without having privilege to do so\n"); 325 disable_pinning = true; 326 break; 327 case -EINVAL: 328 case -EBUSY: 329 pr_warn("Physical cpu %d not available for pinning. Check Xen cpu configuration.\n", 330 cpu); 331 break; 332 case 0: 333 break; 334 default: 335 WARN(1, "rc %d while trying to pin vcpu\n", ret); 336 disable_pinning = true; 337 } 338} 339 340#ifdef CONFIG_HOTPLUG_CPU 341void xen_arch_register_cpu(int num) 342{ 343 arch_register_cpu(num); 344} 345EXPORT_SYMBOL(xen_arch_register_cpu); 346 347void xen_arch_unregister_cpu(int num) 348{ 349 arch_unregister_cpu(num); 350} 351EXPORT_SYMBOL(xen_arch_unregister_cpu); 352#endif 353