xref: /kernel/linux/linux-5.10/arch/x86/xen/smp_pv.c (revision 8c2ecf20) 
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Xen SMP support
4 *
5 * This file implements the Xen versions of smp_ops.  SMP under Xen is
6 * very straightforward.  Bringing a CPU up is simply a matter of
7 * loading its initial context and setting it running.
8 *
9 * IPIs are handled through the Xen event mechanism.
10 *
11 * Because virtual CPUs can be scheduled onto any real CPU, there's no
12 * useful topology information for the kernel to make use of.  As a
13 * result, all CPUs are treated as if they're single-core and
14 * single-threaded.
15 */
16#include <linux/sched.h>
17#include <linux/sched/task_stack.h>
18#include <linux/err.h>
19#include <linux/slab.h>
20#include <linux/smp.h>
21#include <linux/irq_work.h>
22#include <linux/tick.h>
23#include <linux/nmi.h>
24#include <linux/cpuhotplug.h>
25#include <linux/stackprotector.h>
26#include <linux/pgtable.h>
27
28#include <asm/paravirt.h>
29#include <asm/idtentry.h>
30#include <asm/desc.h>
31#include <asm/cpu.h>
32#include <asm/io_apic.h>
33#include <asm/fpu/internal.h>
34
35#include <xen/interface/xen.h>
36#include <xen/interface/vcpu.h>
37#include <xen/interface/xenpmu.h>
38
39#include <asm/spec-ctrl.h>
40#include <asm/xen/interface.h>
41#include <asm/xen/hypercall.h>
42
43#include <xen/xen.h>
44#include <xen/page.h>
45#include <xen/events.h>
46
47#include <xen/hvc-console.h>
48#include "xen-ops.h"
49#include "mmu.h"
50#include "smp.h"
51#include "pmu.h"
52
53cpumask_var_t xen_cpu_initialized_map;
54
55static DEFINE_PER_CPU(struct xen_common_irq, xen_irq_work) = { .irq = -1 };
56static DEFINE_PER_CPU(struct xen_common_irq, xen_pmu_irq) = { .irq = -1 };
57
58static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id);
59void asm_cpu_bringup_and_idle(void);
60
61static void cpu_bringup(void)
62{
63	int cpu;
64
65	cr4_init();
66	cpu_init();
67	fpu__init_cpu();
68	touch_softlockup_watchdog();
69	preempt_disable();
70
71	/* PVH runs in ring 0 and allows us to do native syscalls. Yay! */
72	if (!xen_feature(XENFEAT_supervisor_mode_kernel)) {
73		xen_enable_sysenter();
74		xen_enable_syscall();
75	}
76	cpu = smp_processor_id();
77	smp_store_cpu_info(cpu);
78	cpu_data(cpu).x86_max_cores = 1;
79	set_cpu_sibling_map(cpu);
80
81	speculative_store_bypass_ht_init();
82
83	xen_setup_cpu_clockevents();
84
85	notify_cpu_starting(cpu);
86
87	set_cpu_online(cpu, true);
88
89	cpu_set_state_online(cpu);  /* Implies full memory barrier. */
90
91	/* We can take interrupts now: we're officially "up". */
92	local_irq_enable();
93}
94
95asmlinkage __visible void cpu_bringup_and_idle(void)
96{
97	cpu_bringup();
98	cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
99}
100
101void xen_smp_intr_free_pv(unsigned int cpu)
102{
103	kfree(per_cpu(xen_irq_work, cpu).name);
104	per_cpu(xen_irq_work, cpu).name = NULL;
105	if (per_cpu(xen_irq_work, cpu).irq >= 0) {
106		unbind_from_irqhandler(per_cpu(xen_irq_work, cpu).irq, NULL);
107		per_cpu(xen_irq_work, cpu).irq = -1;
108	}
109
110	kfree(per_cpu(xen_pmu_irq, cpu).name);
111	per_cpu(xen_pmu_irq, cpu).name = NULL;
112	if (per_cpu(xen_pmu_irq, cpu).irq >= 0) {
113		unbind_from_irqhandler(per_cpu(xen_pmu_irq, cpu).irq, NULL);
114		per_cpu(xen_pmu_irq, cpu).irq = -1;
115	}
116}
117
118int xen_smp_intr_init_pv(unsigned int cpu)
119{
120	int rc;
121	char *callfunc_name, *pmu_name;
122
123	callfunc_name = kasprintf(GFP_KERNEL, "irqwork%d", cpu);
124	per_cpu(xen_irq_work, cpu).name = callfunc_name;
125	rc = bind_ipi_to_irqhandler(XEN_IRQ_WORK_VECTOR,
126				    cpu,
127				    xen_irq_work_interrupt,
128				    IRQF_PERCPU|IRQF_NOBALANCING,
129				    callfunc_name,
130				    NULL);
131	if (rc < 0)
132		goto fail;
133	per_cpu(xen_irq_work, cpu).irq = rc;
134
135	if (is_xen_pmu) {
136		pmu_name = kasprintf(GFP_KERNEL, "pmu%d", cpu);
137		per_cpu(xen_pmu_irq, cpu).name = pmu_name;
138		rc = bind_virq_to_irqhandler(VIRQ_XENPMU, cpu,
139					     xen_pmu_irq_handler,
140					     IRQF_PERCPU|IRQF_NOBALANCING,
141					     pmu_name, NULL);
142		if (rc < 0)
143			goto fail;
144		per_cpu(xen_pmu_irq, cpu).irq = rc;
145	}
146
147	return 0;
148
149 fail:
150	xen_smp_intr_free_pv(cpu);
151	return rc;
152}
153
154static void __init _get_smp_config(unsigned int early)
155{
156	int i, rc;
157	unsigned int subtract = 0;
158
159	if (early)
160		return;
161
162	num_processors = 0;
163	disabled_cpus = 0;
164	for (i = 0; i < nr_cpu_ids; i++) {
165		rc = HYPERVISOR_vcpu_op(VCPUOP_is_up, i, NULL);
166		if (rc >= 0) {
167			num_processors++;
168			set_cpu_possible(i, true);
169		} else {
170			set_cpu_possible(i, false);
171			set_cpu_present(i, false);
172			subtract++;
173		}
174	}
175#ifdef CONFIG_HOTPLUG_CPU
176	/* This is akin to using 'nr_cpus' on the Linux command line.
177	 * Which is OK as when we use 'dom0_max_vcpus=X' we can only
178	 * have up to X, while nr_cpu_ids is greater than X. This
179	 * normally is not a problem, except when CPU hotplugging
180	 * is involved and then there might be more than X CPUs
181	 * in the guest - which will not work as there is no
182	 * hypercall to expand the max number of VCPUs an already
183	 * running guest has. So cap it up to X. */
184	if (subtract)
185		nr_cpu_ids = nr_cpu_ids - subtract;
186#endif
187
188}
189
190static void __init xen_pv_smp_prepare_boot_cpu(void)
191{
192	BUG_ON(smp_processor_id() != 0);
193	native_smp_prepare_boot_cpu();
194
195	if (!xen_feature(XENFEAT_writable_page_tables))
196		/* We've switched to the "real" per-cpu gdt, so make
197		 * sure the old memory can be recycled. */
198		make_lowmem_page_readwrite(xen_initial_gdt);
199
200	xen_setup_vcpu_info_placement();
201
202	/*
203	 * The alternative logic (which patches the unlock/lock) runs before
204	 * the smp bootup up code is activated. Hence we need to set this up
205	 * the core kernel is being patched. Otherwise we will have only
206	 * modules patched but not core code.
207	 */
208	xen_init_spinlocks();
209}
210
211static void __init xen_pv_smp_prepare_cpus(unsigned int max_cpus)
212{
213	unsigned cpu;
214	unsigned int i;
215
216	if (skip_ioapic_setup) {
217		char *m = (max_cpus == 0) ?
218			"The nosmp parameter is incompatible with Xen; " \
219			"use Xen dom0_max_vcpus=1 parameter" :
220			"The noapic parameter is incompatible with Xen";
221
222		xen_raw_printk(m);
223		panic(m);
224	}
225	xen_init_lock_cpu(0);
226
227	smp_store_boot_cpu_info();
228	cpu_data(0).x86_max_cores = 1;
229
230	for_each_possible_cpu(i) {
231		zalloc_cpumask_var(&per_cpu(cpu_sibling_map, i), GFP_KERNEL);
232		zalloc_cpumask_var(&per_cpu(cpu_core_map, i), GFP_KERNEL);
233		zalloc_cpumask_var(&per_cpu(cpu_die_map, i), GFP_KERNEL);
234		zalloc_cpumask_var(&per_cpu(cpu_llc_shared_map, i), GFP_KERNEL);
235	}
236	set_cpu_sibling_map(0);
237
238	speculative_store_bypass_ht_init();
239
240	xen_pmu_init(0);
241
242	if (xen_smp_intr_init(0) || xen_smp_intr_init_pv(0))
243		BUG();
244
245	if (!alloc_cpumask_var(&xen_cpu_initialized_map, GFP_KERNEL))
246		panic("could not allocate xen_cpu_initialized_map\n");
247
248	cpumask_copy(xen_cpu_initialized_map, cpumask_of(0));
249
250	/* Restrict the possible_map according to max_cpus. */
251	while ((num_possible_cpus() > 1) && (num_possible_cpus() > max_cpus)) {
252		for (cpu = nr_cpu_ids - 1; !cpu_possible(cpu); cpu--)
253			continue;
254		set_cpu_possible(cpu, false);
255	}
256
257	for_each_possible_cpu(cpu)
258		set_cpu_present(cpu, true);
259}
260
261static int
262cpu_initialize_context(unsigned int cpu, struct task_struct *idle)
263{
264	struct vcpu_guest_context *ctxt;
265	struct desc_struct *gdt;
266	unsigned long gdt_mfn;
267
268	/* used to tell cpu_init() that it can proceed with initialization */
269	cpumask_set_cpu(cpu, cpu_callout_mask);
270	if (cpumask_test_and_set_cpu(cpu, xen_cpu_initialized_map))
271		return 0;
272
273	ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL);
274	if (ctxt == NULL)
275		return -ENOMEM;
276
277	gdt = get_cpu_gdt_rw(cpu);
278
279	memset(&ctxt->fpu_ctxt, 0, sizeof(ctxt->fpu_ctxt));
280
281	/*
282	 * Bring up the CPU in cpu_bringup_and_idle() with the stack
283	 * pointing just below where pt_regs would be if it were a normal
284	 * kernel entry.
285	 */
286	ctxt->user_regs.eip = (unsigned long)asm_cpu_bringup_and_idle;
287	ctxt->flags = VGCF_IN_KERNEL;
288	ctxt->user_regs.eflags = 0x1000; /* IOPL_RING1 */
289	ctxt->user_regs.ds = __USER_DS;
290	ctxt->user_regs.es = __USER_DS;
291	ctxt->user_regs.ss = __KERNEL_DS;
292	ctxt->user_regs.cs = __KERNEL_CS;
293	ctxt->user_regs.esp = (unsigned long)task_pt_regs(idle);
294
295	xen_copy_trap_info(ctxt->trap_ctxt);
296
297	ctxt->ldt_ents = 0;
298
299	BUG_ON((unsigned long)gdt & ~PAGE_MASK);
300
301	gdt_mfn = arbitrary_virt_to_mfn(gdt);
302	make_lowmem_page_readonly(gdt);
303	make_lowmem_page_readonly(mfn_to_virt(gdt_mfn));
304
305	ctxt->gdt_frames[0] = gdt_mfn;
306	ctxt->gdt_ents      = GDT_ENTRIES;
307
308	/*
309	 * Set SS:SP that Xen will use when entering guest kernel mode
310	 * from guest user mode.  Subsequent calls to load_sp0() can
311	 * change this value.
312	 */
313	ctxt->kernel_ss = __KERNEL_DS;
314	ctxt->kernel_sp = task_top_of_stack(idle);
315
316	ctxt->gs_base_kernel = per_cpu_offset(cpu);
317	ctxt->event_callback_eip    =
318		(unsigned long)xen_asm_exc_xen_hypervisor_callback;
319	ctxt->failsafe_callback_eip =
320		(unsigned long)xen_failsafe_callback;
321	per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir);
322
323	ctxt->ctrlreg[3] = xen_pfn_to_cr3(virt_to_gfn(swapper_pg_dir));
324	if (HYPERVISOR_vcpu_op(VCPUOP_initialise, xen_vcpu_nr(cpu), ctxt))
325		BUG();
326
327	kfree(ctxt);
328	return 0;
329}
330
331static int xen_pv_cpu_up(unsigned int cpu, struct task_struct *idle)
332{
333	int rc;
334
335	rc = common_cpu_up(cpu, idle);
336	if (rc)
337		return rc;
338
339	xen_setup_runstate_info(cpu);
340
341	/*
342	 * PV VCPUs are always successfully taken down (see 'while' loop
343	 * in xen_cpu_die()), so -EBUSY is an error.
344	 */
345	rc = cpu_check_up_prepare(cpu);
346	if (rc)
347		return rc;
348
349	/* make sure interrupts start blocked */
350	per_cpu(xen_vcpu, cpu)->evtchn_upcall_mask = 1;
351
352	rc = cpu_initialize_context(cpu, idle);
353	if (rc)
354		return rc;
355
356	xen_pmu_init(cpu);
357
358	rc = HYPERVISOR_vcpu_op(VCPUOP_up, xen_vcpu_nr(cpu), NULL);
359	BUG_ON(rc);
360
361	while (cpu_report_state(cpu) != CPU_ONLINE)
362		HYPERVISOR_sched_op(SCHEDOP_yield, NULL);
363
364	return 0;
365}
366
367#ifdef CONFIG_HOTPLUG_CPU
368static int xen_pv_cpu_disable(void)
369{
370	unsigned int cpu = smp_processor_id();
371	if (cpu == 0)
372		return -EBUSY;
373
374	cpu_disable_common();
375
376	load_cr3(swapper_pg_dir);
377	return 0;
378}
379
380static void xen_pv_cpu_die(unsigned int cpu)
381{
382	while (HYPERVISOR_vcpu_op(VCPUOP_is_up,
383				  xen_vcpu_nr(cpu), NULL)) {
384		__set_current_state(TASK_UNINTERRUPTIBLE);
385		schedule_timeout(HZ/10);
386	}
387
388	if (common_cpu_die(cpu) == 0) {
389		xen_smp_intr_free(cpu);
390		xen_uninit_lock_cpu(cpu);
391		xen_teardown_timer(cpu);
392		xen_pmu_finish(cpu);
393	}
394}
395
396static void xen_pv_play_dead(void) /* used only with HOTPLUG_CPU */
397{
398	play_dead_common();
399	HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(smp_processor_id()), NULL);
400	cpu_bringup();
401	/*
402	 * commit 4b0c0f294 (tick: Cleanup NOHZ per cpu data on cpu down)
403	 * clears certain data that the cpu_idle loop (which called us
404	 * and that we return from) expects. The only way to get that
405	 * data back is to call:
406	 */
407	tick_nohz_idle_enter();
408	tick_nohz_idle_stop_tick_protected();
409
410	cpuhp_online_idle(CPUHP_AP_ONLINE_IDLE);
411}
412
413#else /* !CONFIG_HOTPLUG_CPU */
414static int xen_pv_cpu_disable(void)
415{
416	return -ENOSYS;
417}
418
419static void xen_pv_cpu_die(unsigned int cpu)
420{
421	BUG();
422}
423
424static void xen_pv_play_dead(void)
425{
426	BUG();
427}
428
429#endif
430static void stop_self(void *v)
431{
432	int cpu = smp_processor_id();
433
434	/* make sure we're not pinning something down */
435	load_cr3(swapper_pg_dir);
436	/* should set up a minimal gdt */
437
438	set_cpu_online(cpu, false);
439
440	HYPERVISOR_vcpu_op(VCPUOP_down, xen_vcpu_nr(cpu), NULL);
441	BUG();
442}
443
444static void xen_pv_stop_other_cpus(int wait)
445{
446	smp_call_function(stop_self, NULL, wait);
447}
448
449static irqreturn_t xen_irq_work_interrupt(int irq, void *dev_id)
450{
451	irq_enter();
452	irq_work_run();
453	inc_irq_stat(apic_irq_work_irqs);
454	irq_exit();
455
456	return IRQ_HANDLED;
457}
458
459static const struct smp_ops xen_smp_ops __initconst = {
460	.smp_prepare_boot_cpu = xen_pv_smp_prepare_boot_cpu,
461	.smp_prepare_cpus = xen_pv_smp_prepare_cpus,
462	.smp_cpus_done = xen_smp_cpus_done,
463
464	.cpu_up = xen_pv_cpu_up,
465	.cpu_die = xen_pv_cpu_die,
466	.cpu_disable = xen_pv_cpu_disable,
467	.play_dead = xen_pv_play_dead,
468
469	.stop_other_cpus = xen_pv_stop_other_cpus,
470	.smp_send_reschedule = xen_smp_send_reschedule,
471
472	.send_call_func_ipi = xen_smp_send_call_function_ipi,
473	.send_call_func_single_ipi = xen_smp_send_call_function_single_ipi,
474};
475
476void __init xen_smp_init(void)
477{
478	smp_ops = xen_smp_ops;
479
480	/* Avoid searching for BIOS MP tables */
481	x86_init.mpparse.find_smp_config = x86_init_noop;
482	x86_init.mpparse.get_smp_config = _get_smp_config;
483}
484