1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * acpi_pad.c ACPI Processor Aggregator Driver
4 *
5 * Copyright (c) 2009, Intel Corporation.
6 */
7
8#include <linux/kernel.h>
9#include <linux/cpumask.h>
10#include <linux/module.h>
11#include <linux/init.h>
12#include <linux/types.h>
13#include <linux/kthread.h>
14#include <uapi/linux/sched/types.h>
15#include <linux/freezer.h>
16#include <linux/cpu.h>
17#include <linux/tick.h>
18#include <linux/slab.h>
19#include <linux/acpi.h>
20#include <asm/mwait.h>
21#include <xen/xen.h>
22
23#define ACPI_PROCESSOR_AGGREGATOR_CLASS	"acpi_pad"
24#define ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME "Processor Aggregator"
25#define ACPI_PROCESSOR_AGGREGATOR_NOTIFY 0x80
26static DEFINE_MUTEX(isolated_cpus_lock);
27static DEFINE_MUTEX(round_robin_lock);
28
29static unsigned long power_saving_mwait_eax;
30
31static unsigned char tsc_detected_unstable;
32static unsigned char tsc_marked_unstable;
33
34static void power_saving_mwait_init(void)
35{
36	unsigned int eax, ebx, ecx, edx;
37	unsigned int highest_cstate = 0;
38	unsigned int highest_subcstate = 0;
39	int i;
40
41	if (!boot_cpu_has(X86_FEATURE_MWAIT))
42		return;
43	if (boot_cpu_data.cpuid_level < CPUID_MWAIT_LEAF)
44		return;
45
46	cpuid(CPUID_MWAIT_LEAF, &eax, &ebx, &ecx, &edx);
47
48	if (!(ecx & CPUID5_ECX_EXTENSIONS_SUPPORTED) ||
49	    !(ecx & CPUID5_ECX_INTERRUPT_BREAK))
50		return;
51
52	edx >>= MWAIT_SUBSTATE_SIZE;
53	for (i = 0; i < 7 && edx; i++, edx >>= MWAIT_SUBSTATE_SIZE) {
54		if (edx & MWAIT_SUBSTATE_MASK) {
55			highest_cstate = i;
56			highest_subcstate = edx & MWAIT_SUBSTATE_MASK;
57		}
58	}
59	power_saving_mwait_eax = (highest_cstate << MWAIT_SUBSTATE_SIZE) |
60		(highest_subcstate - 1);
61
62#if defined(CONFIG_X86)
63	switch (boot_cpu_data.x86_vendor) {
64	case X86_VENDOR_HYGON:
65	case X86_VENDOR_AMD:
66	case X86_VENDOR_INTEL:
67	case X86_VENDOR_ZHAOXIN:
68		/*
69		 * AMD Fam10h TSC will tick in all
70		 * C/P/S0/S1 states when this bit is set.
71		 */
72		if (!boot_cpu_has(X86_FEATURE_NONSTOP_TSC))
73			tsc_detected_unstable = 1;
74		break;
75	default:
76		/* TSC could halt in idle */
77		tsc_detected_unstable = 1;
78	}
79#endif
80}
81
82static unsigned long cpu_weight[NR_CPUS];
83static int tsk_in_cpu[NR_CPUS] = {[0 ... NR_CPUS-1] = -1};
84static DECLARE_BITMAP(pad_busy_cpus_bits, NR_CPUS);
85static void round_robin_cpu(unsigned int tsk_index)
86{
87	struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
88	cpumask_var_t tmp;
89	int cpu;
90	unsigned long min_weight = -1;
91	unsigned long preferred_cpu;
92
93	if (!alloc_cpumask_var(&tmp, GFP_KERNEL))
94		return;
95
96	mutex_lock(&round_robin_lock);
97	cpumask_clear(tmp);
98	for_each_cpu(cpu, pad_busy_cpus)
99		cpumask_or(tmp, tmp, topology_sibling_cpumask(cpu));
100	cpumask_andnot(tmp, cpu_online_mask, tmp);
101	/* avoid HT sibilings if possible */
102	if (cpumask_empty(tmp))
103		cpumask_andnot(tmp, cpu_online_mask, pad_busy_cpus);
104	if (cpumask_empty(tmp)) {
105		mutex_unlock(&round_robin_lock);
106		free_cpumask_var(tmp);
107		return;
108	}
109	for_each_cpu(cpu, tmp) {
110		if (cpu_weight[cpu] < min_weight) {
111			min_weight = cpu_weight[cpu];
112			preferred_cpu = cpu;
113		}
114	}
115
116	if (tsk_in_cpu[tsk_index] != -1)
117		cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
118	tsk_in_cpu[tsk_index] = preferred_cpu;
119	cpumask_set_cpu(preferred_cpu, pad_busy_cpus);
120	cpu_weight[preferred_cpu]++;
121	mutex_unlock(&round_robin_lock);
122
123	set_cpus_allowed_ptr(current, cpumask_of(preferred_cpu));
124
125	free_cpumask_var(tmp);
126}
127
128static void exit_round_robin(unsigned int tsk_index)
129{
130	struct cpumask *pad_busy_cpus = to_cpumask(pad_busy_cpus_bits);
131	cpumask_clear_cpu(tsk_in_cpu[tsk_index], pad_busy_cpus);
132	tsk_in_cpu[tsk_index] = -1;
133}
134
135static unsigned int idle_pct = 5; /* percentage */
136static unsigned int round_robin_time = 1; /* second */
137static int power_saving_thread(void *data)
138{
139	int do_sleep;
140	unsigned int tsk_index = (unsigned long)data;
141	u64 last_jiffies = 0;
142
143	sched_set_fifo_low(current);
144
145	while (!kthread_should_stop()) {
146		unsigned long expire_time;
147
148		/* round robin to cpus */
149		expire_time = last_jiffies + round_robin_time * HZ;
150		if (time_before(expire_time, jiffies)) {
151			last_jiffies = jiffies;
152			round_robin_cpu(tsk_index);
153		}
154
155		do_sleep = 0;
156
157		expire_time = jiffies + HZ * (100 - idle_pct) / 100;
158
159		while (!need_resched()) {
160			if (tsc_detected_unstable && !tsc_marked_unstable) {
161				/* TSC could halt in idle, so notify users */
162				mark_tsc_unstable("TSC halts in idle");
163				tsc_marked_unstable = 1;
164			}
165			local_irq_disable();
166			tick_broadcast_enable();
167			tick_broadcast_enter();
168			stop_critical_timings();
169
170			mwait_idle_with_hints(power_saving_mwait_eax, 1);
171
172			start_critical_timings();
173			tick_broadcast_exit();
174			local_irq_enable();
175
176			if (time_before(expire_time, jiffies)) {
177				do_sleep = 1;
178				break;
179			}
180		}
181
182		/*
183		 * current sched_rt has threshold for rt task running time.
184		 * When a rt task uses 95% CPU time, the rt thread will be
185		 * scheduled out for 5% CPU time to not starve other tasks. But
186		 * the mechanism only works when all CPUs have RT task running,
187		 * as if one CPU hasn't RT task, RT task from other CPUs will
188		 * borrow CPU time from this CPU and cause RT task use > 95%
189		 * CPU time. To make 'avoid starvation' work, takes a nap here.
190		 */
191		if (unlikely(do_sleep))
192			schedule_timeout_killable(HZ * idle_pct / 100);
193
194		/* If an external event has set the need_resched flag, then
195		 * we need to deal with it, or this loop will continue to
196		 * spin without calling __mwait().
197		 */
198		if (unlikely(need_resched()))
199			schedule();
200	}
201
202	exit_round_robin(tsk_index);
203	return 0;
204}
205
206static struct task_struct *ps_tsks[NR_CPUS];
207static unsigned int ps_tsk_num;
208static int create_power_saving_task(void)
209{
210	int rc;
211
212	ps_tsks[ps_tsk_num] = kthread_run(power_saving_thread,
213		(void *)(unsigned long)ps_tsk_num,
214		"acpi_pad/%d", ps_tsk_num);
215
216	if (IS_ERR(ps_tsks[ps_tsk_num])) {
217		rc = PTR_ERR(ps_tsks[ps_tsk_num]);
218		ps_tsks[ps_tsk_num] = NULL;
219	} else {
220		rc = 0;
221		ps_tsk_num++;
222	}
223
224	return rc;
225}
226
227static void destroy_power_saving_task(void)
228{
229	if (ps_tsk_num > 0) {
230		ps_tsk_num--;
231		kthread_stop(ps_tsks[ps_tsk_num]);
232		ps_tsks[ps_tsk_num] = NULL;
233	}
234}
235
236static void set_power_saving_task_num(unsigned int num)
237{
238	if (num > ps_tsk_num) {
239		while (ps_tsk_num < num) {
240			if (create_power_saving_task())
241				return;
242		}
243	} else if (num < ps_tsk_num) {
244		while (ps_tsk_num > num)
245			destroy_power_saving_task();
246	}
247}
248
249static void acpi_pad_idle_cpus(unsigned int num_cpus)
250{
251	get_online_cpus();
252
253	num_cpus = min_t(unsigned int, num_cpus, num_online_cpus());
254	set_power_saving_task_num(num_cpus);
255
256	put_online_cpus();
257}
258
259static uint32_t acpi_pad_idle_cpus_num(void)
260{
261	return ps_tsk_num;
262}
263
264static ssize_t rrtime_store(struct device *dev,
265	struct device_attribute *attr, const char *buf, size_t count)
266{
267	unsigned long num;
268	if (kstrtoul(buf, 0, &num))
269		return -EINVAL;
270	if (num < 1 || num >= 100)
271		return -EINVAL;
272	mutex_lock(&isolated_cpus_lock);
273	round_robin_time = num;
274	mutex_unlock(&isolated_cpus_lock);
275	return count;
276}
277
278static ssize_t rrtime_show(struct device *dev,
279	struct device_attribute *attr, char *buf)
280{
281	return scnprintf(buf, PAGE_SIZE, "%d\n", round_robin_time);
282}
283static DEVICE_ATTR_RW(rrtime);
284
285static ssize_t idlepct_store(struct device *dev,
286	struct device_attribute *attr, const char *buf, size_t count)
287{
288	unsigned long num;
289	if (kstrtoul(buf, 0, &num))
290		return -EINVAL;
291	if (num < 1 || num >= 100)
292		return -EINVAL;
293	mutex_lock(&isolated_cpus_lock);
294	idle_pct = num;
295	mutex_unlock(&isolated_cpus_lock);
296	return count;
297}
298
299static ssize_t idlepct_show(struct device *dev,
300	struct device_attribute *attr, char *buf)
301{
302	return scnprintf(buf, PAGE_SIZE, "%d\n", idle_pct);
303}
304static DEVICE_ATTR_RW(idlepct);
305
306static ssize_t idlecpus_store(struct device *dev,
307	struct device_attribute *attr, const char *buf, size_t count)
308{
309	unsigned long num;
310	if (kstrtoul(buf, 0, &num))
311		return -EINVAL;
312	mutex_lock(&isolated_cpus_lock);
313	acpi_pad_idle_cpus(num);
314	mutex_unlock(&isolated_cpus_lock);
315	return count;
316}
317
318static ssize_t idlecpus_show(struct device *dev,
319	struct device_attribute *attr, char *buf)
320{
321	return cpumap_print_to_pagebuf(false, buf,
322				       to_cpumask(pad_busy_cpus_bits));
323}
324
325static DEVICE_ATTR_RW(idlecpus);
326
327static int acpi_pad_add_sysfs(struct acpi_device *device)
328{
329	int result;
330
331	result = device_create_file(&device->dev, &dev_attr_idlecpus);
332	if (result)
333		return -ENODEV;
334	result = device_create_file(&device->dev, &dev_attr_idlepct);
335	if (result) {
336		device_remove_file(&device->dev, &dev_attr_idlecpus);
337		return -ENODEV;
338	}
339	result = device_create_file(&device->dev, &dev_attr_rrtime);
340	if (result) {
341		device_remove_file(&device->dev, &dev_attr_idlecpus);
342		device_remove_file(&device->dev, &dev_attr_idlepct);
343		return -ENODEV;
344	}
345	return 0;
346}
347
348static void acpi_pad_remove_sysfs(struct acpi_device *device)
349{
350	device_remove_file(&device->dev, &dev_attr_idlecpus);
351	device_remove_file(&device->dev, &dev_attr_idlepct);
352	device_remove_file(&device->dev, &dev_attr_rrtime);
353}
354
355/*
356 * Query firmware how many CPUs should be idle
357 * return -1 on failure
358 */
359static int acpi_pad_pur(acpi_handle handle)
360{
361	struct acpi_buffer buffer = {ACPI_ALLOCATE_BUFFER, NULL};
362	union acpi_object *package;
363	int num = -1;
364
365	if (ACPI_FAILURE(acpi_evaluate_object(handle, "_PUR", NULL, &buffer)))
366		return num;
367
368	if (!buffer.length || !buffer.pointer)
369		return num;
370
371	package = buffer.pointer;
372
373	if (package->type == ACPI_TYPE_PACKAGE &&
374		package->package.count == 2 &&
375		package->package.elements[0].integer.value == 1) /* rev 1 */
376
377		num = package->package.elements[1].integer.value;
378
379	kfree(buffer.pointer);
380	return num;
381}
382
383static void acpi_pad_handle_notify(acpi_handle handle)
384{
385	int num_cpus;
386	uint32_t idle_cpus;
387	struct acpi_buffer param = {
388		.length = 4,
389		.pointer = (void *)&idle_cpus,
390	};
391
392	mutex_lock(&isolated_cpus_lock);
393	num_cpus = acpi_pad_pur(handle);
394	if (num_cpus < 0) {
395		mutex_unlock(&isolated_cpus_lock);
396		return;
397	}
398	acpi_pad_idle_cpus(num_cpus);
399	idle_cpus = acpi_pad_idle_cpus_num();
400	acpi_evaluate_ost(handle, ACPI_PROCESSOR_AGGREGATOR_NOTIFY, 0, &param);
401	mutex_unlock(&isolated_cpus_lock);
402}
403
404static void acpi_pad_notify(acpi_handle handle, u32 event,
405	void *data)
406{
407	struct acpi_device *device = data;
408
409	switch (event) {
410	case ACPI_PROCESSOR_AGGREGATOR_NOTIFY:
411		acpi_pad_handle_notify(handle);
412		acpi_bus_generate_netlink_event(device->pnp.device_class,
413			dev_name(&device->dev), event, 0);
414		break;
415	default:
416		pr_warn("Unsupported event [0x%x]\n", event);
417		break;
418	}
419}
420
421static int acpi_pad_add(struct acpi_device *device)
422{
423	acpi_status status;
424
425	strcpy(acpi_device_name(device), ACPI_PROCESSOR_AGGREGATOR_DEVICE_NAME);
426	strcpy(acpi_device_class(device), ACPI_PROCESSOR_AGGREGATOR_CLASS);
427
428	if (acpi_pad_add_sysfs(device))
429		return -ENODEV;
430
431	status = acpi_install_notify_handler(device->handle,
432		ACPI_DEVICE_NOTIFY, acpi_pad_notify, device);
433	if (ACPI_FAILURE(status)) {
434		acpi_pad_remove_sysfs(device);
435		return -ENODEV;
436	}
437
438	return 0;
439}
440
441static int acpi_pad_remove(struct acpi_device *device)
442{
443	mutex_lock(&isolated_cpus_lock);
444	acpi_pad_idle_cpus(0);
445	mutex_unlock(&isolated_cpus_lock);
446
447	acpi_remove_notify_handler(device->handle,
448		ACPI_DEVICE_NOTIFY, acpi_pad_notify);
449	acpi_pad_remove_sysfs(device);
450	return 0;
451}
452
453static const struct acpi_device_id pad_device_ids[] = {
454	{"ACPI000C", 0},
455	{"", 0},
456};
457MODULE_DEVICE_TABLE(acpi, pad_device_ids);
458
459static struct acpi_driver acpi_pad_driver = {
460	.name = "processor_aggregator",
461	.class = ACPI_PROCESSOR_AGGREGATOR_CLASS,
462	.ids = pad_device_ids,
463	.ops = {
464		.add = acpi_pad_add,
465		.remove = acpi_pad_remove,
466	},
467};
468
469static int __init acpi_pad_init(void)
470{
471	/* Xen ACPI PAD is used when running as Xen Dom0. */
472	if (xen_initial_domain())
473		return -ENODEV;
474
475	power_saving_mwait_init();
476	if (power_saving_mwait_eax == 0)
477		return -EINVAL;
478
479	return acpi_bus_register_driver(&acpi_pad_driver);
480}
481
482static void __exit acpi_pad_exit(void)
483{
484	acpi_bus_unregister_driver(&acpi_pad_driver);
485}
486
487module_init(acpi_pad_init);
488module_exit(acpi_pad_exit);
489MODULE_AUTHOR("Shaohua Li<shaohua.li@intel.com>");
490MODULE_DESCRIPTION("ACPI Processor Aggregator Driver");
491MODULE_LICENSE("GPL");
492