1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * x86_pkg_temp_thermal driver
4 * Copyright (c) 2013, Intel Corporation.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/module.h>
9#include <linux/init.h>
10#include <linux/err.h>
11#include <linux/param.h>
12#include <linux/device.h>
13#include <linux/platform_device.h>
14#include <linux/cpu.h>
15#include <linux/smp.h>
16#include <linux/slab.h>
17#include <linux/pm.h>
18#include <linux/thermal.h>
19#include <linux/debugfs.h>
20#include <asm/cpu_device_id.h>
21#include <asm/mce.h>
22
23/*
24* Rate control delay: Idea is to introduce denounce effect
25* This should be long enough to avoid reduce events, when
26* threshold is set to a temperature, which is constantly
27* violated, but at the short enough to take any action.
28* The action can be remove threshold or change it to next
29* interesting setting. Based on experiments, in around
30* every 5 seconds under load will give us a significant
31* temperature change.
32*/
33#define PKG_TEMP_THERMAL_NOTIFY_DELAY	5000
34static int notify_delay_ms = PKG_TEMP_THERMAL_NOTIFY_DELAY;
35module_param(notify_delay_ms, int, 0644);
36MODULE_PARM_DESC(notify_delay_ms,
37	"User space notification delay in milli seconds.");
38
39/* Number of trip points in thermal zone. Currently it can't
40* be more than 2. MSR can allow setting and getting notifications
41* for only 2 thresholds. This define enforces this, if there
42* is some wrong values returned by cpuid for number of thresholds.
43*/
44#define MAX_NUMBER_OF_TRIPS	2
45
46struct zone_device {
47	int				cpu;
48	bool				work_scheduled;
49	u32				tj_max;
50	u32				msr_pkg_therm_low;
51	u32				msr_pkg_therm_high;
52	struct delayed_work		work;
53	struct thermal_zone_device	*tzone;
54	struct cpumask			cpumask;
55};
56
57static struct thermal_zone_params pkg_temp_tz_params = {
58	.no_hwmon	= true,
59};
60
61/* Keep track of how many zone pointers we allocated in init() */
62static int max_id __read_mostly;
63/* Array of zone pointers */
64static struct zone_device **zones;
65/* Serializes interrupt notification, work and hotplug */
66static DEFINE_RAW_SPINLOCK(pkg_temp_lock);
67/* Protects zone operation in the work function against hotplug removal */
68static DEFINE_MUTEX(thermal_zone_mutex);
69
70/* The dynamically assigned cpu hotplug state for module_exit() */
71static enum cpuhp_state pkg_thermal_hp_state __read_mostly;
72
73/* Debug counters to show using debugfs */
74static struct dentry *debugfs;
75static unsigned int pkg_interrupt_cnt;
76static unsigned int pkg_work_cnt;
77
78static void pkg_temp_debugfs_init(void)
79{
80	debugfs = debugfs_create_dir("pkg_temp_thermal", NULL);
81
82	debugfs_create_u32("pkg_thres_interrupt", S_IRUGO, debugfs,
83			   &pkg_interrupt_cnt);
84	debugfs_create_u32("pkg_thres_work", S_IRUGO, debugfs,
85			   &pkg_work_cnt);
86}
87
88/*
89 * Protection:
90 *
91 * - cpu hotplug: Read serialized by cpu hotplug lock
92 *		  Write must hold pkg_temp_lock
93 *
94 * - Other callsites: Must hold pkg_temp_lock
95 */
96static struct zone_device *pkg_temp_thermal_get_dev(unsigned int cpu)
97{
98	int id = topology_logical_die_id(cpu);
99
100	if (id >= 0 && id < max_id)
101		return zones[id];
102	return NULL;
103}
104
105/*
106* tj-max is is interesting because threshold is set relative to this
107* temperature.
108*/
109static int get_tj_max(int cpu, u32 *tj_max)
110{
111	u32 eax, edx, val;
112	int err;
113
114	err = rdmsr_safe_on_cpu(cpu, MSR_IA32_TEMPERATURE_TARGET, &eax, &edx);
115	if (err)
116		return err;
117
118	val = (eax >> 16) & 0xff;
119	*tj_max = val * 1000;
120
121	return val ? 0 : -EINVAL;
122}
123
124static int sys_get_curr_temp(struct thermal_zone_device *tzd, int *temp)
125{
126	struct zone_device *zonedev = tzd->devdata;
127	u32 eax, edx;
128
129	rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_STATUS,
130			&eax, &edx);
131	if (eax & 0x80000000) {
132		*temp = zonedev->tj_max - ((eax >> 16) & 0x7f) * 1000;
133		pr_debug("sys_get_curr_temp %d\n", *temp);
134		return 0;
135	}
136	return -EINVAL;
137}
138
139static int sys_get_trip_temp(struct thermal_zone_device *tzd,
140			     int trip, int *temp)
141{
142	struct zone_device *zonedev = tzd->devdata;
143	unsigned long thres_reg_value;
144	u32 mask, shift, eax, edx;
145	int ret;
146
147	if (trip >= MAX_NUMBER_OF_TRIPS)
148		return -EINVAL;
149
150	if (trip) {
151		mask = THERM_MASK_THRESHOLD1;
152		shift = THERM_SHIFT_THRESHOLD1;
153	} else {
154		mask = THERM_MASK_THRESHOLD0;
155		shift = THERM_SHIFT_THRESHOLD0;
156	}
157
158	ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
159			   &eax, &edx);
160	if (ret < 0)
161		return ret;
162
163	thres_reg_value = (eax & mask) >> shift;
164	if (thres_reg_value)
165		*temp = zonedev->tj_max - thres_reg_value * 1000;
166	else
167		*temp = THERMAL_TEMP_INVALID;
168	pr_debug("sys_get_trip_temp %d\n", *temp);
169
170	return 0;
171}
172
173static int
174sys_set_trip_temp(struct thermal_zone_device *tzd, int trip, int temp)
175{
176	struct zone_device *zonedev = tzd->devdata;
177	u32 l, h, mask, shift, intr;
178	int ret;
179
180	if (trip >= MAX_NUMBER_OF_TRIPS || temp >= zonedev->tj_max)
181		return -EINVAL;
182
183	ret = rdmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
184			   &l, &h);
185	if (ret < 0)
186		return ret;
187
188	if (trip) {
189		mask = THERM_MASK_THRESHOLD1;
190		shift = THERM_SHIFT_THRESHOLD1;
191		intr = THERM_INT_THRESHOLD1_ENABLE;
192	} else {
193		mask = THERM_MASK_THRESHOLD0;
194		shift = THERM_SHIFT_THRESHOLD0;
195		intr = THERM_INT_THRESHOLD0_ENABLE;
196	}
197	l &= ~mask;
198	/*
199	* When users space sets a trip temperature == 0, which is indication
200	* that, it is no longer interested in receiving notifications.
201	*/
202	if (!temp) {
203		l &= ~intr;
204	} else {
205		l |= (zonedev->tj_max - temp)/1000 << shift;
206		l |= intr;
207	}
208
209	return wrmsr_on_cpu(zonedev->cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT,
210			l, h);
211}
212
213static int sys_get_trip_type(struct thermal_zone_device *thermal, int trip,
214			     enum thermal_trip_type *type)
215{
216	*type = THERMAL_TRIP_PASSIVE;
217	return 0;
218}
219
220/* Thermal zone callback registry */
221static struct thermal_zone_device_ops tzone_ops = {
222	.get_temp = sys_get_curr_temp,
223	.get_trip_temp = sys_get_trip_temp,
224	.get_trip_type = sys_get_trip_type,
225	.set_trip_temp = sys_set_trip_temp,
226};
227
228static bool pkg_thermal_rate_control(void)
229{
230	return true;
231}
232
233/* Enable threshold interrupt on local package/cpu */
234static inline void enable_pkg_thres_interrupt(void)
235{
236	u8 thres_0, thres_1;
237	u32 l, h;
238
239	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
240	/* only enable/disable if it had valid threshold value */
241	thres_0 = (l & THERM_MASK_THRESHOLD0) >> THERM_SHIFT_THRESHOLD0;
242	thres_1 = (l & THERM_MASK_THRESHOLD1) >> THERM_SHIFT_THRESHOLD1;
243	if (thres_0)
244		l |= THERM_INT_THRESHOLD0_ENABLE;
245	if (thres_1)
246		l |= THERM_INT_THRESHOLD1_ENABLE;
247	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
248}
249
250/* Disable threshold interrupt on local package/cpu */
251static inline void disable_pkg_thres_interrupt(void)
252{
253	u32 l, h;
254
255	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
256
257	l &= ~(THERM_INT_THRESHOLD0_ENABLE | THERM_INT_THRESHOLD1_ENABLE);
258	wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, l, h);
259}
260
261static void pkg_temp_thermal_threshold_work_fn(struct work_struct *work)
262{
263	struct thermal_zone_device *tzone = NULL;
264	int cpu = smp_processor_id();
265	struct zone_device *zonedev;
266	u64 msr_val, wr_val;
267
268	mutex_lock(&thermal_zone_mutex);
269	raw_spin_lock_irq(&pkg_temp_lock);
270	++pkg_work_cnt;
271
272	zonedev = pkg_temp_thermal_get_dev(cpu);
273	if (!zonedev) {
274		raw_spin_unlock_irq(&pkg_temp_lock);
275		mutex_unlock(&thermal_zone_mutex);
276		return;
277	}
278	zonedev->work_scheduled = false;
279
280	rdmsrl(MSR_IA32_PACKAGE_THERM_STATUS, msr_val);
281	wr_val = msr_val & ~(THERM_LOG_THRESHOLD0 | THERM_LOG_THRESHOLD1);
282	if (wr_val != msr_val) {
283		wrmsrl(MSR_IA32_PACKAGE_THERM_STATUS, wr_val);
284		tzone = zonedev->tzone;
285	}
286
287	enable_pkg_thres_interrupt();
288	raw_spin_unlock_irq(&pkg_temp_lock);
289
290	/*
291	 * If tzone is not NULL, then thermal_zone_mutex will prevent the
292	 * concurrent removal in the cpu offline callback.
293	 */
294	if (tzone)
295		thermal_zone_device_update(tzone, THERMAL_EVENT_UNSPECIFIED);
296
297	mutex_unlock(&thermal_zone_mutex);
298}
299
300static void pkg_thermal_schedule_work(int cpu, struct delayed_work *work)
301{
302	unsigned long ms = msecs_to_jiffies(notify_delay_ms);
303
304	schedule_delayed_work_on(cpu, work, ms);
305}
306
307static int pkg_thermal_notify(u64 msr_val)
308{
309	int cpu = smp_processor_id();
310	struct zone_device *zonedev;
311	unsigned long flags;
312
313	raw_spin_lock_irqsave(&pkg_temp_lock, flags);
314	++pkg_interrupt_cnt;
315
316	disable_pkg_thres_interrupt();
317
318	/* Work is per package, so scheduling it once is enough. */
319	zonedev = pkg_temp_thermal_get_dev(cpu);
320	if (zonedev && !zonedev->work_scheduled) {
321		zonedev->work_scheduled = true;
322		pkg_thermal_schedule_work(zonedev->cpu, &zonedev->work);
323	}
324
325	raw_spin_unlock_irqrestore(&pkg_temp_lock, flags);
326	return 0;
327}
328
329static int pkg_temp_thermal_device_add(unsigned int cpu)
330{
331	int id = topology_logical_die_id(cpu);
332	u32 tj_max, eax, ebx, ecx, edx;
333	struct zone_device *zonedev;
334	int thres_count, err;
335
336	if (id >= max_id)
337		return -ENOMEM;
338
339	cpuid(6, &eax, &ebx, &ecx, &edx);
340	thres_count = ebx & 0x07;
341	if (!thres_count)
342		return -ENODEV;
343
344	thres_count = clamp_val(thres_count, 0, MAX_NUMBER_OF_TRIPS);
345
346	err = get_tj_max(cpu, &tj_max);
347	if (err)
348		return err;
349
350	zonedev = kzalloc(sizeof(*zonedev), GFP_KERNEL);
351	if (!zonedev)
352		return -ENOMEM;
353
354	INIT_DELAYED_WORK(&zonedev->work, pkg_temp_thermal_threshold_work_fn);
355	zonedev->cpu = cpu;
356	zonedev->tj_max = tj_max;
357	zonedev->tzone = thermal_zone_device_register("x86_pkg_temp",
358			thres_count,
359			(thres_count == MAX_NUMBER_OF_TRIPS) ? 0x03 : 0x01,
360			zonedev, &tzone_ops, &pkg_temp_tz_params, 0, 0);
361	if (IS_ERR(zonedev->tzone)) {
362		err = PTR_ERR(zonedev->tzone);
363		kfree(zonedev);
364		return err;
365	}
366	err = thermal_zone_device_enable(zonedev->tzone);
367	if (err) {
368		thermal_zone_device_unregister(zonedev->tzone);
369		kfree(zonedev);
370		return err;
371	}
372	/* Store MSR value for package thermal interrupt, to restore at exit */
373	rdmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT, zonedev->msr_pkg_therm_low,
374	      zonedev->msr_pkg_therm_high);
375
376	cpumask_set_cpu(cpu, &zonedev->cpumask);
377	raw_spin_lock_irq(&pkg_temp_lock);
378	zones[id] = zonedev;
379	raw_spin_unlock_irq(&pkg_temp_lock);
380	return 0;
381}
382
383static int pkg_thermal_cpu_offline(unsigned int cpu)
384{
385	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
386	bool lastcpu, was_target;
387	int target;
388
389	if (!zonedev)
390		return 0;
391
392	target = cpumask_any_but(&zonedev->cpumask, cpu);
393	cpumask_clear_cpu(cpu, &zonedev->cpumask);
394	lastcpu = target >= nr_cpu_ids;
395	/*
396	 * Remove the sysfs files, if this is the last cpu in the package
397	 * before doing further cleanups.
398	 */
399	if (lastcpu) {
400		struct thermal_zone_device *tzone = zonedev->tzone;
401
402		/*
403		 * We must protect against a work function calling
404		 * thermal_zone_update, after/while unregister. We null out
405		 * the pointer under the zone mutex, so the worker function
406		 * won't try to call.
407		 */
408		mutex_lock(&thermal_zone_mutex);
409		zonedev->tzone = NULL;
410		mutex_unlock(&thermal_zone_mutex);
411
412		thermal_zone_device_unregister(tzone);
413	}
414
415	/* Protect against work and interrupts */
416	raw_spin_lock_irq(&pkg_temp_lock);
417
418	/*
419	 * Check whether this cpu was the current target and store the new
420	 * one. When we drop the lock, then the interrupt notify function
421	 * will see the new target.
422	 */
423	was_target = zonedev->cpu == cpu;
424	zonedev->cpu = target;
425
426	/*
427	 * If this is the last CPU in the package remove the package
428	 * reference from the array and restore the interrupt MSR. When we
429	 * drop the lock neither the interrupt notify function nor the
430	 * worker will see the package anymore.
431	 */
432	if (lastcpu) {
433		zones[topology_logical_die_id(cpu)] = NULL;
434		/* After this point nothing touches the MSR anymore. */
435		wrmsr(MSR_IA32_PACKAGE_THERM_INTERRUPT,
436		      zonedev->msr_pkg_therm_low, zonedev->msr_pkg_therm_high);
437	}
438
439	/*
440	 * Check whether there is work scheduled and whether the work is
441	 * targeted at the outgoing CPU.
442	 */
443	if (zonedev->work_scheduled && was_target) {
444		/*
445		 * To cancel the work we need to drop the lock, otherwise
446		 * we might deadlock if the work needs to be flushed.
447		 */
448		raw_spin_unlock_irq(&pkg_temp_lock);
449		cancel_delayed_work_sync(&zonedev->work);
450		raw_spin_lock_irq(&pkg_temp_lock);
451		/*
452		 * If this is not the last cpu in the package and the work
453		 * did not run after we dropped the lock above, then we
454		 * need to reschedule the work, otherwise the interrupt
455		 * stays disabled forever.
456		 */
457		if (!lastcpu && zonedev->work_scheduled)
458			pkg_thermal_schedule_work(target, &zonedev->work);
459	}
460
461	raw_spin_unlock_irq(&pkg_temp_lock);
462
463	/* Final cleanup if this is the last cpu */
464	if (lastcpu)
465		kfree(zonedev);
466	return 0;
467}
468
469static int pkg_thermal_cpu_online(unsigned int cpu)
470{
471	struct zone_device *zonedev = pkg_temp_thermal_get_dev(cpu);
472	struct cpuinfo_x86 *c = &cpu_data(cpu);
473
474	/* Paranoia check */
475	if (!cpu_has(c, X86_FEATURE_DTHERM) || !cpu_has(c, X86_FEATURE_PTS))
476		return -ENODEV;
477
478	/* If the package exists, nothing to do */
479	if (zonedev) {
480		cpumask_set_cpu(cpu, &zonedev->cpumask);
481		return 0;
482	}
483	return pkg_temp_thermal_device_add(cpu);
484}
485
486static const struct x86_cpu_id __initconst pkg_temp_thermal_ids[] = {
487	X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_PTS, NULL),
488	{}
489};
490MODULE_DEVICE_TABLE(x86cpu, pkg_temp_thermal_ids);
491
492static int __init pkg_temp_thermal_init(void)
493{
494	int ret;
495
496	if (!x86_match_cpu(pkg_temp_thermal_ids))
497		return -ENODEV;
498
499	max_id = topology_max_packages() * topology_max_die_per_package();
500	zones = kcalloc(max_id, sizeof(struct zone_device *),
501			   GFP_KERNEL);
502	if (!zones)
503		return -ENOMEM;
504
505	ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "thermal/x86_pkg:online",
506				pkg_thermal_cpu_online,	pkg_thermal_cpu_offline);
507	if (ret < 0)
508		goto err;
509
510	/* Store the state for module exit */
511	pkg_thermal_hp_state = ret;
512
513	platform_thermal_package_notify = pkg_thermal_notify;
514	platform_thermal_package_rate_control = pkg_thermal_rate_control;
515
516	 /* Don't care if it fails */
517	pkg_temp_debugfs_init();
518	return 0;
519
520err:
521	kfree(zones);
522	return ret;
523}
524module_init(pkg_temp_thermal_init)
525
526static void __exit pkg_temp_thermal_exit(void)
527{
528	platform_thermal_package_notify = NULL;
529	platform_thermal_package_rate_control = NULL;
530
531	cpuhp_remove_state(pkg_thermal_hp_state);
532	debugfs_remove_recursive(debugfs);
533	kfree(zones);
534}
535module_exit(pkg_temp_thermal_exit)
536
537MODULE_DESCRIPTION("X86 PKG TEMP Thermal Driver");
538MODULE_AUTHOR("Srinivas Pandruvada <srinivas.pandruvada@linux.intel.com>");
539MODULE_LICENSE("GPL v2");
540