1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * CPU idle driver for Tegra CPUs
4 *
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 * Copyright (c) 2011 Google, Inc.
7 * Author: Colin Cross <ccross@android.com>
8 *         Gary King <gking@nvidia.com>
9 *
10 * Rework for 3.3 by Peter De Schrijver <pdeschrijver@nvidia.com>
11 *
12 * Tegra20/124 driver unification by Dmitry Osipenko <digetx@gmail.com>
13 */
14
15#define pr_fmt(fmt)	"tegra-cpuidle: " fmt
16
17#include <linux/atomic.h>
18#include <linux/cpuidle.h>
19#include <linux/cpumask.h>
20#include <linux/cpu_pm.h>
21#include <linux/delay.h>
22#include <linux/errno.h>
23#include <linux/platform_device.h>
24#include <linux/types.h>
25
26#include <linux/clk/tegra.h>
27#include <linux/firmware/trusted_foundations.h>
28
29#include <soc/tegra/cpuidle.h>
30#include <soc/tegra/flowctrl.h>
31#include <soc/tegra/fuse.h>
32#include <soc/tegra/irq.h>
33#include <soc/tegra/pm.h>
34#include <soc/tegra/pmc.h>
35
36#include <asm/cpuidle.h>
37#include <asm/firmware.h>
38#include <asm/smp_plat.h>
39#include <asm/suspend.h>
40
41enum tegra_state {
42	TEGRA_C1,
43	TEGRA_C7,
44	TEGRA_CC6,
45	TEGRA_STATE_COUNT,
46};
47
48static atomic_t tegra_idle_barrier;
49static atomic_t tegra_abort_flag;
50
51static inline bool tegra_cpuidle_using_firmware(void)
52{
53	return firmware_ops->prepare_idle && firmware_ops->do_idle;
54}
55
56static void tegra_cpuidle_report_cpus_state(void)
57{
58	unsigned long cpu, lcpu, csr;
59
60	for_each_cpu(lcpu, cpu_possible_mask) {
61		cpu = cpu_logical_map(lcpu);
62		csr = flowctrl_read_cpu_csr(cpu);
63
64		pr_err("cpu%lu: online=%d flowctrl_csr=0x%08lx\n",
65		       cpu, cpu_online(lcpu), csr);
66	}
67}
68
69static int tegra_cpuidle_wait_for_secondary_cpus_parking(void)
70{
71	unsigned int retries = 3;
72
73	while (retries--) {
74		unsigned int delay_us = 10;
75		unsigned int timeout_us = 500 * 1000 / delay_us;
76
77		/*
78		 * The primary CPU0 core shall wait for the secondaries
79		 * shutdown in order to power-off CPU's cluster safely.
80		 * The timeout value depends on the current CPU frequency,
81		 * it takes about 40-150us in average and over 1000us in
82		 * a worst case scenario.
83		 */
84		do {
85			if (tegra_cpu_rail_off_ready())
86				return 0;
87
88			udelay(delay_us);
89
90		} while (timeout_us--);
91
92		pr_err("secondary CPU taking too long to park\n");
93
94		tegra_cpuidle_report_cpus_state();
95	}
96
97	pr_err("timed out waiting secondaries to park\n");
98
99	return -ETIMEDOUT;
100}
101
102static void tegra_cpuidle_unpark_secondary_cpus(void)
103{
104	unsigned int cpu, lcpu;
105
106	for_each_cpu(lcpu, cpu_online_mask) {
107		cpu = cpu_logical_map(lcpu);
108
109		if (cpu > 0) {
110			tegra_enable_cpu_clock(cpu);
111			tegra_cpu_out_of_reset(cpu);
112			flowctrl_write_cpu_halt(cpu, 0);
113		}
114	}
115}
116
117static int tegra_cpuidle_cc6_enter(unsigned int cpu)
118{
119	int ret;
120
121	if (cpu > 0) {
122		ret = cpu_suspend(cpu, tegra_pm_park_secondary_cpu);
123	} else {
124		ret = tegra_cpuidle_wait_for_secondary_cpus_parking();
125		if (!ret)
126			ret = tegra_pm_enter_lp2();
127
128		tegra_cpuidle_unpark_secondary_cpus();
129	}
130
131	return ret;
132}
133
134static int tegra_cpuidle_c7_enter(void)
135{
136	int err;
137
138	err = call_firmware_op(prepare_idle, TF_PM_MODE_LP2_NOFLUSH_L2);
139	if (err && err != -ENOSYS)
140		return err;
141
142	err = call_firmware_op(do_idle, 0);
143	if (err != -ENOSYS)
144		return err;
145
146	return cpu_suspend(0, tegra30_pm_secondary_cpu_suspend);
147}
148
149static int tegra_cpuidle_coupled_barrier(struct cpuidle_device *dev)
150{
151	if (tegra_pending_sgi()) {
152		/*
153		 * CPU got local interrupt that will be lost after GIC's
154		 * shutdown because GIC driver doesn't save/restore the
155		 * pending SGI state across CPU cluster PM.  Abort and retry
156		 * next time.
157		 */
158		atomic_set(&tegra_abort_flag, 1);
159	}
160
161	cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);
162
163	if (atomic_read(&tegra_abort_flag)) {
164		cpuidle_coupled_parallel_barrier(dev, &tegra_idle_barrier);
165		atomic_set(&tegra_abort_flag, 0);
166		return -EINTR;
167	}
168
169	return 0;
170}
171
172static int tegra_cpuidle_state_enter(struct cpuidle_device *dev,
173				     int index, unsigned int cpu)
174{
175	int err;
176
177	/*
178	 * CC6 state is the "CPU cluster power-off" state.  In order to
179	 * enter this state, at first the secondary CPU cores need to be
180	 * parked into offline mode, then the last CPU should clean out
181	 * remaining dirty cache lines into DRAM and trigger Flow Controller
182	 * logic that turns off the cluster's power domain (which includes
183	 * CPU cores, GIC and L2 cache).
184	 */
185	if (index == TEGRA_CC6) {
186		err = tegra_cpuidle_coupled_barrier(dev);
187		if (err)
188			return err;
189	}
190
191	local_fiq_disable();
192	RCU_NONIDLE(tegra_pm_set_cpu_in_lp2());
193	cpu_pm_enter();
194
195	switch (index) {
196	case TEGRA_C7:
197		err = tegra_cpuidle_c7_enter();
198		break;
199
200	case TEGRA_CC6:
201		err = tegra_cpuidle_cc6_enter(cpu);
202		break;
203
204	default:
205		err = -EINVAL;
206		break;
207	}
208
209	cpu_pm_exit();
210	RCU_NONIDLE(tegra_pm_clear_cpu_in_lp2());
211	local_fiq_enable();
212
213	return err ?: index;
214}
215
216static int tegra_cpuidle_adjust_state_index(int index, unsigned int cpu)
217{
218	/*
219	 * On Tegra30 CPU0 can't be power-gated separately from secondary
220	 * cores because it gates the whole CPU cluster.
221	 */
222	if (cpu > 0 || index != TEGRA_C7 || tegra_get_chip_id() != TEGRA30)
223		return index;
224
225	/* put CPU0 into C1 if C7 is requested and secondaries are online */
226	if (!IS_ENABLED(CONFIG_PM_SLEEP) || num_online_cpus() > 1)
227		index = TEGRA_C1;
228	else
229		index = TEGRA_CC6;
230
231	return index;
232}
233
234static int tegra_cpuidle_enter(struct cpuidle_device *dev,
235			       struct cpuidle_driver *drv,
236			       int index)
237{
238	unsigned int cpu = cpu_logical_map(dev->cpu);
239	int ret;
240
241	index = tegra_cpuidle_adjust_state_index(index, cpu);
242	if (dev->states_usage[index].disable)
243		return -1;
244
245	if (index == TEGRA_C1)
246		ret = arm_cpuidle_simple_enter(dev, drv, index);
247	else
248		ret = tegra_cpuidle_state_enter(dev, index, cpu);
249
250	if (ret < 0) {
251		if (ret != -EINTR || index != TEGRA_CC6)
252			pr_err_once("failed to enter state %d err: %d\n",
253				    index, ret);
254		index = -1;
255	} else {
256		index = ret;
257	}
258
259	return index;
260}
261
262static int tegra114_enter_s2idle(struct cpuidle_device *dev,
263				 struct cpuidle_driver *drv,
264				 int index)
265{
266	tegra_cpuidle_enter(dev, drv, index);
267
268	return 0;
269}
270
271/*
272 * The previous versions of Tegra CPUIDLE driver used a different "legacy"
273 * terminology for naming of the idling states, while this driver uses the
274 * new terminology.
275 *
276 * Mapping of the old terms into the new ones:
277 *
278 * Old | New
279 * ---------
280 * LP3 | C1	(CPU core clock gating)
281 * LP2 | C7	(CPU core power gating)
282 * LP2 | CC6	(CPU cluster power gating)
283 *
284 * Note that that the older CPUIDLE driver versions didn't explicitly
285 * differentiate the LP2 states because these states either used the same
286 * code path or because CC6 wasn't supported.
287 */
288static struct cpuidle_driver tegra_idle_driver = {
289	.name = "tegra_idle",
290	.states = {
291		[TEGRA_C1] = ARM_CPUIDLE_WFI_STATE_PWR(600),
292		[TEGRA_C7] = {
293			.enter			= tegra_cpuidle_enter,
294			.exit_latency		= 2000,
295			.target_residency	= 2200,
296			.power_usage		= 100,
297			.flags			= CPUIDLE_FLAG_TIMER_STOP,
298			.name			= "C7",
299			.desc			= "CPU core powered off",
300		},
301		[TEGRA_CC6] = {
302			.enter			= tegra_cpuidle_enter,
303			.exit_latency		= 5000,
304			.target_residency	= 10000,
305			.power_usage		= 0,
306			.flags			= CPUIDLE_FLAG_TIMER_STOP |
307						  CPUIDLE_FLAG_COUPLED,
308			.name			= "CC6",
309			.desc			= "CPU cluster powered off",
310		},
311	},
312	.state_count = TEGRA_STATE_COUNT,
313	.safe_state_index = TEGRA_C1,
314};
315
316static inline void tegra_cpuidle_disable_state(enum tegra_state state)
317{
318	cpuidle_driver_state_disabled(&tegra_idle_driver, state, true);
319}
320
321/*
322 * Tegra20 HW appears to have a bug such that PCIe device interrupts, whether
323 * they are legacy IRQs or MSI, are lost when CC6 is enabled.  To work around
324 * this, simply disable CC6 if the PCI driver and DT node are both enabled.
325 */
326void tegra_cpuidle_pcie_irqs_in_use(void)
327{
328	struct cpuidle_state *state_cc6 = &tegra_idle_driver.states[TEGRA_CC6];
329
330	if ((state_cc6->flags & CPUIDLE_FLAG_UNUSABLE) ||
331	    tegra_get_chip_id() != TEGRA20)
332		return;
333
334	pr_info("disabling CC6 state, since PCIe IRQs are in use\n");
335	tegra_cpuidle_disable_state(TEGRA_CC6);
336}
337
338static void tegra_cpuidle_setup_tegra114_c7_state(void)
339{
340	struct cpuidle_state *s = &tegra_idle_driver.states[TEGRA_C7];
341
342	s->enter_s2idle = tegra114_enter_s2idle;
343	s->target_residency = 1000;
344	s->exit_latency = 500;
345}
346
347static int tegra_cpuidle_probe(struct platform_device *pdev)
348{
349	/* LP2 could be disabled in device-tree */
350	if (tegra_pmc_get_suspend_mode() < TEGRA_SUSPEND_LP2)
351		tegra_cpuidle_disable_state(TEGRA_CC6);
352
353	/*
354	 * Required suspend-resume functionality, which is provided by the
355	 * Tegra-arch core and PMC driver, is unavailable if PM-sleep option
356	 * is disabled.
357	 */
358	if (!IS_ENABLED(CONFIG_PM_SLEEP)) {
359		if (!tegra_cpuidle_using_firmware())
360			tegra_cpuidle_disable_state(TEGRA_C7);
361
362		tegra_cpuidle_disable_state(TEGRA_CC6);
363	}
364
365	/*
366	 * Generic WFI state (also known as C1 or LP3) and the coupled CPU
367	 * cluster power-off (CC6 or LP2) states are common for all Tegra SoCs.
368	 */
369	switch (tegra_get_chip_id()) {
370	case TEGRA20:
371		/* Tegra20 isn't capable to power-off individual CPU cores */
372		tegra_cpuidle_disable_state(TEGRA_C7);
373		break;
374
375	case TEGRA30:
376		break;
377
378	case TEGRA114:
379	case TEGRA124:
380		tegra_cpuidle_setup_tegra114_c7_state();
381
382		/* coupled CC6 (LP2) state isn't implemented yet */
383		tegra_cpuidle_disable_state(TEGRA_CC6);
384		break;
385
386	default:
387		return -EINVAL;
388	}
389
390	return cpuidle_register(&tegra_idle_driver, cpu_possible_mask);
391}
392
393static struct platform_driver tegra_cpuidle_driver = {
394	.probe = tegra_cpuidle_probe,
395	.driver = {
396		.name = "tegra-cpuidle",
397	},
398};
399builtin_platform_driver(tegra_cpuidle_driver);
400