1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2018 Marty E. Plummer <hanetzer@startmail.com> */
3/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */
4
5#include <linux/clk.h>
6#include <linux/reset.h>
7#include <linux/platform_device.h>
8#include <linux/pm_domain.h>
9#include <linux/pm_runtime.h>
10#include <linux/regulator/consumer.h>
11
12#include "panfrost_device.h"
13#include "panfrost_devfreq.h"
14#include "panfrost_features.h"
15#include "panfrost_issues.h"
16#include "panfrost_gpu.h"
17#include "panfrost_job.h"
18#include "panfrost_mmu.h"
19#include "panfrost_perfcnt.h"
20
21static int panfrost_reset_init(struct panfrost_device *pfdev)
22{
23	pfdev->rstc = devm_reset_control_array_get_optional_exclusive(pfdev->dev);
24	if (IS_ERR(pfdev->rstc)) {
25		dev_err(pfdev->dev, "get reset failed %ld\n", PTR_ERR(pfdev->rstc));
26		return PTR_ERR(pfdev->rstc);
27	}
28
29	return reset_control_deassert(pfdev->rstc);
30}
31
32static void panfrost_reset_fini(struct panfrost_device *pfdev)
33{
34	reset_control_assert(pfdev->rstc);
35}
36
37static int panfrost_clk_init(struct panfrost_device *pfdev)
38{
39	int err;
40	unsigned long rate;
41
42	pfdev->clock = devm_clk_get(pfdev->dev, NULL);
43	if (IS_ERR(pfdev->clock)) {
44		dev_err(pfdev->dev, "get clock failed %ld\n", PTR_ERR(pfdev->clock));
45		return PTR_ERR(pfdev->clock);
46	}
47
48	rate = clk_get_rate(pfdev->clock);
49	dev_info(pfdev->dev, "clock rate = %lu\n", rate);
50
51	err = clk_prepare_enable(pfdev->clock);
52	if (err)
53		return err;
54
55	pfdev->bus_clock = devm_clk_get_optional(pfdev->dev, "bus");
56	if (IS_ERR(pfdev->bus_clock)) {
57		dev_err(pfdev->dev, "get bus_clock failed %ld\n",
58			PTR_ERR(pfdev->bus_clock));
59		err = PTR_ERR(pfdev->bus_clock);
60		goto disable_clock;
61	}
62
63	if (pfdev->bus_clock) {
64		rate = clk_get_rate(pfdev->bus_clock);
65		dev_info(pfdev->dev, "bus_clock rate = %lu\n", rate);
66
67		err = clk_prepare_enable(pfdev->bus_clock);
68		if (err)
69			goto disable_clock;
70	}
71
72	return 0;
73
74disable_clock:
75	clk_disable_unprepare(pfdev->clock);
76
77	return err;
78}
79
80static void panfrost_clk_fini(struct panfrost_device *pfdev)
81{
82	clk_disable_unprepare(pfdev->bus_clock);
83	clk_disable_unprepare(pfdev->clock);
84}
85
86static int panfrost_regulator_init(struct panfrost_device *pfdev)
87{
88	int ret, i;
89
90	pfdev->regulators = devm_kcalloc(pfdev->dev, pfdev->comp->num_supplies,
91					 sizeof(*pfdev->regulators),
92					 GFP_KERNEL);
93	if (!pfdev->regulators)
94		return -ENOMEM;
95
96	for (i = 0; i < pfdev->comp->num_supplies; i++)
97		pfdev->regulators[i].supply = pfdev->comp->supply_names[i];
98
99	ret = devm_regulator_bulk_get(pfdev->dev,
100				      pfdev->comp->num_supplies,
101				      pfdev->regulators);
102	if (ret < 0) {
103		if (ret != -EPROBE_DEFER)
104			dev_err(pfdev->dev, "failed to get regulators: %d\n",
105				ret);
106		return ret;
107	}
108
109	ret = regulator_bulk_enable(pfdev->comp->num_supplies,
110				    pfdev->regulators);
111	if (ret < 0) {
112		dev_err(pfdev->dev, "failed to enable regulators: %d\n", ret);
113		return ret;
114	}
115
116	return 0;
117}
118
119static void panfrost_regulator_fini(struct panfrost_device *pfdev)
120{
121	if (!pfdev->regulators)
122		return;
123
124	regulator_bulk_disable(pfdev->comp->num_supplies, pfdev->regulators);
125}
126
127static void panfrost_pm_domain_fini(struct panfrost_device *pfdev)
128{
129	int i;
130
131	for (i = 0; i < ARRAY_SIZE(pfdev->pm_domain_devs); i++) {
132		if (!pfdev->pm_domain_devs[i])
133			break;
134
135		if (pfdev->pm_domain_links[i])
136			device_link_del(pfdev->pm_domain_links[i]);
137
138		dev_pm_domain_detach(pfdev->pm_domain_devs[i], true);
139	}
140}
141
142static int panfrost_pm_domain_init(struct panfrost_device *pfdev)
143{
144	int err;
145	int i, num_domains;
146
147	num_domains = of_count_phandle_with_args(pfdev->dev->of_node,
148						 "power-domains",
149						 "#power-domain-cells");
150
151	/*
152	 * Single domain is handled by the core, and, if only a single power
153	 * the power domain is requested, the property is optional.
154	 */
155	if (num_domains < 2 && pfdev->comp->num_pm_domains < 2)
156		return 0;
157
158	if (num_domains != pfdev->comp->num_pm_domains) {
159		dev_err(pfdev->dev,
160			"Incorrect number of power domains: %d provided, %d needed\n",
161			num_domains, pfdev->comp->num_pm_domains);
162		return -EINVAL;
163	}
164
165	if (WARN(num_domains > ARRAY_SIZE(pfdev->pm_domain_devs),
166			"Too many supplies in compatible structure.\n"))
167		return -EINVAL;
168
169	for (i = 0; i < num_domains; i++) {
170		pfdev->pm_domain_devs[i] =
171			dev_pm_domain_attach_by_name(pfdev->dev,
172					pfdev->comp->pm_domain_names[i]);
173		if (IS_ERR_OR_NULL(pfdev->pm_domain_devs[i])) {
174			err = PTR_ERR(pfdev->pm_domain_devs[i]) ? : -ENODATA;
175			pfdev->pm_domain_devs[i] = NULL;
176			dev_err(pfdev->dev,
177				"failed to get pm-domain %s(%d): %d\n",
178				pfdev->comp->pm_domain_names[i], i, err);
179			goto err;
180		}
181
182		pfdev->pm_domain_links[i] = device_link_add(pfdev->dev,
183				pfdev->pm_domain_devs[i], DL_FLAG_PM_RUNTIME |
184				DL_FLAG_STATELESS | DL_FLAG_RPM_ACTIVE);
185		if (!pfdev->pm_domain_links[i]) {
186			dev_err(pfdev->pm_domain_devs[i],
187				"adding device link failed!\n");
188			err = -ENODEV;
189			goto err;
190		}
191	}
192
193	return 0;
194
195err:
196	panfrost_pm_domain_fini(pfdev);
197	return err;
198}
199
200int panfrost_device_init(struct panfrost_device *pfdev)
201{
202	int err;
203
204	mutex_init(&pfdev->sched_lock);
205	INIT_LIST_HEAD(&pfdev->scheduled_jobs);
206	INIT_LIST_HEAD(&pfdev->as_lru_list);
207
208	spin_lock_init(&pfdev->as_lock);
209
210	err = panfrost_clk_init(pfdev);
211	if (err) {
212		dev_err(pfdev->dev, "clk init failed %d\n", err);
213		return err;
214	}
215
216	err = panfrost_devfreq_init(pfdev);
217	if (err) {
218		if (err != -EPROBE_DEFER)
219			dev_err(pfdev->dev, "devfreq init failed %d\n", err);
220		goto out_clk;
221	}
222
223	/* OPP will handle regulators */
224	if (!pfdev->pfdevfreq.opp_of_table_added) {
225		err = panfrost_regulator_init(pfdev);
226		if (err)
227			goto out_devfreq;
228	}
229
230	err = panfrost_reset_init(pfdev);
231	if (err) {
232		dev_err(pfdev->dev, "reset init failed %d\n", err);
233		goto out_regulator;
234	}
235
236	err = panfrost_pm_domain_init(pfdev);
237	if (err)
238		goto out_reset;
239
240	pfdev->iomem = devm_platform_ioremap_resource(pfdev->pdev, 0);
241	if (IS_ERR(pfdev->iomem)) {
242		err = PTR_ERR(pfdev->iomem);
243		goto out_pm_domain;
244	}
245
246	err = panfrost_gpu_init(pfdev);
247	if (err)
248		goto out_pm_domain;
249
250	err = panfrost_mmu_init(pfdev);
251	if (err)
252		goto out_gpu;
253
254	err = panfrost_job_init(pfdev);
255	if (err)
256		goto out_mmu;
257
258	err = panfrost_perfcnt_init(pfdev);
259	if (err)
260		goto out_job;
261
262	return 0;
263out_job:
264	panfrost_job_fini(pfdev);
265out_mmu:
266	panfrost_mmu_fini(pfdev);
267out_gpu:
268	panfrost_gpu_fini(pfdev);
269out_pm_domain:
270	panfrost_pm_domain_fini(pfdev);
271out_reset:
272	panfrost_reset_fini(pfdev);
273out_regulator:
274	panfrost_regulator_fini(pfdev);
275out_devfreq:
276	panfrost_devfreq_fini(pfdev);
277out_clk:
278	panfrost_clk_fini(pfdev);
279	return err;
280}
281
282void panfrost_device_fini(struct panfrost_device *pfdev)
283{
284	panfrost_perfcnt_fini(pfdev);
285	panfrost_job_fini(pfdev);
286	panfrost_mmu_fini(pfdev);
287	panfrost_gpu_fini(pfdev);
288	panfrost_pm_domain_fini(pfdev);
289	panfrost_reset_fini(pfdev);
290	panfrost_devfreq_fini(pfdev);
291	panfrost_regulator_fini(pfdev);
292	panfrost_clk_fini(pfdev);
293}
294
295#define PANFROST_EXCEPTION(id) \
296	[DRM_PANFROST_EXCEPTION_ ## id] = { \
297		.name = #id, \
298	}
299
300struct panfrost_exception_info {
301	const char *name;
302};
303
304static const struct panfrost_exception_info panfrost_exception_infos[] = {
305	PANFROST_EXCEPTION(OK),
306	PANFROST_EXCEPTION(DONE),
307	PANFROST_EXCEPTION(INTERRUPTED),
308	PANFROST_EXCEPTION(STOPPED),
309	PANFROST_EXCEPTION(TERMINATED),
310	PANFROST_EXCEPTION(KABOOM),
311	PANFROST_EXCEPTION(EUREKA),
312	PANFROST_EXCEPTION(ACTIVE),
313	PANFROST_EXCEPTION(JOB_CONFIG_FAULT),
314	PANFROST_EXCEPTION(JOB_POWER_FAULT),
315	PANFROST_EXCEPTION(JOB_READ_FAULT),
316	PANFROST_EXCEPTION(JOB_WRITE_FAULT),
317	PANFROST_EXCEPTION(JOB_AFFINITY_FAULT),
318	PANFROST_EXCEPTION(JOB_BUS_FAULT),
319	PANFROST_EXCEPTION(INSTR_INVALID_PC),
320	PANFROST_EXCEPTION(INSTR_INVALID_ENC),
321	PANFROST_EXCEPTION(INSTR_TYPE_MISMATCH),
322	PANFROST_EXCEPTION(INSTR_OPERAND_FAULT),
323	PANFROST_EXCEPTION(INSTR_TLS_FAULT),
324	PANFROST_EXCEPTION(INSTR_BARRIER_FAULT),
325	PANFROST_EXCEPTION(INSTR_ALIGN_FAULT),
326	PANFROST_EXCEPTION(DATA_INVALID_FAULT),
327	PANFROST_EXCEPTION(TILE_RANGE_FAULT),
328	PANFROST_EXCEPTION(ADDR_RANGE_FAULT),
329	PANFROST_EXCEPTION(IMPRECISE_FAULT),
330	PANFROST_EXCEPTION(OOM),
331	PANFROST_EXCEPTION(OOM_AFBC),
332	PANFROST_EXCEPTION(UNKNOWN),
333	PANFROST_EXCEPTION(DELAYED_BUS_FAULT),
334	PANFROST_EXCEPTION(GPU_SHAREABILITY_FAULT),
335	PANFROST_EXCEPTION(SYS_SHAREABILITY_FAULT),
336	PANFROST_EXCEPTION(GPU_CACHEABILITY_FAULT),
337	PANFROST_EXCEPTION(TRANSLATION_FAULT_0),
338	PANFROST_EXCEPTION(TRANSLATION_FAULT_1),
339	PANFROST_EXCEPTION(TRANSLATION_FAULT_2),
340	PANFROST_EXCEPTION(TRANSLATION_FAULT_3),
341	PANFROST_EXCEPTION(TRANSLATION_FAULT_4),
342	PANFROST_EXCEPTION(TRANSLATION_FAULT_IDENTITY),
343	PANFROST_EXCEPTION(PERM_FAULT_0),
344	PANFROST_EXCEPTION(PERM_FAULT_1),
345	PANFROST_EXCEPTION(PERM_FAULT_2),
346	PANFROST_EXCEPTION(PERM_FAULT_3),
347	PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_0),
348	PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_1),
349	PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_2),
350	PANFROST_EXCEPTION(TRANSTAB_BUS_FAULT_3),
351	PANFROST_EXCEPTION(ACCESS_FLAG_0),
352	PANFROST_EXCEPTION(ACCESS_FLAG_1),
353	PANFROST_EXCEPTION(ACCESS_FLAG_2),
354	PANFROST_EXCEPTION(ACCESS_FLAG_3),
355	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN0),
356	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN1),
357	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN2),
358	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_IN3),
359	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT0),
360	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT1),
361	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT2),
362	PANFROST_EXCEPTION(ADDR_SIZE_FAULT_OUT3),
363	PANFROST_EXCEPTION(MEM_ATTR_FAULT_0),
364	PANFROST_EXCEPTION(MEM_ATTR_FAULT_1),
365	PANFROST_EXCEPTION(MEM_ATTR_FAULT_2),
366	PANFROST_EXCEPTION(MEM_ATTR_FAULT_3),
367	PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_0),
368	PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_1),
369	PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_2),
370	PANFROST_EXCEPTION(MEM_ATTR_NONCACHE_3),
371};
372
373const char *panfrost_exception_name(u32 exception_code)
374{
375	if (WARN_ON(exception_code >= ARRAY_SIZE(panfrost_exception_infos) ||
376		    !panfrost_exception_infos[exception_code].name))
377		return "Unknown exception type";
378
379	return panfrost_exception_infos[exception_code].name;
380}
381
382bool panfrost_exception_needs_reset(const struct panfrost_device *pfdev,
383				    u32 exception_code)
384{
385	/* If an occlusion query write causes a bus fault on affected GPUs,
386	 * future fragment jobs may hang. Reset to workaround.
387	 */
388	if (exception_code == DRM_PANFROST_EXCEPTION_JOB_BUS_FAULT)
389		return panfrost_has_hw_issue(pfdev, HW_ISSUE_TTRX_3076);
390
391	/* No other GPUs we support need a reset */
392	return false;
393}
394
395void panfrost_device_reset(struct panfrost_device *pfdev)
396{
397	panfrost_gpu_soft_reset(pfdev);
398
399	panfrost_gpu_power_on(pfdev);
400	panfrost_mmu_reset(pfdev);
401	panfrost_job_enable_interrupts(pfdev);
402}
403
404static int panfrost_device_resume(struct device *dev)
405{
406	struct panfrost_device *pfdev = dev_get_drvdata(dev);
407
408	panfrost_device_reset(pfdev);
409	panfrost_devfreq_resume(pfdev);
410
411	return 0;
412}
413
414static int panfrost_device_suspend(struct device *dev)
415{
416	struct panfrost_device *pfdev = dev_get_drvdata(dev);
417
418	if (!panfrost_job_is_idle(pfdev))
419		return -EBUSY;
420
421	panfrost_devfreq_suspend(pfdev);
422	panfrost_gpu_power_off(pfdev);
423
424	return 0;
425}
426
427EXPORT_GPL_RUNTIME_DEV_PM_OPS(panfrost_pm_ops, panfrost_device_suspend,
428			      panfrost_device_resume, NULL);
429