1// SPDX-License-Identifier: GPL-2.0
2/* Copyright 2019 Collabora Ltd */
3
4#include <drm/drm_file.h>
5#include <drm/drm_gem_shmem_helper.h>
6#include <drm/panfrost_drm.h>
7#include <linux/completion.h>
8#include <linux/iopoll.h>
9#include <linux/pm_runtime.h>
10#include <linux/slab.h>
11#include <linux/uaccess.h>
12
13#include "panfrost_device.h"
14#include "panfrost_features.h"
15#include "panfrost_gem.h"
16#include "panfrost_issues.h"
17#include "panfrost_job.h"
18#include "panfrost_mmu.h"
19#include "panfrost_perfcnt.h"
20#include "panfrost_regs.h"
21
22#define COUNTERS_PER_BLOCK		64
23#define BYTES_PER_COUNTER		4
24#define BLOCKS_PER_COREGROUP		8
25#define V4_SHADERS_PER_COREGROUP	4
26
27struct panfrost_perfcnt {
28	struct panfrost_gem_mapping *mapping;
29	size_t bosize;
30	void *buf;
31	struct panfrost_file_priv *user;
32	struct mutex lock;
33	struct completion dump_comp;
34};
35
36void panfrost_perfcnt_clean_cache_done(struct panfrost_device *pfdev)
37{
38	complete(&pfdev->perfcnt->dump_comp);
39}
40
41void panfrost_perfcnt_sample_done(struct panfrost_device *pfdev)
42{
43	gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_CACHES);
44}
45
46static int panfrost_perfcnt_dump_locked(struct panfrost_device *pfdev)
47{
48	u64 gpuva;
49	int ret;
50
51	reinit_completion(&pfdev->perfcnt->dump_comp);
52	gpuva = pfdev->perfcnt->mapping->mmnode.start << PAGE_SHIFT;
53	gpu_write(pfdev, GPU_PERFCNT_BASE_LO, gpuva);
54	gpu_write(pfdev, GPU_PERFCNT_BASE_HI, gpuva >> 32);
55	gpu_write(pfdev, GPU_INT_CLEAR,
56		  GPU_IRQ_CLEAN_CACHES_COMPLETED |
57		  GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
58	gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_SAMPLE);
59	ret = wait_for_completion_interruptible_timeout(&pfdev->perfcnt->dump_comp,
60							msecs_to_jiffies(1000));
61	if (!ret)
62		ret = -ETIMEDOUT;
63	else if (ret > 0)
64		ret = 0;
65
66	return ret;
67}
68
69static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev,
70					  struct drm_file *file_priv,
71					  unsigned int counterset)
72{
73	struct panfrost_file_priv *user = file_priv->driver_priv;
74	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
75	struct drm_gem_shmem_object *bo;
76	u32 cfg, as;
77	int ret;
78
79	if (user == perfcnt->user)
80		return 0;
81	else if (perfcnt->user)
82		return -EBUSY;
83
84	ret = pm_runtime_get_sync(pfdev->dev);
85	if (ret < 0)
86		goto err_put_pm;
87
88	bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize);
89	if (IS_ERR(bo)) {
90		ret = PTR_ERR(bo);
91		goto err_put_pm;
92	}
93
94	/* Map the perfcnt buf in the address space attached to file_priv. */
95	ret = panfrost_gem_open(&bo->base, file_priv);
96	if (ret)
97		goto err_put_bo;
98
99	perfcnt->mapping = panfrost_gem_mapping_get(to_panfrost_bo(&bo->base),
100						    user);
101	if (!perfcnt->mapping) {
102		ret = -EINVAL;
103		goto err_close_bo;
104	}
105
106	perfcnt->buf = drm_gem_shmem_vmap(&bo->base);
107	if (IS_ERR(perfcnt->buf)) {
108		ret = PTR_ERR(perfcnt->buf);
109		goto err_put_mapping;
110	}
111
112	/*
113	 * Invalidate the cache and clear the counters to start from a fresh
114	 * state.
115	 */
116	reinit_completion(&pfdev->perfcnt->dump_comp);
117	gpu_write(pfdev, GPU_INT_CLEAR,
118		  GPU_IRQ_CLEAN_CACHES_COMPLETED |
119		  GPU_IRQ_PERFCNT_SAMPLE_COMPLETED);
120	gpu_write(pfdev, GPU_CMD, GPU_CMD_PERFCNT_CLEAR);
121	gpu_write(pfdev, GPU_CMD, GPU_CMD_CLEAN_INV_CACHES);
122	ret = wait_for_completion_timeout(&pfdev->perfcnt->dump_comp,
123					  msecs_to_jiffies(1000));
124	if (!ret) {
125		ret = -ETIMEDOUT;
126		goto err_vunmap;
127	}
128
129	perfcnt->user = user;
130
131	as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu);
132	cfg = GPU_PERFCNT_CFG_AS(as) |
133	      GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL);
134
135	/*
136	 * Bifrost GPUs have 2 set of counters, but we're only interested by
137	 * the first one for now.
138	 */
139	if (panfrost_model_is_bifrost(pfdev))
140		cfg |= GPU_PERFCNT_CFG_SETSEL(counterset);
141
142	gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0xffffffff);
143	gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0xffffffff);
144	gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0xffffffff);
145
146	/*
147	 * Due to PRLAM-8186 we need to disable the Tiler before we enable HW
148	 * counters.
149	 */
150	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
151		gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
152	else
153		gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
154
155	gpu_write(pfdev, GPU_PERFCNT_CFG, cfg);
156
157	if (panfrost_has_hw_issue(pfdev, HW_ISSUE_8186))
158		gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0xffffffff);
159
160	/* The BO ref is retained by the mapping. */
161	drm_gem_object_put(&bo->base);
162
163	return 0;
164
165err_vunmap:
166	drm_gem_shmem_vunmap(&bo->base, perfcnt->buf);
167err_put_mapping:
168	panfrost_gem_mapping_put(perfcnt->mapping);
169err_close_bo:
170	panfrost_gem_close(&bo->base, file_priv);
171err_put_bo:
172	drm_gem_object_put(&bo->base);
173err_put_pm:
174	pm_runtime_put(pfdev->dev);
175	return ret;
176}
177
178static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev,
179					   struct drm_file *file_priv)
180{
181	struct panfrost_file_priv *user = file_priv->driver_priv;
182	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
183
184	if (user != perfcnt->user)
185		return -EINVAL;
186
187	gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0x0);
188	gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0x0);
189	gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0x0);
190	gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
191	gpu_write(pfdev, GPU_PERFCNT_CFG,
192		  GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
193
194	perfcnt->user = NULL;
195	drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base.base, perfcnt->buf);
196	perfcnt->buf = NULL;
197	panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv);
198	panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu);
199	panfrost_gem_mapping_put(perfcnt->mapping);
200	perfcnt->mapping = NULL;
201	pm_runtime_mark_last_busy(pfdev->dev);
202	pm_runtime_put_autosuspend(pfdev->dev);
203
204	return 0;
205}
206
207int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data,
208				  struct drm_file *file_priv)
209{
210	struct panfrost_device *pfdev = dev->dev_private;
211	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
212	struct drm_panfrost_perfcnt_enable *req = data;
213	int ret;
214
215	ret = panfrost_unstable_ioctl_check();
216	if (ret)
217		return ret;
218
219	/* Only Bifrost GPUs have 2 set of counters. */
220	if (req->counterset > (panfrost_model_is_bifrost(pfdev) ? 1 : 0))
221		return -EINVAL;
222
223	mutex_lock(&perfcnt->lock);
224	if (req->enable)
225		ret = panfrost_perfcnt_enable_locked(pfdev, file_priv,
226						     req->counterset);
227	else
228		ret = panfrost_perfcnt_disable_locked(pfdev, file_priv);
229	mutex_unlock(&perfcnt->lock);
230
231	return ret;
232}
233
234int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data,
235				struct drm_file *file_priv)
236{
237	struct panfrost_device *pfdev = dev->dev_private;
238	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
239	struct drm_panfrost_perfcnt_dump *req = data;
240	void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr;
241	int ret;
242
243	ret = panfrost_unstable_ioctl_check();
244	if (ret)
245		return ret;
246
247	mutex_lock(&perfcnt->lock);
248	if (perfcnt->user != file_priv->driver_priv) {
249		ret = -EINVAL;
250		goto out;
251	}
252
253	ret = panfrost_perfcnt_dump_locked(pfdev);
254	if (ret)
255		goto out;
256
257	if (copy_to_user(user_ptr, perfcnt->buf, perfcnt->bosize))
258		ret = -EFAULT;
259
260out:
261	mutex_unlock(&perfcnt->lock);
262
263	return ret;
264}
265
266void panfrost_perfcnt_close(struct drm_file *file_priv)
267{
268	struct panfrost_file_priv *pfile = file_priv->driver_priv;
269	struct panfrost_device *pfdev = pfile->pfdev;
270	struct panfrost_perfcnt *perfcnt = pfdev->perfcnt;
271
272	pm_runtime_get_sync(pfdev->dev);
273	mutex_lock(&perfcnt->lock);
274	if (perfcnt->user == pfile)
275		panfrost_perfcnt_disable_locked(pfdev, file_priv);
276	mutex_unlock(&perfcnt->lock);
277	pm_runtime_mark_last_busy(pfdev->dev);
278	pm_runtime_put_autosuspend(pfdev->dev);
279}
280
281int panfrost_perfcnt_init(struct panfrost_device *pfdev)
282{
283	struct panfrost_perfcnt *perfcnt;
284	size_t size;
285
286	if (panfrost_has_hw_feature(pfdev, HW_FEATURE_V4)) {
287		unsigned int ncoregroups;
288
289		ncoregroups = hweight64(pfdev->features.l2_present);
290		size = ncoregroups * BLOCKS_PER_COREGROUP *
291		       COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
292	} else {
293		unsigned int nl2c, ncores;
294
295		/*
296		 * TODO: define a macro to extract the number of l2 caches from
297		 * mem_features.
298		 */
299		nl2c = ((pfdev->features.mem_features >> 8) & GENMASK(3, 0)) + 1;
300
301		/*
302		 * shader_present might be sparse, but the counters layout
303		 * forces to dump unused regions too, hence the fls64() call
304		 * instead of hweight64().
305		 */
306		ncores = fls64(pfdev->features.shader_present);
307
308		/*
309		 * There's always one JM and one Tiler block, hence the '+ 2'
310		 * here.
311		 */
312		size = (nl2c + ncores + 2) *
313		       COUNTERS_PER_BLOCK * BYTES_PER_COUNTER;
314	}
315
316	perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL);
317	if (!perfcnt)
318		return -ENOMEM;
319
320	perfcnt->bosize = size;
321
322	/* Start with everything disabled. */
323	gpu_write(pfdev, GPU_PERFCNT_CFG,
324		  GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
325	gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
326	gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
327	gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
328	gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
329
330	init_completion(&perfcnt->dump_comp);
331	mutex_init(&perfcnt->lock);
332	pfdev->perfcnt = perfcnt;
333
334	return 0;
335}
336
337void panfrost_perfcnt_fini(struct panfrost_device *pfdev)
338{
339	/* Disable everything before leaving. */
340	gpu_write(pfdev, GPU_PERFCNT_CFG,
341		  GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF));
342	gpu_write(pfdev, GPU_PRFCNT_JM_EN, 0);
343	gpu_write(pfdev, GPU_PRFCNT_SHADER_EN, 0);
344	gpu_write(pfdev, GPU_PRFCNT_MMU_L2_EN, 0);
345	gpu_write(pfdev, GPU_PRFCNT_TILER_EN, 0);
346}
347