1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (c) 2017 The Linux Foundation. All rights reserved.
3 */
4
5#include "msm_gem.h"
6#include "a5xx_gpu.h"
7
8/*
9 * Try to transition the preemption state from old to new. Return
10 * true on success or false if the original state wasn't 'old'
11 */
12static inline bool try_preempt_state(struct a5xx_gpu *a5xx_gpu,
13		enum preempt_state old, enum preempt_state new)
14{
15	enum preempt_state cur = atomic_cmpxchg(&a5xx_gpu->preempt_state,
16		old, new);
17
18	return (cur == old);
19}
20
21/*
22 * Force the preemption state to the specified state.  This is used in cases
23 * where the current state is known and won't change
24 */
25static inline void set_preempt_state(struct a5xx_gpu *gpu,
26		enum preempt_state new)
27{
28	/*
29	 * preempt_state may be read by other cores trying to trigger a
30	 * preemption or in the interrupt handler so barriers are needed
31	 * before...
32	 */
33	smp_mb__before_atomic();
34	atomic_set(&gpu->preempt_state, new);
35	/* ... and after*/
36	smp_mb__after_atomic();
37}
38
39/* Write the most recent wptr for the given ring into the hardware */
40static inline void update_wptr(struct msm_gpu *gpu, struct msm_ringbuffer *ring)
41{
42	unsigned long flags;
43	uint32_t wptr;
44
45	if (!ring)
46		return;
47
48	spin_lock_irqsave(&ring->preempt_lock, flags);
49	wptr = get_wptr(ring);
50	spin_unlock_irqrestore(&ring->preempt_lock, flags);
51
52	gpu_write(gpu, REG_A5XX_CP_RB_WPTR, wptr);
53}
54
55/* Return the highest priority ringbuffer with something in it */
56static struct msm_ringbuffer *get_next_ring(struct msm_gpu *gpu)
57{
58	unsigned long flags;
59	int i;
60
61	for (i = 0; i < gpu->nr_rings; i++) {
62		bool empty;
63		struct msm_ringbuffer *ring = gpu->rb[i];
64
65		spin_lock_irqsave(&ring->preempt_lock, flags);
66		empty = (get_wptr(ring) == gpu->funcs->get_rptr(gpu, ring));
67		spin_unlock_irqrestore(&ring->preempt_lock, flags);
68
69		if (!empty)
70			return ring;
71	}
72
73	return NULL;
74}
75
76static void a5xx_preempt_timer(struct timer_list *t)
77{
78	struct a5xx_gpu *a5xx_gpu = from_timer(a5xx_gpu, t, preempt_timer);
79	struct msm_gpu *gpu = &a5xx_gpu->base.base;
80	struct drm_device *dev = gpu->dev;
81	struct msm_drm_private *priv = dev->dev_private;
82
83	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_FAULTED))
84		return;
85
86	DRM_DEV_ERROR(dev->dev, "%s: preemption timed out\n", gpu->name);
87	queue_work(priv->wq, &gpu->recover_work);
88}
89
90/* Try to trigger a preemption switch */
91void a5xx_preempt_trigger(struct msm_gpu *gpu)
92{
93	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
94	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
95	unsigned long flags;
96	struct msm_ringbuffer *ring;
97
98	if (gpu->nr_rings == 1)
99		return;
100
101	/*
102	 * Try to start preemption by moving from NONE to START. If
103	 * unsuccessful, a preemption is already in flight
104	 */
105	if (!try_preempt_state(a5xx_gpu, PREEMPT_NONE, PREEMPT_START))
106		return;
107
108	/* Get the next ring to preempt to */
109	ring = get_next_ring(gpu);
110
111	/*
112	 * If no ring is populated or the highest priority ring is the current
113	 * one do nothing except to update the wptr to the latest and greatest
114	 */
115	if (!ring || (a5xx_gpu->cur_ring == ring)) {
116		/*
117		 * Its possible that while a preemption request is in progress
118		 * from an irq context, a user context trying to submit might
119		 * fail to update the write pointer, because it determines
120		 * that the preempt state is not PREEMPT_NONE.
121		 *
122		 * Close the race by introducing an intermediate
123		 * state PREEMPT_ABORT to let the submit path
124		 * know that the ringbuffer is not going to change
125		 * and can safely update the write pointer.
126		 */
127
128		set_preempt_state(a5xx_gpu, PREEMPT_ABORT);
129		update_wptr(gpu, a5xx_gpu->cur_ring);
130		set_preempt_state(a5xx_gpu, PREEMPT_NONE);
131		return;
132	}
133
134	/* Make sure the wptr doesn't update while we're in motion */
135	spin_lock_irqsave(&ring->preempt_lock, flags);
136	a5xx_gpu->preempt[ring->id]->wptr = get_wptr(ring);
137	spin_unlock_irqrestore(&ring->preempt_lock, flags);
138
139	/* Set the address of the incoming preemption record */
140	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_LO,
141		REG_A5XX_CP_CONTEXT_SWITCH_RESTORE_ADDR_HI,
142		a5xx_gpu->preempt_iova[ring->id]);
143
144	a5xx_gpu->next_ring = ring;
145
146	/* Start a timer to catch a stuck preemption */
147	mod_timer(&a5xx_gpu->preempt_timer, jiffies + msecs_to_jiffies(10000));
148
149	/* Set the preemption state to triggered */
150	set_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED);
151
152	/* Make sure everything is written before hitting the button */
153	wmb();
154
155	/* And actually start the preemption */
156	gpu_write(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL, 1);
157}
158
159void a5xx_preempt_irq(struct msm_gpu *gpu)
160{
161	uint32_t status;
162	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
163	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
164	struct drm_device *dev = gpu->dev;
165	struct msm_drm_private *priv = dev->dev_private;
166
167	if (!try_preempt_state(a5xx_gpu, PREEMPT_TRIGGERED, PREEMPT_PENDING))
168		return;
169
170	/* Delete the preemption watchdog timer */
171	del_timer(&a5xx_gpu->preempt_timer);
172
173	/*
174	 * The hardware should be setting CP_CONTEXT_SWITCH_CNTL to zero before
175	 * firing the interrupt, but there is a non zero chance of a hardware
176	 * condition or a software race that could set it again before we have a
177	 * chance to finish. If that happens, log and go for recovery
178	 */
179	status = gpu_read(gpu, REG_A5XX_CP_CONTEXT_SWITCH_CNTL);
180	if (unlikely(status)) {
181		set_preempt_state(a5xx_gpu, PREEMPT_FAULTED);
182		DRM_DEV_ERROR(dev->dev, "%s: Preemption failed to complete\n",
183			gpu->name);
184		queue_work(priv->wq, &gpu->recover_work);
185		return;
186	}
187
188	a5xx_gpu->cur_ring = a5xx_gpu->next_ring;
189	a5xx_gpu->next_ring = NULL;
190
191	update_wptr(gpu, a5xx_gpu->cur_ring);
192
193	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
194}
195
196void a5xx_preempt_hw_init(struct msm_gpu *gpu)
197{
198	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
199	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
200	int i;
201
202	/* Always come up on rb 0 */
203	a5xx_gpu->cur_ring = gpu->rb[0];
204
205	/* No preemption if we only have one ring */
206	if (gpu->nr_rings == 1)
207		return;
208
209	for (i = 0; i < gpu->nr_rings; i++) {
210		a5xx_gpu->preempt[i]->wptr = 0;
211		a5xx_gpu->preempt[i]->rptr = 0;
212		a5xx_gpu->preempt[i]->rbase = gpu->rb[i]->iova;
213		a5xx_gpu->preempt[i]->rptr_addr = shadowptr(a5xx_gpu, gpu->rb[i]);
214	}
215
216	/* Write a 0 to signal that we aren't switching pagetables */
217	gpu_write64(gpu, REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
218		REG_A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI, 0);
219
220	/* Reset the preemption state */
221	set_preempt_state(a5xx_gpu, PREEMPT_NONE);
222}
223
224static int preempt_init_ring(struct a5xx_gpu *a5xx_gpu,
225		struct msm_ringbuffer *ring)
226{
227	struct adreno_gpu *adreno_gpu = &a5xx_gpu->base;
228	struct msm_gpu *gpu = &adreno_gpu->base;
229	struct a5xx_preempt_record *ptr;
230	void *counters;
231	struct drm_gem_object *bo = NULL, *counters_bo = NULL;
232	u64 iova = 0, counters_iova = 0;
233
234	ptr = msm_gem_kernel_new(gpu->dev,
235		A5XX_PREEMPT_RECORD_SIZE + A5XX_PREEMPT_COUNTER_SIZE,
236		MSM_BO_UNCACHED | MSM_BO_MAP_PRIV, gpu->aspace, &bo, &iova);
237
238	if (IS_ERR(ptr))
239		return PTR_ERR(ptr);
240
241	/* The buffer to store counters needs to be unprivileged */
242	counters = msm_gem_kernel_new(gpu->dev,
243		A5XX_PREEMPT_COUNTER_SIZE,
244		MSM_BO_UNCACHED, gpu->aspace, &counters_bo, &counters_iova);
245	if (IS_ERR(counters)) {
246		msm_gem_kernel_put(bo, gpu->aspace, true);
247		return PTR_ERR(counters);
248	}
249
250	msm_gem_object_set_name(bo, "preempt");
251	msm_gem_object_set_name(counters_bo, "preempt_counters");
252
253	a5xx_gpu->preempt_bo[ring->id] = bo;
254	a5xx_gpu->preempt_counters_bo[ring->id] = counters_bo;
255	a5xx_gpu->preempt_iova[ring->id] = iova;
256	a5xx_gpu->preempt[ring->id] = ptr;
257
258	/* Set up the defaults on the preemption record */
259
260	ptr->magic = A5XX_PREEMPT_RECORD_MAGIC;
261	ptr->info = 0;
262	ptr->data = 0;
263	ptr->cntl = MSM_GPU_RB_CNTL_DEFAULT | AXXX_CP_RB_CNTL_NO_UPDATE;
264
265	ptr->counter = counters_iova;
266
267	return 0;
268}
269
270void a5xx_preempt_fini(struct msm_gpu *gpu)
271{
272	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
273	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
274	int i;
275
276	for (i = 0; i < gpu->nr_rings; i++) {
277		msm_gem_kernel_put(a5xx_gpu->preempt_bo[i], gpu->aspace, true);
278		msm_gem_kernel_put(a5xx_gpu->preempt_counters_bo[i],
279			gpu->aspace, true);
280	}
281}
282
283void a5xx_preempt_init(struct msm_gpu *gpu)
284{
285	struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
286	struct a5xx_gpu *a5xx_gpu = to_a5xx_gpu(adreno_gpu);
287	int i;
288
289	/* No preemption if we only have one ring */
290	if (gpu->nr_rings <= 1)
291		return;
292
293	for (i = 0; i < gpu->nr_rings; i++) {
294		if (preempt_init_ring(a5xx_gpu, gpu->rb[i])) {
295			/*
296			 * On any failure our adventure is over. Clean up and
297			 * set nr_rings to 1 to force preemption off
298			 */
299			a5xx_preempt_fini(gpu);
300			gpu->nr_rings = 1;
301
302			return;
303		}
304	}
305
306	timer_setup(&a5xx_gpu->preempt_timer, a5xx_preempt_timer, 0);
307}
308