1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 *
5 */
6
7#include "gem/i915_gem_internal.h"
8
9#include "i915_drv.h"
10#include "i915_reg.h"
11#include "intel_de.h"
12#include "intel_display_types.h"
13#include "intel_dsb.h"
14#include "intel_dsb_regs.h"
15
16struct i915_vma;
17
18enum dsb_id {
19	INVALID_DSB = -1,
20	DSB1,
21	DSB2,
22	DSB3,
23	MAX_DSB_PER_PIPE
24};
25
26struct intel_dsb {
27	enum dsb_id id;
28
29	u32 *cmd_buf;
30	struct i915_vma *vma;
31	struct intel_crtc *crtc;
32
33	/*
34	 * maximum number of dwords the buffer will hold.
35	 */
36	unsigned int size;
37
38	/*
39	 * free_pos will point the first free dword and
40	 * help in calculating tail of command buffer.
41	 */
42	unsigned int free_pos;
43
44	/*
45	 * ins_start_offset will help to store start dword of the dsb
46	 * instuction and help in identifying the batch of auto-increment
47	 * register.
48	 */
49	unsigned int ins_start_offset;
50};
51
52/**
53 * DOC: DSB
54 *
55 * A DSB (Display State Buffer) is a queue of MMIO instructions in the memory
56 * which can be offloaded to DSB HW in Display Controller. DSB HW is a DMA
57 * engine that can be programmed to download the DSB from memory.
58 * It allows driver to batch submit display HW programming. This helps to
59 * reduce loading time and CPU activity, thereby making the context switch
60 * faster. DSB Support added from Gen12 Intel graphics based platform.
61 *
62 * DSB's can access only the pipe, plane, and transcoder Data Island Packet
63 * registers.
64 *
65 * DSB HW can support only register writes (both indexed and direct MMIO
66 * writes). There are no registers reads possible with DSB HW engine.
67 */
68
69/* DSB opcodes. */
70#define DSB_OPCODE_SHIFT		24
71#define DSB_OPCODE_NOOP			0x0
72#define DSB_OPCODE_MMIO_WRITE		0x1
73#define DSB_OPCODE_WAIT_USEC		0x2
74#define DSB_OPCODE_WAIT_LINES		0x3
75#define DSB_OPCODE_WAIT_VBLANKS		0x4
76#define DSB_OPCODE_WAIT_DSL_IN		0x5
77#define DSB_OPCODE_WAIT_DSL_OUT		0x6
78#define DSB_OPCODE_INTERRUPT		0x7
79#define DSB_OPCODE_INDEXED_WRITE	0x9
80#define DSB_OPCODE_POLL			0xA
81#define DSB_BYTE_EN			0xF
82#define DSB_BYTE_EN_SHIFT		20
83#define DSB_REG_VALUE_MASK		0xfffff
84
85static bool assert_dsb_has_room(struct intel_dsb *dsb)
86{
87	struct intel_crtc *crtc = dsb->crtc;
88	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
89
90	/* each instruction is 2 dwords */
91	return !drm_WARN(&i915->drm, dsb->free_pos > dsb->size - 2,
92			 "[CRTC:%d:%s] DSB %d buffer overflow\n",
93			 crtc->base.base.id, crtc->base.name, dsb->id);
94}
95
96static bool is_dsb_busy(struct drm_i915_private *i915, enum pipe pipe,
97			enum dsb_id id)
98{
99	return intel_de_read(i915, DSB_CTRL(pipe, id)) & DSB_STATUS_BUSY;
100}
101
102static void intel_dsb_emit(struct intel_dsb *dsb, u32 ldw, u32 udw)
103{
104	u32 *buf = dsb->cmd_buf;
105
106	if (!assert_dsb_has_room(dsb))
107		return;
108
109	/* Every instruction should be 8 byte aligned. */
110	dsb->free_pos = ALIGN(dsb->free_pos, 2);
111
112	dsb->ins_start_offset = dsb->free_pos;
113
114	buf[dsb->free_pos++] = ldw;
115	buf[dsb->free_pos++] = udw;
116}
117
118static bool intel_dsb_prev_ins_is_write(struct intel_dsb *dsb,
119					u32 opcode, i915_reg_t reg)
120{
121	const u32 *buf = dsb->cmd_buf;
122	u32 prev_opcode, prev_reg;
123
124	prev_opcode = buf[dsb->ins_start_offset + 1] >> DSB_OPCODE_SHIFT;
125	prev_reg = buf[dsb->ins_start_offset + 1] & DSB_REG_VALUE_MASK;
126
127	return prev_opcode == opcode && prev_reg == i915_mmio_reg_offset(reg);
128}
129
130static bool intel_dsb_prev_ins_is_mmio_write(struct intel_dsb *dsb, i915_reg_t reg)
131{
132	return intel_dsb_prev_ins_is_write(dsb, DSB_OPCODE_MMIO_WRITE, reg);
133}
134
135static bool intel_dsb_prev_ins_is_indexed_write(struct intel_dsb *dsb, i915_reg_t reg)
136{
137	return intel_dsb_prev_ins_is_write(dsb, DSB_OPCODE_INDEXED_WRITE, reg);
138}
139
140/**
141 * intel_dsb_reg_write() - Emit register wriite to the DSB context
142 * @dsb: DSB context
143 * @reg: register address.
144 * @val: value.
145 *
146 * This function is used for writing register-value pair in command
147 * buffer of DSB.
148 */
149void intel_dsb_reg_write(struct intel_dsb *dsb,
150			 i915_reg_t reg, u32 val)
151{
152	/*
153	 * For example the buffer will look like below for 3 dwords for auto
154	 * increment register:
155	 * +--------------------------------------------------------+
156	 * | size = 3 | offset &| value1 | value2 | value3 | zero   |
157	 * |          | opcode  |        |        |        |        |
158	 * +--------------------------------------------------------+
159	 * +          +         +        +        +        +        +
160	 * 0          4         8        12       16       20       24
161	 * Byte
162	 *
163	 * As every instruction is 8 byte aligned the index of dsb instruction
164	 * will start always from even number while dealing with u32 array. If
165	 * we are writing odd no of dwords, Zeros will be added in the end for
166	 * padding.
167	 */
168	if (!intel_dsb_prev_ins_is_mmio_write(dsb, reg) &&
169	    !intel_dsb_prev_ins_is_indexed_write(dsb, reg)) {
170		intel_dsb_emit(dsb, val,
171			       (DSB_OPCODE_MMIO_WRITE << DSB_OPCODE_SHIFT) |
172			       (DSB_BYTE_EN << DSB_BYTE_EN_SHIFT) |
173			       i915_mmio_reg_offset(reg));
174	} else {
175		u32 *buf = dsb->cmd_buf;
176
177		if (!assert_dsb_has_room(dsb))
178			return;
179
180		/* convert to indexed write? */
181		if (intel_dsb_prev_ins_is_mmio_write(dsb, reg)) {
182			u32 prev_val = buf[dsb->ins_start_offset + 0];
183
184			buf[dsb->ins_start_offset + 0] = 1; /* count */
185			buf[dsb->ins_start_offset + 1] =
186				(DSB_OPCODE_INDEXED_WRITE << DSB_OPCODE_SHIFT) |
187				i915_mmio_reg_offset(reg);
188			buf[dsb->ins_start_offset + 2] = prev_val;
189
190			dsb->free_pos++;
191		}
192
193		buf[dsb->free_pos++] = val;
194		/* Update the count */
195		buf[dsb->ins_start_offset]++;
196
197		/* if number of data words is odd, then the last dword should be 0.*/
198		if (dsb->free_pos & 0x1)
199			buf[dsb->free_pos] = 0;
200	}
201}
202
203static void intel_dsb_align_tail(struct intel_dsb *dsb)
204{
205	u32 aligned_tail, tail;
206
207	tail = dsb->free_pos * 4;
208	aligned_tail = ALIGN(tail, CACHELINE_BYTES);
209
210	if (aligned_tail > tail)
211		memset(&dsb->cmd_buf[dsb->free_pos], 0,
212		       aligned_tail - tail);
213
214	dsb->free_pos = aligned_tail / 4;
215}
216
217void intel_dsb_finish(struct intel_dsb *dsb)
218{
219	intel_dsb_align_tail(dsb);
220}
221
222/**
223 * intel_dsb_commit() - Trigger workload execution of DSB.
224 * @dsb: DSB context
225 * @wait_for_vblank: wait for vblank before executing
226 *
227 * This function is used to do actual write to hardware using DSB.
228 */
229void intel_dsb_commit(struct intel_dsb *dsb, bool wait_for_vblank)
230{
231	struct intel_crtc *crtc = dsb->crtc;
232	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
233	enum pipe pipe = crtc->pipe;
234	u32 tail;
235
236	tail = dsb->free_pos * 4;
237	if (drm_WARN_ON(&dev_priv->drm, !IS_ALIGNED(tail, CACHELINE_BYTES)))
238		return;
239
240	if (is_dsb_busy(dev_priv, pipe, dsb->id)) {
241		drm_err(&dev_priv->drm, "[CRTC:%d:%s] DSB %d is busy\n",
242			crtc->base.base.id, crtc->base.name, dsb->id);
243		return;
244	}
245
246	intel_de_write(dev_priv, DSB_CTRL(pipe, dsb->id),
247		       (wait_for_vblank ? DSB_WAIT_FOR_VBLANK : 0) |
248		       DSB_ENABLE);
249	intel_de_write(dev_priv, DSB_HEAD(pipe, dsb->id),
250		       i915_ggtt_offset(dsb->vma));
251	intel_de_write(dev_priv, DSB_TAIL(pipe, dsb->id),
252		       i915_ggtt_offset(dsb->vma) + tail);
253}
254
255void intel_dsb_wait(struct intel_dsb *dsb)
256{
257	struct intel_crtc *crtc = dsb->crtc;
258	struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
259	enum pipe pipe = crtc->pipe;
260
261	if (wait_for(!is_dsb_busy(dev_priv, pipe, dsb->id), 1))
262		drm_err(&dev_priv->drm,
263			"[CRTC:%d:%s] DSB %d timed out waiting for idle\n",
264			crtc->base.base.id, crtc->base.name, dsb->id);
265
266	/* Attempt to reset it */
267	dsb->free_pos = 0;
268	dsb->ins_start_offset = 0;
269	intel_de_write(dev_priv, DSB_CTRL(pipe, dsb->id), 0);
270}
271
272/**
273 * intel_dsb_prepare() - Allocate, pin and map the DSB command buffer.
274 * @crtc: the CRTC
275 * @max_cmds: number of commands we need to fit into command buffer
276 *
277 * This function prepare the command buffer which is used to store dsb
278 * instructions with data.
279 *
280 * Returns:
281 * DSB context, NULL on failure
282 */
283struct intel_dsb *intel_dsb_prepare(struct intel_crtc *crtc,
284				    unsigned int max_cmds)
285{
286	struct drm_i915_private *i915 = to_i915(crtc->base.dev);
287	struct drm_i915_gem_object *obj;
288	intel_wakeref_t wakeref;
289	struct intel_dsb *dsb;
290	struct i915_vma *vma;
291	unsigned int size;
292	u32 *buf;
293
294	if (!HAS_DSB(i915))
295		return NULL;
296
297	dsb = kzalloc(sizeof(*dsb), GFP_KERNEL);
298	if (!dsb)
299		goto out;
300
301	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
302
303	/* ~1 qword per instruction, full cachelines */
304	size = ALIGN(max_cmds * 8, CACHELINE_BYTES);
305
306	obj = i915_gem_object_create_internal(i915, PAGE_ALIGN(size));
307	if (IS_ERR(obj))
308		goto out_put_rpm;
309
310	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
311	if (IS_ERR(vma)) {
312		i915_gem_object_put(obj);
313		goto out_put_rpm;
314	}
315
316	buf = i915_gem_object_pin_map_unlocked(vma->obj, I915_MAP_WC);
317	if (IS_ERR(buf)) {
318		i915_vma_unpin_and_release(&vma, I915_VMA_RELEASE_MAP);
319		goto out_put_rpm;
320	}
321
322	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
323
324	dsb->id = DSB1;
325	dsb->vma = vma;
326	dsb->crtc = crtc;
327	dsb->cmd_buf = buf;
328	dsb->size = size / 4; /* in dwords */
329	dsb->free_pos = 0;
330	dsb->ins_start_offset = 0;
331
332	return dsb;
333
334out_put_rpm:
335	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
336	kfree(dsb);
337out:
338	drm_info_once(&i915->drm,
339		      "[CRTC:%d:%s] DSB %d queue setup failed, will fallback to MMIO for display HW programming\n",
340		      crtc->base.base.id, crtc->base.name, DSB1);
341
342	return NULL;
343}
344
345/**
346 * intel_dsb_cleanup() - To cleanup DSB context.
347 * @dsb: DSB context
348 *
349 * This function cleanup the DSB context by unpinning and releasing
350 * the VMA object associated with it.
351 */
352void intel_dsb_cleanup(struct intel_dsb *dsb)
353{
354	i915_vma_unpin_and_release(&dsb->vma, I915_VMA_RELEASE_MAP);
355	kfree(dsb);
356}
357