1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Tegra host1x Channel
4 *
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
6 */
7
8#include <linux/host1x.h>
9#include <linux/iommu.h>
10#include <linux/slab.h>
11
12#include <trace/events/host1x.h>
13
14#include "../channel.h"
15#include "../dev.h"
16#include "../intr.h"
17#include "../job.h"
18
19#define TRACE_MAX_LENGTH 128U
20
21static void trace_write_gather(struct host1x_cdma *cdma, struct host1x_bo *bo,
22			       u32 offset, u32 words)
23{
24	struct device *dev = cdma_to_channel(cdma)->dev;
25	void *mem = NULL;
26
27	if (host1x_debug_trace_cmdbuf)
28		mem = host1x_bo_mmap(bo);
29
30	if (mem) {
31		u32 i;
32		/*
33		 * Write in batches of 128 as there seems to be a limit
34		 * of how much you can output to ftrace at once.
35		 */
36		for (i = 0; i < words; i += TRACE_MAX_LENGTH) {
37			u32 num_words = min(words - i, TRACE_MAX_LENGTH);
38
39			offset += i * sizeof(u32);
40
41			trace_host1x_cdma_push_gather(dev_name(dev), bo,
42						      num_words, offset,
43						      mem);
44		}
45
46		host1x_bo_munmap(bo, mem);
47	}
48}
49
50static void submit_gathers(struct host1x_job *job)
51{
52	struct host1x_cdma *cdma = &job->channel->cdma;
53#if HOST1X_HW < 6
54	struct device *dev = job->channel->dev;
55#endif
56	unsigned int i;
57
58	for (i = 0; i < job->num_gathers; i++) {
59		struct host1x_job_gather *g = &job->gathers[i];
60		dma_addr_t addr = g->base + g->offset;
61		u32 op2, op3;
62
63		op2 = lower_32_bits(addr);
64		op3 = upper_32_bits(addr);
65
66		trace_write_gather(cdma, g->bo, g->offset, g->words);
67
68		if (op3 != 0) {
69#if HOST1X_HW >= 6
70			u32 op1 = host1x_opcode_gather_wide(g->words);
71			u32 op4 = HOST1X_OPCODE_NOP;
72
73			host1x_cdma_push_wide(cdma, op1, op2, op3, op4);
74#else
75			dev_err(dev, "invalid gather for push buffer %pad\n",
76				&addr);
77			continue;
78#endif
79		} else {
80			u32 op1 = host1x_opcode_gather(g->words);
81
82			host1x_cdma_push(cdma, op1, op2);
83		}
84	}
85}
86
87static inline void synchronize_syncpt_base(struct host1x_job *job)
88{
89	struct host1x *host = dev_get_drvdata(job->channel->dev->parent);
90	struct host1x_syncpt *sp = host->syncpt + job->syncpt_id;
91	unsigned int id;
92	u32 value;
93
94	value = host1x_syncpt_read_max(sp);
95	id = sp->base->id;
96
97	host1x_cdma_push(&job->channel->cdma,
98			 host1x_opcode_setclass(HOST1X_CLASS_HOST1X,
99				HOST1X_UCLASS_LOAD_SYNCPT_BASE, 1),
100			 HOST1X_UCLASS_LOAD_SYNCPT_BASE_BASE_INDX_F(id) |
101			 HOST1X_UCLASS_LOAD_SYNCPT_BASE_VALUE_F(value));
102}
103
104static void host1x_channel_set_streamid(struct host1x_channel *channel)
105{
106#if HOST1X_HW >= 6
107	u32 sid = 0x7f;
108#ifdef CONFIG_IOMMU_API
109	struct iommu_fwspec *spec = dev_iommu_fwspec_get(channel->dev->parent);
110	if (spec)
111		sid = spec->ids[0] & 0xffff;
112#endif
113
114	host1x_ch_writel(channel, sid, HOST1X_CHANNEL_SMMU_STREAMID);
115#endif
116}
117
118static int channel_submit(struct host1x_job *job)
119{
120	struct host1x_channel *ch = job->channel;
121	struct host1x_syncpt *sp;
122	u32 user_syncpt_incrs = job->syncpt_incrs;
123	u32 prev_max = 0;
124	u32 syncval;
125	int err;
126	struct host1x_waitlist *completed_waiter = NULL;
127	struct host1x *host = dev_get_drvdata(ch->dev->parent);
128
129	sp = host->syncpt + job->syncpt_id;
130	trace_host1x_channel_submit(dev_name(ch->dev),
131				    job->num_gathers, job->num_relocs,
132				    job->syncpt_id, job->syncpt_incrs);
133
134	/* before error checks, return current max */
135	prev_max = job->syncpt_end = host1x_syncpt_read_max(sp);
136
137	/* get submit lock */
138	err = mutex_lock_interruptible(&ch->submitlock);
139	if (err)
140		goto error;
141
142	completed_waiter = kzalloc(sizeof(*completed_waiter), GFP_KERNEL);
143	if (!completed_waiter) {
144		mutex_unlock(&ch->submitlock);
145		err = -ENOMEM;
146		goto error;
147	}
148
149	host1x_channel_set_streamid(ch);
150
151	/* begin a CDMA submit */
152	err = host1x_cdma_begin(&ch->cdma, job);
153	if (err) {
154		mutex_unlock(&ch->submitlock);
155		goto error;
156	}
157
158	if (job->serialize) {
159		/*
160		 * Force serialization by inserting a host wait for the
161		 * previous job to finish before this one can commence.
162		 */
163		host1x_cdma_push(&ch->cdma,
164				 host1x_opcode_setclass(HOST1X_CLASS_HOST1X,
165					host1x_uclass_wait_syncpt_r(), 1),
166				 host1x_class_host_wait_syncpt(job->syncpt_id,
167					host1x_syncpt_read_max(sp)));
168	}
169
170	/* Synchronize base register to allow using it for relative waiting */
171	if (sp->base)
172		synchronize_syncpt_base(job);
173
174	syncval = host1x_syncpt_incr_max(sp, user_syncpt_incrs);
175
176	host1x_hw_syncpt_assign_to_channel(host, sp, ch);
177
178	job->syncpt_end = syncval;
179
180	/* add a setclass for modules that require it */
181	if (job->class)
182		host1x_cdma_push(&ch->cdma,
183				 host1x_opcode_setclass(job->class, 0, 0),
184				 HOST1X_OPCODE_NOP);
185
186	submit_gathers(job);
187
188	/* end CDMA submit & stash pinned hMems into sync queue */
189	host1x_cdma_end(&ch->cdma, job);
190
191	trace_host1x_channel_submitted(dev_name(ch->dev), prev_max, syncval);
192
193	/* schedule a submit complete interrupt */
194	err = host1x_intr_add_action(host, sp, syncval,
195				     HOST1X_INTR_ACTION_SUBMIT_COMPLETE, ch,
196				     completed_waiter, NULL);
197	completed_waiter = NULL;
198	WARN(err, "Failed to set submit complete interrupt");
199
200	mutex_unlock(&ch->submitlock);
201
202	return 0;
203
204error:
205	kfree(completed_waiter);
206	return err;
207}
208
209static void enable_gather_filter(struct host1x *host,
210				 struct host1x_channel *ch)
211{
212#if HOST1X_HW >= 6
213	u32 val;
214
215	if (!host->hv_regs)
216		return;
217
218	val = host1x_hypervisor_readl(
219		host, HOST1X_HV_CH_KERNEL_FILTER_GBUFFER(ch->id / 32));
220	val |= BIT(ch->id % 32);
221	host1x_hypervisor_writel(
222		host, val, HOST1X_HV_CH_KERNEL_FILTER_GBUFFER(ch->id / 32));
223#elif HOST1X_HW >= 4
224	host1x_ch_writel(ch,
225			 HOST1X_CHANNEL_CHANNELCTRL_KERNEL_FILTER_GBUFFER(1),
226			 HOST1X_CHANNEL_CHANNELCTRL);
227#endif
228}
229
230static int host1x_channel_init(struct host1x_channel *ch, struct host1x *dev,
231			       unsigned int index)
232{
233#if HOST1X_HW < 6
234	ch->regs = dev->regs + index * 0x4000;
235#else
236	ch->regs = dev->regs + index * 0x100;
237#endif
238	enable_gather_filter(dev, ch);
239	return 0;
240}
241
242static const struct host1x_channel_ops host1x_channel_ops = {
243	.init = host1x_channel_init,
244	.submit = channel_submit,
245};
246