1// SPDX-License-Identifier: GPL-2.0-only
2// Miscellaneous Arm SMMU implementation and integration quirks
3// Copyright (C) 2019 Arm Limited
4
5#define pr_fmt(fmt) "arm-smmu: " fmt
6
7#include <linux/bitfield.h>
8#include <linux/of.h>
9
10#include "arm-smmu.h"
11
12
13static int arm_smmu_gr0_ns(int offset)
14{
15	switch(offset) {
16	case ARM_SMMU_GR0_sCR0:
17	case ARM_SMMU_GR0_sACR:
18	case ARM_SMMU_GR0_sGFSR:
19	case ARM_SMMU_GR0_sGFSYNR0:
20	case ARM_SMMU_GR0_sGFSYNR1:
21	case ARM_SMMU_GR0_sGFSYNR2:
22		return offset + 0x400;
23	default:
24		return offset;
25	}
26}
27
28static u32 arm_smmu_read_ns(struct arm_smmu_device *smmu, int page,
29			    int offset)
30{
31	if (page == ARM_SMMU_GR0)
32		offset = arm_smmu_gr0_ns(offset);
33	return readl_relaxed(arm_smmu_page(smmu, page) + offset);
34}
35
36static void arm_smmu_write_ns(struct arm_smmu_device *smmu, int page,
37			      int offset, u32 val)
38{
39	if (page == ARM_SMMU_GR0)
40		offset = arm_smmu_gr0_ns(offset);
41	writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
42}
43
44/* Since we don't care for sGFAR, we can do without 64-bit accessors */
45static const struct arm_smmu_impl calxeda_impl = {
46	.read_reg = arm_smmu_read_ns,
47	.write_reg = arm_smmu_write_ns,
48};
49
50
51struct cavium_smmu {
52	struct arm_smmu_device smmu;
53	u32 id_base;
54};
55
56static int cavium_cfg_probe(struct arm_smmu_device *smmu)
57{
58	static atomic_t context_count = ATOMIC_INIT(0);
59	struct cavium_smmu *cs = container_of(smmu, struct cavium_smmu, smmu);
60	/*
61	 * Cavium CN88xx erratum #27704.
62	 * Ensure ASID and VMID allocation is unique across all SMMUs in
63	 * the system.
64	 */
65	cs->id_base = atomic_fetch_add(smmu->num_context_banks, &context_count);
66	dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
67
68	return 0;
69}
70
71static int cavium_init_context(struct arm_smmu_domain *smmu_domain,
72		struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
73{
74	struct cavium_smmu *cs = container_of(smmu_domain->smmu,
75					      struct cavium_smmu, smmu);
76
77	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
78		smmu_domain->cfg.vmid += cs->id_base;
79	else
80		smmu_domain->cfg.asid += cs->id_base;
81
82	return 0;
83}
84
85static const struct arm_smmu_impl cavium_impl = {
86	.cfg_probe = cavium_cfg_probe,
87	.init_context = cavium_init_context,
88};
89
90static struct arm_smmu_device *cavium_smmu_impl_init(struct arm_smmu_device *smmu)
91{
92	struct cavium_smmu *cs;
93
94	cs = devm_kzalloc(smmu->dev, sizeof(*cs), GFP_KERNEL);
95	if (!cs)
96		return ERR_PTR(-ENOMEM);
97
98	cs->smmu = *smmu;
99	cs->smmu.impl = &cavium_impl;
100
101	devm_kfree(smmu->dev, smmu);
102
103	return &cs->smmu;
104}
105
106
107#define ARM_MMU500_ACTLR_CPRE		(1 << 1)
108
109#define ARM_MMU500_ACR_CACHE_LOCK	(1 << 26)
110#define ARM_MMU500_ACR_S2CRB_TLBEN	(1 << 10)
111#define ARM_MMU500_ACR_SMTNMB_TLBEN	(1 << 8)
112
113int arm_mmu500_reset(struct arm_smmu_device *smmu)
114{
115	u32 reg, major;
116	int i;
117	/*
118	 * On MMU-500 r2p0 onwards we need to clear ACR.CACHE_LOCK before
119	 * writes to the context bank ACTLRs will stick. And we just hope that
120	 * Secure has also cleared SACR.CACHE_LOCK for this to take effect...
121	 */
122	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);
123	major = FIELD_GET(ARM_SMMU_ID7_MAJOR, reg);
124	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);
125	if (major >= 2)
126		reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
127	/*
128	 * Allow unmatched Stream IDs to allocate bypass
129	 * TLB entries for reduced latency.
130	 */
131	reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
132	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, reg);
133
134	/*
135	 * Disable MMU-500's not-particularly-beneficial next-page
136	 * prefetcher for the sake of errata #841119 and #826419.
137	 */
138	for (i = 0; i < smmu->num_context_banks; ++i) {
139		reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
140		reg &= ~ARM_MMU500_ACTLR_CPRE;
141		arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg);
142	}
143
144	return 0;
145}
146
147static const struct arm_smmu_impl arm_mmu500_impl = {
148	.reset = arm_mmu500_reset,
149};
150
151static u64 mrvl_mmu500_readq(struct arm_smmu_device *smmu, int page, int off)
152{
153	/*
154	 * Marvell Armada-AP806 erratum #582743.
155	 * Split all the readq to double readl
156	 */
157	return hi_lo_readq_relaxed(arm_smmu_page(smmu, page) + off);
158}
159
160static void mrvl_mmu500_writeq(struct arm_smmu_device *smmu, int page, int off,
161			       u64 val)
162{
163	/*
164	 * Marvell Armada-AP806 erratum #582743.
165	 * Split all the writeq to double writel
166	 */
167	hi_lo_writeq_relaxed(val, arm_smmu_page(smmu, page) + off);
168}
169
170static int mrvl_mmu500_cfg_probe(struct arm_smmu_device *smmu)
171{
172
173	/*
174	 * Armada-AP806 erratum #582743.
175	 * Hide the SMMU_IDR2.PTFSv8 fields to sidestep the AArch64
176	 * formats altogether and allow using 32 bits access on the
177	 * interconnect.
178	 */
179	smmu->features &= ~(ARM_SMMU_FEAT_FMT_AARCH64_4K |
180			    ARM_SMMU_FEAT_FMT_AARCH64_16K |
181			    ARM_SMMU_FEAT_FMT_AARCH64_64K);
182
183	return 0;
184}
185
186static const struct arm_smmu_impl mrvl_mmu500_impl = {
187	.read_reg64 = mrvl_mmu500_readq,
188	.write_reg64 = mrvl_mmu500_writeq,
189	.cfg_probe = mrvl_mmu500_cfg_probe,
190	.reset = arm_mmu500_reset,
191};
192
193
194struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu)
195{
196	const struct device_node *np = smmu->dev->of_node;
197
198	/*
199	 * Set the impl for model-specific implementation quirks first,
200	 * such that platform integration quirks can pick it up and
201	 * inherit from it if necessary.
202	 */
203	switch (smmu->model) {
204	case ARM_MMU500:
205		smmu->impl = &arm_mmu500_impl;
206		break;
207	case CAVIUM_SMMUV2:
208		return cavium_smmu_impl_init(smmu);
209	default:
210		break;
211	}
212
213	/* This is implicitly MMU-400 */
214	if (of_property_read_bool(np, "calxeda,smmu-secure-config-access"))
215		smmu->impl = &calxeda_impl;
216
217	if (of_device_is_compatible(np, "nvidia,tegra194-smmu"))
218		return nvidia_smmu_impl_init(smmu);
219
220	if (of_device_is_compatible(np, "qcom,sdm845-smmu-500") ||
221	    of_device_is_compatible(np, "qcom,sc7180-smmu-500") ||
222	    of_device_is_compatible(np, "qcom,sm8150-smmu-500") ||
223	    of_device_is_compatible(np, "qcom,sm8250-smmu-500"))
224		return qcom_smmu_impl_init(smmu);
225
226	if (of_device_is_compatible(np, "marvell,ap806-smmu-500"))
227		smmu->impl = &mrvl_mmu500_impl;
228
229	return smmu;
230}
231