1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 *  Based on linux/arch/arm/mm/dma-mapping.c
4 *
5 *  Copyright (C) 2000-2004 Russell King
6 */
7
8#include <linux/export.h>
9#include <linux/mm.h>
10#include <linux/dma-direct.h>
11#include <linux/dma-map-ops.h>
12#include <linux/scatterlist.h>
13
14#include <asm/cachetype.h>
15#include <asm/cacheflush.h>
16#include <asm/outercache.h>
17#include <asm/cp15.h>
18
19#include "dma.h"
20
21/*
22 *  The generic direct mapping code is used if
23 *   - MMU/MPU is off
24 *   - cpu is v7m w/o cache support
25 *   - device is coherent
26 *  otherwise arm_nommu_dma_ops is used.
27 *
28 *  arm_nommu_dma_ops rely on consistent DMA memory (please, refer to
29 *  [1] on how to declare such memory).
30 *
31 *  [1] Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
32 */
33
34static void *arm_nommu_dma_alloc(struct device *dev, size_t size,
35				 dma_addr_t *dma_handle, gfp_t gfp,
36				 unsigned long attrs)
37
38{
39	void *ret = dma_alloc_from_global_coherent(dev, size, dma_handle);
40
41	/*
42	 * dma_alloc_from_global_coherent() may fail because:
43	 *
44	 * - no consistent DMA region has been defined, so we can't
45	 *   continue.
46	 * - there is no space left in consistent DMA region, so we
47	 *   only can fallback to generic allocator if we are
48	 *   advertised that consistency is not required.
49	 */
50
51	WARN_ON_ONCE(ret == NULL);
52	return ret;
53}
54
55static void arm_nommu_dma_free(struct device *dev, size_t size,
56			       void *cpu_addr, dma_addr_t dma_addr,
57			       unsigned long attrs)
58{
59	int ret = dma_release_from_global_coherent(get_order(size), cpu_addr);
60
61	WARN_ON_ONCE(ret == 0);
62}
63
64static int arm_nommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
65			      void *cpu_addr, dma_addr_t dma_addr, size_t size,
66			      unsigned long attrs)
67{
68	int ret;
69
70	if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))
71		return ret;
72	if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
73		return ret;
74	return -ENXIO;
75}
76
77
78static void __dma_page_cpu_to_dev(phys_addr_t paddr, size_t size,
79				  enum dma_data_direction dir)
80{
81	dmac_map_area(__va(paddr), size, dir);
82
83	if (dir == DMA_FROM_DEVICE)
84		outer_inv_range(paddr, paddr + size);
85	else
86		outer_clean_range(paddr, paddr + size);
87}
88
89static void __dma_page_dev_to_cpu(phys_addr_t paddr, size_t size,
90				  enum dma_data_direction dir)
91{
92	if (dir != DMA_TO_DEVICE) {
93		outer_inv_range(paddr, paddr + size);
94		dmac_unmap_area(__va(paddr), size, dir);
95	}
96}
97
98static dma_addr_t arm_nommu_dma_map_page(struct device *dev, struct page *page,
99					 unsigned long offset, size_t size,
100					 enum dma_data_direction dir,
101					 unsigned long attrs)
102{
103	dma_addr_t handle = page_to_phys(page) + offset;
104
105	__dma_page_cpu_to_dev(handle, size, dir);
106
107	return handle;
108}
109
110static void arm_nommu_dma_unmap_page(struct device *dev, dma_addr_t handle,
111				     size_t size, enum dma_data_direction dir,
112				     unsigned long attrs)
113{
114	__dma_page_dev_to_cpu(handle, size, dir);
115}
116
117
118static int arm_nommu_dma_map_sg(struct device *dev, struct scatterlist *sgl,
119				int nents, enum dma_data_direction dir,
120				unsigned long attrs)
121{
122	int i;
123	struct scatterlist *sg;
124
125	for_each_sg(sgl, sg, nents, i) {
126		sg_dma_address(sg) = sg_phys(sg);
127		sg_dma_len(sg) = sg->length;
128		__dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir);
129	}
130
131	return nents;
132}
133
134static void arm_nommu_dma_unmap_sg(struct device *dev, struct scatterlist *sgl,
135				   int nents, enum dma_data_direction dir,
136				   unsigned long attrs)
137{
138	struct scatterlist *sg;
139	int i;
140
141	for_each_sg(sgl, sg, nents, i)
142		__dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir);
143}
144
145static void arm_nommu_dma_sync_single_for_device(struct device *dev,
146		dma_addr_t handle, size_t size, enum dma_data_direction dir)
147{
148	__dma_page_cpu_to_dev(handle, size, dir);
149}
150
151static void arm_nommu_dma_sync_single_for_cpu(struct device *dev,
152		dma_addr_t handle, size_t size, enum dma_data_direction dir)
153{
154	__dma_page_cpu_to_dev(handle, size, dir);
155}
156
157static void arm_nommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
158					     int nents, enum dma_data_direction dir)
159{
160	struct scatterlist *sg;
161	int i;
162
163	for_each_sg(sgl, sg, nents, i)
164		__dma_page_cpu_to_dev(sg_dma_address(sg), sg_dma_len(sg), dir);
165}
166
167static void arm_nommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
168					  int nents, enum dma_data_direction dir)
169{
170	struct scatterlist *sg;
171	int i;
172
173	for_each_sg(sgl, sg, nents, i)
174		__dma_page_dev_to_cpu(sg_dma_address(sg), sg_dma_len(sg), dir);
175}
176
177const struct dma_map_ops arm_nommu_dma_ops = {
178	.alloc			= arm_nommu_dma_alloc,
179	.free			= arm_nommu_dma_free,
180	.alloc_pages		= dma_direct_alloc_pages,
181	.free_pages		= dma_direct_free_pages,
182	.mmap			= arm_nommu_dma_mmap,
183	.map_page		= arm_nommu_dma_map_page,
184	.unmap_page		= arm_nommu_dma_unmap_page,
185	.map_sg			= arm_nommu_dma_map_sg,
186	.unmap_sg		= arm_nommu_dma_unmap_sg,
187	.sync_single_for_device	= arm_nommu_dma_sync_single_for_device,
188	.sync_single_for_cpu	= arm_nommu_dma_sync_single_for_cpu,
189	.sync_sg_for_device	= arm_nommu_dma_sync_sg_for_device,
190	.sync_sg_for_cpu	= arm_nommu_dma_sync_sg_for_cpu,
191};
192EXPORT_SYMBOL(arm_nommu_dma_ops);
193
194void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
195			const struct iommu_ops *iommu, bool coherent)
196{
197	if (IS_ENABLED(CONFIG_CPU_V7M)) {
198		/*
199		 * Cache support for v7m is optional, so can be treated as
200		 * coherent if no cache has been detected. Note that it is not
201		 * enough to check if MPU is in use or not since in absense of
202		 * MPU system memory map is used.
203		 */
204		dev->archdata.dma_coherent = (cacheid) ? coherent : true;
205	} else {
206		/*
207		 * Assume coherent DMA in case MMU/MPU has not been set up.
208		 */
209		dev->archdata.dma_coherent = (get_cr() & CR_M) ? coherent : true;
210	}
211
212	if (!dev->archdata.dma_coherent)
213		set_dma_ops(dev, &arm_nommu_dma_ops);
214}
215