1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 *  IOMMU helpers in MMU context.
4 *
5 *  Copyright (C) 2015 IBM Corp. <aik@ozlabs.ru>
6 */
7
8#include <linux/sched/signal.h>
9#include <linux/slab.h>
10#include <linux/rculist.h>
11#include <linux/vmalloc.h>
12#include <linux/mutex.h>
13#include <linux/migrate.h>
14#include <linux/hugetlb.h>
15#include <linux/swap.h>
16#include <linux/sizes.h>
17#include <linux/mm.h>
18#include <asm/mmu_context.h>
19#include <asm/pte-walk.h>
20#include <linux/mm_inline.h>
21
22static DEFINE_MUTEX(mem_list_mutex);
23
24#define MM_IOMMU_TABLE_GROUP_PAGE_DIRTY	0x1
25#define MM_IOMMU_TABLE_GROUP_PAGE_MASK	~(SZ_4K - 1)
26
27struct mm_iommu_table_group_mem_t {
28	struct list_head next;
29	struct rcu_head rcu;
30	unsigned long used;
31	atomic64_t mapped;
32	unsigned int pageshift;
33	u64 ua;			/* userspace address */
34	u64 entries;		/* number of entries in hpas/hpages[] */
35	/*
36	 * in mm_iommu_get we temporarily use this to store
37	 * struct page address.
38	 *
39	 * We need to convert ua to hpa in real mode. Make it
40	 * simpler by storing physical address.
41	 */
42	union {
43		struct page **hpages;	/* vmalloc'ed */
44		phys_addr_t *hpas;
45	};
46#define MM_IOMMU_TABLE_INVALID_HPA	((uint64_t)-1)
47	u64 dev_hpa;		/* Device memory base address */
48};
49
50bool mm_iommu_preregistered(struct mm_struct *mm)
51{
52	return !list_empty(&mm->context.iommu_group_mem_list);
53}
54EXPORT_SYMBOL_GPL(mm_iommu_preregistered);
55
56static long mm_iommu_do_alloc(struct mm_struct *mm, unsigned long ua,
57			      unsigned long entries, unsigned long dev_hpa,
58			      struct mm_iommu_table_group_mem_t **pmem)
59{
60	struct mm_iommu_table_group_mem_t *mem, *mem2;
61	long i, ret, locked_entries = 0, pinned = 0;
62	unsigned int pageshift;
63	unsigned long entry, chunk;
64
65	if (dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) {
66		ret = account_locked_vm(mm, entries, true);
67		if (ret)
68			return ret;
69
70		locked_entries = entries;
71	}
72
73	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
74	if (!mem) {
75		ret = -ENOMEM;
76		goto unlock_exit;
77	}
78
79	if (dev_hpa != MM_IOMMU_TABLE_INVALID_HPA) {
80		mem->pageshift = __ffs(dev_hpa | (entries << PAGE_SHIFT));
81		mem->dev_hpa = dev_hpa;
82		goto good_exit;
83	}
84	mem->dev_hpa = MM_IOMMU_TABLE_INVALID_HPA;
85
86	/*
87	 * For a starting point for a maximum page size calculation
88	 * we use @ua and @entries natural alignment to allow IOMMU pages
89	 * smaller than huge pages but still bigger than PAGE_SIZE.
90	 */
91	mem->pageshift = __ffs(ua | (entries << PAGE_SHIFT));
92	mem->hpas = vzalloc(array_size(entries, sizeof(mem->hpas[0])));
93	if (!mem->hpas) {
94		kfree(mem);
95		ret = -ENOMEM;
96		goto unlock_exit;
97	}
98
99	mmap_read_lock(mm);
100	chunk = (1UL << (PAGE_SHIFT + MAX_ORDER)) /
101			sizeof(struct vm_area_struct *);
102	chunk = min(chunk, entries);
103	for (entry = 0; entry < entries; entry += chunk) {
104		unsigned long n = min(entries - entry, chunk);
105
106		ret = pin_user_pages(ua + (entry << PAGE_SHIFT), n,
107				FOLL_WRITE | FOLL_LONGTERM,
108				mem->hpages + entry);
109		if (ret == n) {
110			pinned += n;
111			continue;
112		}
113		if (ret > 0)
114			pinned += ret;
115		break;
116	}
117	mmap_read_unlock(mm);
118	if (pinned != entries) {
119		if (!ret)
120			ret = -EFAULT;
121		goto free_exit;
122	}
123
124good_exit:
125	atomic64_set(&mem->mapped, 1);
126	mem->used = 1;
127	mem->ua = ua;
128	mem->entries = entries;
129
130	mutex_lock(&mem_list_mutex);
131
132	list_for_each_entry_rcu(mem2, &mm->context.iommu_group_mem_list, next,
133				lockdep_is_held(&mem_list_mutex)) {
134		/* Overlap? */
135		if ((mem2->ua < (ua + (entries << PAGE_SHIFT))) &&
136				(ua < (mem2->ua +
137				       (mem2->entries << PAGE_SHIFT)))) {
138			ret = -EINVAL;
139			mutex_unlock(&mem_list_mutex);
140			goto free_exit;
141		}
142	}
143
144	if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA) {
145		/*
146		 * Allow to use larger than 64k IOMMU pages. Only do that
147		 * if we are backed by hugetlb. Skip device memory as it is not
148		 * backed with page structs.
149		 */
150		pageshift = PAGE_SHIFT;
151		for (i = 0; i < entries; ++i) {
152			struct page *page = mem->hpages[i];
153
154			if ((mem->pageshift > PAGE_SHIFT) && PageHuge(page))
155				pageshift = page_shift(compound_head(page));
156			mem->pageshift = min(mem->pageshift, pageshift);
157			/*
158			 * We don't need struct page reference any more, switch
159			 * to physical address.
160			 */
161			mem->hpas[i] = page_to_pfn(page) << PAGE_SHIFT;
162		}
163	}
164
165	list_add_rcu(&mem->next, &mm->context.iommu_group_mem_list);
166
167	mutex_unlock(&mem_list_mutex);
168
169	*pmem = mem;
170
171	return 0;
172
173free_exit:
174	/* free the references taken */
175	unpin_user_pages(mem->hpages, pinned);
176
177	vfree(mem->hpas);
178	kfree(mem);
179
180unlock_exit:
181	account_locked_vm(mm, locked_entries, false);
182
183	return ret;
184}
185
186long mm_iommu_new(struct mm_struct *mm, unsigned long ua, unsigned long entries,
187		struct mm_iommu_table_group_mem_t **pmem)
188{
189	return mm_iommu_do_alloc(mm, ua, entries, MM_IOMMU_TABLE_INVALID_HPA,
190			pmem);
191}
192EXPORT_SYMBOL_GPL(mm_iommu_new);
193
194long mm_iommu_newdev(struct mm_struct *mm, unsigned long ua,
195		unsigned long entries, unsigned long dev_hpa,
196		struct mm_iommu_table_group_mem_t **pmem)
197{
198	return mm_iommu_do_alloc(mm, ua, entries, dev_hpa, pmem);
199}
200EXPORT_SYMBOL_GPL(mm_iommu_newdev);
201
202static void mm_iommu_unpin(struct mm_iommu_table_group_mem_t *mem)
203{
204	long i;
205	struct page *page = NULL;
206
207	if (!mem->hpas)
208		return;
209
210	for (i = 0; i < mem->entries; ++i) {
211		if (!mem->hpas[i])
212			continue;
213
214		page = pfn_to_page(mem->hpas[i] >> PAGE_SHIFT);
215		if (!page)
216			continue;
217
218		if (mem->hpas[i] & MM_IOMMU_TABLE_GROUP_PAGE_DIRTY)
219			SetPageDirty(page);
220
221		unpin_user_page(page);
222
223		mem->hpas[i] = 0;
224	}
225}
226
227static void mm_iommu_do_free(struct mm_iommu_table_group_mem_t *mem)
228{
229
230	mm_iommu_unpin(mem);
231	vfree(mem->hpas);
232	kfree(mem);
233}
234
235static void mm_iommu_free(struct rcu_head *head)
236{
237	struct mm_iommu_table_group_mem_t *mem = container_of(head,
238			struct mm_iommu_table_group_mem_t, rcu);
239
240	mm_iommu_do_free(mem);
241}
242
243static void mm_iommu_release(struct mm_iommu_table_group_mem_t *mem)
244{
245	list_del_rcu(&mem->next);
246	call_rcu(&mem->rcu, mm_iommu_free);
247}
248
249long mm_iommu_put(struct mm_struct *mm, struct mm_iommu_table_group_mem_t *mem)
250{
251	long ret = 0;
252	unsigned long unlock_entries = 0;
253
254	mutex_lock(&mem_list_mutex);
255
256	if (mem->used == 0) {
257		ret = -ENOENT;
258		goto unlock_exit;
259	}
260
261	--mem->used;
262	/* There are still users, exit */
263	if (mem->used)
264		goto unlock_exit;
265
266	/* Are there still mappings? */
267	if (atomic64_cmpxchg(&mem->mapped, 1, 0) != 1) {
268		++mem->used;
269		ret = -EBUSY;
270		goto unlock_exit;
271	}
272
273	if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA)
274		unlock_entries = mem->entries;
275
276	/* @mapped became 0 so now mappings are disabled, release the region */
277	mm_iommu_release(mem);
278
279unlock_exit:
280	mutex_unlock(&mem_list_mutex);
281
282	account_locked_vm(mm, unlock_entries, false);
283
284	return ret;
285}
286EXPORT_SYMBOL_GPL(mm_iommu_put);
287
288struct mm_iommu_table_group_mem_t *mm_iommu_lookup(struct mm_struct *mm,
289		unsigned long ua, unsigned long size)
290{
291	struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
292
293	rcu_read_lock();
294	list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
295		if ((mem->ua <= ua) &&
296				(ua + size <= mem->ua +
297				 (mem->entries << PAGE_SHIFT))) {
298			ret = mem;
299			break;
300		}
301	}
302	rcu_read_unlock();
303
304	return ret;
305}
306EXPORT_SYMBOL_GPL(mm_iommu_lookup);
307
308struct mm_iommu_table_group_mem_t *mm_iommu_get(struct mm_struct *mm,
309		unsigned long ua, unsigned long entries)
310{
311	struct mm_iommu_table_group_mem_t *mem, *ret = NULL;
312
313	mutex_lock(&mem_list_mutex);
314
315	list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next,
316				lockdep_is_held(&mem_list_mutex)) {
317		if ((mem->ua == ua) && (mem->entries == entries)) {
318			ret = mem;
319			++mem->used;
320			break;
321		}
322	}
323
324	mutex_unlock(&mem_list_mutex);
325
326	return ret;
327}
328EXPORT_SYMBOL_GPL(mm_iommu_get);
329
330long mm_iommu_ua_to_hpa(struct mm_iommu_table_group_mem_t *mem,
331		unsigned long ua, unsigned int pageshift, unsigned long *hpa)
332{
333	const long entry = (ua - mem->ua) >> PAGE_SHIFT;
334	u64 *va;
335
336	if (entry >= mem->entries)
337		return -EFAULT;
338
339	if (pageshift > mem->pageshift)
340		return -EFAULT;
341
342	if (!mem->hpas) {
343		*hpa = mem->dev_hpa + (ua - mem->ua);
344		return 0;
345	}
346
347	va = &mem->hpas[entry];
348	*hpa = (*va & MM_IOMMU_TABLE_GROUP_PAGE_MASK) | (ua & ~PAGE_MASK);
349
350	return 0;
351}
352EXPORT_SYMBOL_GPL(mm_iommu_ua_to_hpa);
353
354bool mm_iommu_is_devmem(struct mm_struct *mm, unsigned long hpa,
355		unsigned int pageshift, unsigned long *size)
356{
357	struct mm_iommu_table_group_mem_t *mem;
358	unsigned long end;
359
360	rcu_read_lock();
361	list_for_each_entry_rcu(mem, &mm->context.iommu_group_mem_list, next) {
362		if (mem->dev_hpa == MM_IOMMU_TABLE_INVALID_HPA)
363			continue;
364
365		end = mem->dev_hpa + (mem->entries << PAGE_SHIFT);
366		if ((mem->dev_hpa <= hpa) && (hpa < end)) {
367			/*
368			 * Since the IOMMU page size might be bigger than
369			 * PAGE_SIZE, the amount of preregistered memory
370			 * starting from @hpa might be smaller than 1<<pageshift
371			 * and the caller needs to distinguish this situation.
372			 */
373			*size = min(1UL << pageshift, end - hpa);
374			return true;
375		}
376	}
377	rcu_read_unlock();
378
379	return false;
380}
381EXPORT_SYMBOL_GPL(mm_iommu_is_devmem);
382
383long mm_iommu_mapped_inc(struct mm_iommu_table_group_mem_t *mem)
384{
385	if (atomic64_inc_not_zero(&mem->mapped))
386		return 0;
387
388	/* Last mm_iommu_put() has been called, no more mappings allowed() */
389	return -ENXIO;
390}
391EXPORT_SYMBOL_GPL(mm_iommu_mapped_inc);
392
393void mm_iommu_mapped_dec(struct mm_iommu_table_group_mem_t *mem)
394{
395	atomic64_add_unless(&mem->mapped, -1, 1);
396}
397EXPORT_SYMBOL_GPL(mm_iommu_mapped_dec);
398
399void mm_iommu_init(struct mm_struct *mm)
400{
401	INIT_LIST_HEAD_RCU(&mm->context.iommu_group_mem_list);
402}
403