1/*
2 * Copyright 2018 Advanced Micro Devices, Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22
23#include <linux/kconfig.h>
24
25#if IS_REACHABLE(CONFIG_AMD_IOMMU_V2)
26
27#include <linux/printk.h>
28#include <linux/device.h>
29#include <linux/slab.h>
30#include <linux/pci.h>
31#include <linux/amd-iommu.h>
32#include "kfd_priv.h"
33#include "kfd_dbgmgr.h"
34#include "kfd_topology.h"
35#include "kfd_iommu.h"
36
37static const u32 required_iommu_flags = AMD_IOMMU_DEVICE_FLAG_ATS_SUP |
38					AMD_IOMMU_DEVICE_FLAG_PRI_SUP |
39					AMD_IOMMU_DEVICE_FLAG_PASID_SUP;
40
41/** kfd_iommu_check_device - Check whether IOMMU is available for device
42 */
43int kfd_iommu_check_device(struct kfd_dev *kfd)
44{
45	struct amd_iommu_device_info iommu_info;
46	int err;
47
48	if (!kfd->use_iommu_v2)
49		return -ENODEV;
50
51	iommu_info.flags = 0;
52	err = amd_iommu_device_info(kfd->pdev, &iommu_info);
53	if (err)
54		return err;
55
56	if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags)
57		return -ENODEV;
58
59	return 0;
60}
61
62/** kfd_iommu_device_init - Initialize IOMMU for device
63 */
64int kfd_iommu_device_init(struct kfd_dev *kfd)
65{
66	struct amd_iommu_device_info iommu_info;
67	unsigned int pasid_limit;
68	int err;
69
70	if (!kfd->use_iommu_v2)
71		return 0;
72
73	iommu_info.flags = 0;
74	err = amd_iommu_device_info(kfd->pdev, &iommu_info);
75	if (err < 0) {
76		dev_err(kfd_device,
77			"error getting iommu info. is the iommu enabled?\n");
78		return -ENODEV;
79	}
80
81	if ((iommu_info.flags & required_iommu_flags) != required_iommu_flags) {
82		dev_err(kfd_device,
83			"error required iommu flags ats %i, pri %i, pasid %i\n",
84		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_ATS_SUP) != 0,
85		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PRI_SUP) != 0,
86		       (iommu_info.flags & AMD_IOMMU_DEVICE_FLAG_PASID_SUP)
87									!= 0);
88		return -ENODEV;
89	}
90
91	pasid_limit = min_t(unsigned int,
92			(unsigned int)(1 << kfd->device_info->max_pasid_bits),
93			iommu_info.max_pasids);
94
95	if (!kfd_set_pasid_limit(pasid_limit)) {
96		dev_err(kfd_device, "error setting pasid limit\n");
97		return -EBUSY;
98	}
99
100	return 0;
101}
102
103/** kfd_iommu_bind_process_to_device - Have the IOMMU bind a process
104 *
105 * Binds the given process to the given device using its PASID. This
106 * enables IOMMUv2 address translation for the process on the device.
107 *
108 * This function assumes that the process mutex is held.
109 */
110int kfd_iommu_bind_process_to_device(struct kfd_process_device *pdd)
111{
112	struct kfd_dev *dev = pdd->dev;
113	struct kfd_process *p = pdd->process;
114	int err;
115
116	if (!dev->use_iommu_v2 || pdd->bound == PDD_BOUND)
117		return 0;
118
119	if (unlikely(pdd->bound == PDD_BOUND_SUSPENDED)) {
120		pr_err("Binding PDD_BOUND_SUSPENDED pdd is unexpected!\n");
121		return -EINVAL;
122	}
123
124	err = amd_iommu_bind_pasid(dev->pdev, p->pasid, p->lead_thread);
125	if (!err)
126		pdd->bound = PDD_BOUND;
127
128	return err;
129}
130
131/** kfd_iommu_unbind_process - Unbind process from all devices
132 *
133 * This removes all IOMMU device bindings of the process. To be used
134 * before process termination.
135 */
136void kfd_iommu_unbind_process(struct kfd_process *p)
137{
138	struct kfd_process_device *pdd;
139
140	list_for_each_entry(pdd, &p->per_device_data, per_device_list)
141		if (pdd->bound == PDD_BOUND)
142			amd_iommu_unbind_pasid(pdd->dev->pdev, p->pasid);
143}
144
145/* Callback for process shutdown invoked by the IOMMU driver */
146static void iommu_pasid_shutdown_callback(struct pci_dev *pdev, u32 pasid)
147{
148	struct kfd_dev *dev = kfd_device_by_pci_dev(pdev);
149	struct kfd_process *p;
150	struct kfd_process_device *pdd;
151
152	if (!dev)
153		return;
154
155	/*
156	 * Look for the process that matches the pasid. If there is no such
157	 * process, we either released it in amdkfd's own notifier, or there
158	 * is a bug. Unfortunately, there is no way to tell...
159	 */
160	p = kfd_lookup_process_by_pasid(pasid);
161	if (!p)
162		return;
163
164	pr_debug("Unbinding process 0x%x from IOMMU\n", pasid);
165
166	mutex_lock(kfd_get_dbgmgr_mutex());
167
168	if (dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
169		if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
170			kfd_dbgmgr_destroy(dev->dbgmgr);
171			dev->dbgmgr = NULL;
172		}
173	}
174
175	mutex_unlock(kfd_get_dbgmgr_mutex());
176
177	mutex_lock(&p->mutex);
178
179	pdd = kfd_get_process_device_data(dev, p);
180	if (pdd)
181		/* For GPU relying on IOMMU, we need to dequeue here
182		 * when PASID is still bound.
183		 */
184		kfd_process_dequeue_from_device(pdd);
185
186	mutex_unlock(&p->mutex);
187
188	kfd_unref_process(p);
189}
190
191/* This function called by IOMMU driver on PPR failure */
192static int iommu_invalid_ppr_cb(struct pci_dev *pdev, u32 pasid,
193				unsigned long address, u16 flags)
194{
195	struct kfd_dev *dev;
196
197	dev_warn_ratelimited(kfd_device,
198			"Invalid PPR device %x:%x.%x pasid 0x%x address 0x%lX flags 0x%X",
199			pdev->bus->number,
200			PCI_SLOT(pdev->devfn),
201			PCI_FUNC(pdev->devfn),
202			pasid,
203			address,
204			flags);
205
206	dev = kfd_device_by_pci_dev(pdev);
207	if (!WARN_ON(!dev))
208		kfd_signal_iommu_event(dev, pasid, address,
209			flags & PPR_FAULT_WRITE, flags & PPR_FAULT_EXEC);
210
211	return AMD_IOMMU_INV_PRI_RSP_INVALID;
212}
213
214/*
215 * Bind processes do the device that have been temporarily unbound
216 * (PDD_BOUND_SUSPENDED) in kfd_unbind_processes_from_device.
217 */
218static int kfd_bind_processes_to_device(struct kfd_dev *kfd)
219{
220	struct kfd_process_device *pdd;
221	struct kfd_process *p;
222	unsigned int temp;
223	int err = 0;
224
225	int idx = srcu_read_lock(&kfd_processes_srcu);
226
227	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
228		mutex_lock(&p->mutex);
229		pdd = kfd_get_process_device_data(kfd, p);
230
231		if (WARN_ON(!pdd) || pdd->bound != PDD_BOUND_SUSPENDED) {
232			mutex_unlock(&p->mutex);
233			continue;
234		}
235
236		err = amd_iommu_bind_pasid(kfd->pdev, p->pasid,
237				p->lead_thread);
238		if (err < 0) {
239			pr_err("Unexpected pasid 0x%x binding failure\n",
240					p->pasid);
241			mutex_unlock(&p->mutex);
242			break;
243		}
244
245		pdd->bound = PDD_BOUND;
246		mutex_unlock(&p->mutex);
247	}
248
249	srcu_read_unlock(&kfd_processes_srcu, idx);
250
251	return err;
252}
253
254/*
255 * Mark currently bound processes as PDD_BOUND_SUSPENDED. These
256 * processes will be restored to PDD_BOUND state in
257 * kfd_bind_processes_to_device.
258 */
259static void kfd_unbind_processes_from_device(struct kfd_dev *kfd)
260{
261	struct kfd_process_device *pdd;
262	struct kfd_process *p;
263	unsigned int temp;
264
265	int idx = srcu_read_lock(&kfd_processes_srcu);
266
267	hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
268		mutex_lock(&p->mutex);
269		pdd = kfd_get_process_device_data(kfd, p);
270
271		if (WARN_ON(!pdd)) {
272			mutex_unlock(&p->mutex);
273			continue;
274		}
275
276		if (pdd->bound == PDD_BOUND)
277			pdd->bound = PDD_BOUND_SUSPENDED;
278		mutex_unlock(&p->mutex);
279	}
280
281	srcu_read_unlock(&kfd_processes_srcu, idx);
282}
283
284/** kfd_iommu_suspend - Prepare IOMMU for suspend
285 *
286 * This unbinds processes from the device and disables the IOMMU for
287 * the device.
288 */
289void kfd_iommu_suspend(struct kfd_dev *kfd)
290{
291	if (!kfd->use_iommu_v2)
292		return;
293
294	kfd_unbind_processes_from_device(kfd);
295
296	amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
297	amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
298	amd_iommu_free_device(kfd->pdev);
299}
300
301/** kfd_iommu_resume - Restore IOMMU after resume
302 *
303 * This reinitializes the IOMMU for the device and re-binds previously
304 * suspended processes to the device.
305 */
306int kfd_iommu_resume(struct kfd_dev *kfd)
307{
308	unsigned int pasid_limit;
309	int err;
310
311	if (!kfd->use_iommu_v2)
312		return 0;
313
314	pasid_limit = kfd_get_pasid_limit();
315
316	err = amd_iommu_init_device(kfd->pdev, pasid_limit);
317	if (err)
318		return -ENXIO;
319
320	amd_iommu_set_invalidate_ctx_cb(kfd->pdev,
321					iommu_pasid_shutdown_callback);
322	amd_iommu_set_invalid_ppr_cb(kfd->pdev,
323				     iommu_invalid_ppr_cb);
324
325	err = kfd_bind_processes_to_device(kfd);
326	if (err) {
327		amd_iommu_set_invalidate_ctx_cb(kfd->pdev, NULL);
328		amd_iommu_set_invalid_ppr_cb(kfd->pdev, NULL);
329		amd_iommu_free_device(kfd->pdev);
330		return err;
331	}
332
333	return 0;
334}
335
336extern bool amd_iommu_pc_supported(void);
337extern u8 amd_iommu_pc_get_max_banks(u16 devid);
338extern u8 amd_iommu_pc_get_max_counters(u16 devid);
339
340/** kfd_iommu_add_perf_counters - Add IOMMU performance counters to topology
341 */
342int kfd_iommu_add_perf_counters(struct kfd_topology_device *kdev)
343{
344	struct kfd_perf_properties *props;
345
346	if (!(kdev->node_props.capability & HSA_CAP_ATS_PRESENT))
347		return 0;
348
349	if (!amd_iommu_pc_supported())
350		return 0;
351
352	props = kfd_alloc_struct(props);
353	if (!props)
354		return -ENOMEM;
355	strcpy(props->block_name, "iommu");
356	props->max_concurrent = amd_iommu_pc_get_max_banks(0) *
357		amd_iommu_pc_get_max_counters(0); /* assume one iommu */
358	list_add_tail(&props->list, &kdev->perf_props);
359
360	return 0;
361}
362
363#endif
364