1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/compat.h>
3#include <linux/dma-mapping.h>
4#include <linux/iommu.h>
5#include <linux/module.h>
6#include <linux/poll.h>
7#include <linux/slab.h>
8#include <linux/uacce.h>
9
10static struct class *uacce_class;
11static dev_t uacce_devt;
12static DEFINE_XARRAY_ALLOC(uacce_xa);
13
14/*
15 * If the parent driver or the device disappears, the queue state is invalid and
16 * ops are not usable anymore.
17 */
18static bool uacce_queue_is_valid(struct uacce_queue *q)
19{
20	return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED;
21}
22
23static int uacce_start_queue(struct uacce_queue *q)
24{
25	int ret;
26
27	if (q->state != UACCE_Q_INIT)
28		return -EINVAL;
29
30	if (q->uacce->ops->start_queue) {
31		ret = q->uacce->ops->start_queue(q);
32		if (ret < 0)
33			return ret;
34	}
35
36	q->state = UACCE_Q_STARTED;
37	return 0;
38}
39
40static int uacce_put_queue(struct uacce_queue *q)
41{
42	struct uacce_device *uacce = q->uacce;
43
44	if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue)
45		uacce->ops->stop_queue(q);
46
47	if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) &&
48	     uacce->ops->put_queue)
49		uacce->ops->put_queue(q);
50
51	q->state = UACCE_Q_ZOMBIE;
52
53	return 0;
54}
55
56static long uacce_fops_unl_ioctl(struct file *filep,
57				 unsigned int cmd, unsigned long arg)
58{
59	struct uacce_queue *q = filep->private_data;
60	struct uacce_device *uacce = q->uacce;
61	long ret = -ENXIO;
62
63	/*
64	 * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from
65	 * user. Avoid a circular lock dependency with uacce_fops_mmap(), which
66	 * gets called with mmap_lock held, by taking uacce->mutex instead of
67	 * q->mutex. Doing this in uacce_fops_mmap() is not possible because
68	 * uacce_fops_open() calls iommu_sva_bind_device(), which takes
69	 * mmap_lock, while holding uacce->mutex.
70	 */
71	mutex_lock(&uacce->mutex);
72	if (!uacce_queue_is_valid(q))
73		goto out_unlock;
74
75	switch (cmd) {
76	case UACCE_CMD_START_Q:
77		ret = uacce_start_queue(q);
78		break;
79	case UACCE_CMD_PUT_Q:
80		ret = uacce_put_queue(q);
81		break;
82	default:
83		if (uacce->ops->ioctl)
84			ret = uacce->ops->ioctl(q, cmd, arg);
85		else
86			ret = -EINVAL;
87	}
88out_unlock:
89	mutex_unlock(&uacce->mutex);
90	return ret;
91}
92
93#ifdef CONFIG_COMPAT
94static long uacce_fops_compat_ioctl(struct file *filep,
95				   unsigned int cmd, unsigned long arg)
96{
97	arg = (unsigned long)compat_ptr(arg);
98
99	return uacce_fops_unl_ioctl(filep, cmd, arg);
100}
101#endif
102
103static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)
104{
105	u32 pasid;
106	struct iommu_sva *handle;
107
108	if (!(uacce->flags & UACCE_DEV_SVA))
109		return 0;
110
111	handle = iommu_sva_bind_device(uacce->parent, current->mm, NULL);
112	if (IS_ERR(handle))
113		return PTR_ERR(handle);
114
115	pasid = iommu_sva_get_pasid(handle);
116	if (pasid == IOMMU_PASID_INVALID) {
117		iommu_sva_unbind_device(handle);
118		return -ENODEV;
119	}
120
121	q->handle = handle;
122	q->pasid = pasid;
123	return 0;
124}
125
126static void uacce_unbind_queue(struct uacce_queue *q)
127{
128	if (!q->handle)
129		return;
130	iommu_sva_unbind_device(q->handle);
131	q->handle = NULL;
132}
133
134static int uacce_fops_open(struct inode *inode, struct file *filep)
135{
136	struct uacce_device *uacce;
137	struct uacce_queue *q;
138	int ret = 0;
139
140	uacce = xa_load(&uacce_xa, iminor(inode));
141	if (!uacce)
142		return -ENODEV;
143
144	q = kzalloc(sizeof(struct uacce_queue), GFP_KERNEL);
145	if (!q)
146		return -ENOMEM;
147
148	mutex_lock(&uacce->mutex);
149
150	if (!uacce->parent) {
151		ret = -EINVAL;
152		goto out_with_mem;
153	}
154
155	ret = uacce_bind_queue(uacce, q);
156	if (ret)
157		goto out_with_mem;
158
159	q->uacce = uacce;
160
161	if (uacce->ops->get_queue) {
162		ret = uacce->ops->get_queue(uacce, q->pasid, q);
163		if (ret < 0)
164			goto out_with_bond;
165	}
166
167	init_waitqueue_head(&q->wait);
168	filep->private_data = q;
169	uacce->inode = inode;
170	q->state = UACCE_Q_INIT;
171	mutex_init(&q->mutex);
172	list_add(&q->list, &uacce->queues);
173	mutex_unlock(&uacce->mutex);
174
175	return 0;
176
177out_with_bond:
178	uacce_unbind_queue(q);
179out_with_mem:
180	kfree(q);
181	mutex_unlock(&uacce->mutex);
182	return ret;
183}
184
185static int uacce_fops_release(struct inode *inode, struct file *filep)
186{
187	struct uacce_queue *q = filep->private_data;
188	struct uacce_device *uacce = q->uacce;
189
190	mutex_lock(&uacce->mutex);
191	uacce_put_queue(q);
192	uacce_unbind_queue(q);
193	list_del(&q->list);
194	mutex_unlock(&uacce->mutex);
195	kfree(q);
196
197	return 0;
198}
199
200static void uacce_vma_close(struct vm_area_struct *vma)
201{
202	struct uacce_queue *q = vma->vm_private_data;
203	struct uacce_qfile_region *qfr = NULL;
204
205	if (vma->vm_pgoff < UACCE_MAX_REGION)
206		qfr = q->qfrs[vma->vm_pgoff];
207
208	kfree(qfr);
209}
210
211static const struct vm_operations_struct uacce_vm_ops = {
212	.close = uacce_vma_close,
213};
214
215static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma)
216{
217	struct uacce_queue *q = filep->private_data;
218	struct uacce_device *uacce = q->uacce;
219	struct uacce_qfile_region *qfr;
220	enum uacce_qfrt type = UACCE_MAX_REGION;
221	int ret = 0;
222
223	if (vma->vm_pgoff < UACCE_MAX_REGION)
224		type = vma->vm_pgoff;
225	else
226		return -EINVAL;
227
228	qfr = kzalloc(sizeof(*qfr), GFP_KERNEL);
229	if (!qfr)
230		return -ENOMEM;
231
232	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK;
233	vma->vm_ops = &uacce_vm_ops;
234	vma->vm_private_data = q;
235	qfr->type = type;
236
237	mutex_lock(&q->mutex);
238	if (!uacce_queue_is_valid(q)) {
239		ret = -ENXIO;
240		goto out_with_lock;
241	}
242
243	if (q->qfrs[type]) {
244		ret = -EEXIST;
245		goto out_with_lock;
246	}
247
248	switch (type) {
249	case UACCE_QFRT_MMIO:
250		if (!uacce->ops->mmap) {
251			ret = -EINVAL;
252			goto out_with_lock;
253		}
254
255		ret = uacce->ops->mmap(q, vma, qfr);
256		if (ret)
257			goto out_with_lock;
258
259		break;
260
261	case UACCE_QFRT_DUS:
262		if (!uacce->ops->mmap) {
263			ret = -EINVAL;
264			goto out_with_lock;
265		}
266
267		ret = uacce->ops->mmap(q, vma, qfr);
268		if (ret)
269			goto out_with_lock;
270		break;
271
272	default:
273		ret = -EINVAL;
274		goto out_with_lock;
275	}
276
277	q->qfrs[type] = qfr;
278	mutex_unlock(&q->mutex);
279
280	return ret;
281
282out_with_lock:
283	mutex_unlock(&q->mutex);
284	kfree(qfr);
285	return ret;
286}
287
288static __poll_t uacce_fops_poll(struct file *file, poll_table *wait)
289{
290	struct uacce_queue *q = file->private_data;
291	struct uacce_device *uacce = q->uacce;
292	__poll_t ret = 0;
293
294	mutex_lock(&q->mutex);
295	if (!uacce_queue_is_valid(q))
296		goto out_unlock;
297
298	poll_wait(file, &q->wait, wait);
299
300	if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q))
301		ret = EPOLLIN | EPOLLRDNORM;
302
303out_unlock:
304	mutex_unlock(&q->mutex);
305	return ret;
306}
307
308static const struct file_operations uacce_fops = {
309	.owner		= THIS_MODULE,
310	.open		= uacce_fops_open,
311	.release	= uacce_fops_release,
312	.unlocked_ioctl	= uacce_fops_unl_ioctl,
313#ifdef CONFIG_COMPAT
314	.compat_ioctl	= uacce_fops_compat_ioctl,
315#endif
316	.mmap		= uacce_fops_mmap,
317	.poll		= uacce_fops_poll,
318};
319
320#define to_uacce_device(dev) container_of(dev, struct uacce_device, dev)
321
322static ssize_t api_show(struct device *dev,
323			struct device_attribute *attr, char *buf)
324{
325	struct uacce_device *uacce = to_uacce_device(dev);
326
327	return sprintf(buf, "%s\n", uacce->api_ver);
328}
329
330static ssize_t flags_show(struct device *dev,
331			  struct device_attribute *attr, char *buf)
332{
333	struct uacce_device *uacce = to_uacce_device(dev);
334
335	return sprintf(buf, "%u\n", uacce->flags);
336}
337
338static ssize_t available_instances_show(struct device *dev,
339					struct device_attribute *attr,
340					char *buf)
341{
342	struct uacce_device *uacce = to_uacce_device(dev);
343
344	if (!uacce->ops->get_available_instances)
345		return -ENODEV;
346
347	return sprintf(buf, "%d\n",
348		       uacce->ops->get_available_instances(uacce));
349}
350
351static ssize_t algorithms_show(struct device *dev,
352			       struct device_attribute *attr, char *buf)
353{
354	struct uacce_device *uacce = to_uacce_device(dev);
355
356	return sprintf(buf, "%s\n", uacce->algs);
357}
358
359static ssize_t region_mmio_size_show(struct device *dev,
360				     struct device_attribute *attr, char *buf)
361{
362	struct uacce_device *uacce = to_uacce_device(dev);
363
364	return sprintf(buf, "%lu\n",
365		       uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT);
366}
367
368static ssize_t region_dus_size_show(struct device *dev,
369				    struct device_attribute *attr, char *buf)
370{
371	struct uacce_device *uacce = to_uacce_device(dev);
372
373	return sprintf(buf, "%lu\n",
374		       uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT);
375}
376
377static DEVICE_ATTR_RO(api);
378static DEVICE_ATTR_RO(flags);
379static DEVICE_ATTR_RO(available_instances);
380static DEVICE_ATTR_RO(algorithms);
381static DEVICE_ATTR_RO(region_mmio_size);
382static DEVICE_ATTR_RO(region_dus_size);
383
384static struct attribute *uacce_dev_attrs[] = {
385	&dev_attr_api.attr,
386	&dev_attr_flags.attr,
387	&dev_attr_available_instances.attr,
388	&dev_attr_algorithms.attr,
389	&dev_attr_region_mmio_size.attr,
390	&dev_attr_region_dus_size.attr,
391	NULL,
392};
393
394static umode_t uacce_dev_is_visible(struct kobject *kobj,
395				    struct attribute *attr, int n)
396{
397	struct device *dev = kobj_to_dev(kobj);
398	struct uacce_device *uacce = to_uacce_device(dev);
399
400	if (((attr == &dev_attr_region_mmio_size.attr) &&
401	    (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) ||
402	    ((attr == &dev_attr_region_dus_size.attr) &&
403	    (!uacce->qf_pg_num[UACCE_QFRT_DUS])))
404		return 0;
405
406	return attr->mode;
407}
408
409static struct attribute_group uacce_dev_group = {
410	.is_visible	= uacce_dev_is_visible,
411	.attrs		= uacce_dev_attrs,
412};
413
414__ATTRIBUTE_GROUPS(uacce_dev);
415
416static void uacce_release(struct device *dev)
417{
418	struct uacce_device *uacce = to_uacce_device(dev);
419
420	kfree(uacce);
421}
422
423/**
424 * uacce_alloc() - alloc an accelerator
425 * @parent: pointer of uacce parent device
426 * @interface: pointer of uacce_interface for register
427 *
428 * Returns uacce pointer if success and ERR_PTR if not
429 * Need check returned negotiated uacce->flags
430 */
431struct uacce_device *uacce_alloc(struct device *parent,
432				 struct uacce_interface *interface)
433{
434	unsigned int flags = interface->flags;
435	struct uacce_device *uacce;
436	int ret;
437
438	uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL);
439	if (!uacce)
440		return ERR_PTR(-ENOMEM);
441
442	if (flags & UACCE_DEV_SVA) {
443		ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA);
444		if (ret)
445			flags &= ~UACCE_DEV_SVA;
446	}
447
448	uacce->parent = parent;
449	uacce->flags = flags;
450	uacce->ops = interface->ops;
451
452	ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b,
453		       GFP_KERNEL);
454	if (ret < 0)
455		goto err_with_uacce;
456
457	INIT_LIST_HEAD(&uacce->queues);
458	mutex_init(&uacce->mutex);
459	device_initialize(&uacce->dev);
460	uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id);
461	uacce->dev.class = uacce_class;
462	uacce->dev.groups = uacce_dev_groups;
463	uacce->dev.parent = uacce->parent;
464	uacce->dev.release = uacce_release;
465	dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id);
466
467	return uacce;
468
469err_with_uacce:
470	if (flags & UACCE_DEV_SVA)
471		iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA);
472	kfree(uacce);
473	return ERR_PTR(ret);
474}
475EXPORT_SYMBOL_GPL(uacce_alloc);
476
477/**
478 * uacce_register() - add the accelerator to cdev and export to user space
479 * @uacce: The initialized uacce device
480 *
481 * Return 0 if register succeeded, or an error.
482 */
483int uacce_register(struct uacce_device *uacce)
484{
485	if (!uacce)
486		return -ENODEV;
487
488	uacce->cdev = cdev_alloc();
489	if (!uacce->cdev)
490		return -ENOMEM;
491
492	uacce->cdev->ops = &uacce_fops;
493	uacce->cdev->owner = THIS_MODULE;
494
495	return cdev_device_add(uacce->cdev, &uacce->dev);
496}
497EXPORT_SYMBOL_GPL(uacce_register);
498
499/**
500 * uacce_remove() - remove the accelerator
501 * @uacce: the accelerator to remove
502 */
503void uacce_remove(struct uacce_device *uacce)
504{
505	struct uacce_queue *q, *next_q;
506
507	if (!uacce)
508		return;
509	/*
510	 * unmap remaining mapping from user space, preventing user still
511	 * access the mmaped area while parent device is already removed
512	 */
513	if (uacce->inode)
514		unmap_mapping_range(uacce->inode->i_mapping, 0, 0, 1);
515
516	/*
517	 * uacce_fops_open() may be running concurrently, even after we remove
518	 * the cdev. Holding uacce->mutex ensures that open() does not obtain a
519	 * removed uacce device.
520	 */
521	mutex_lock(&uacce->mutex);
522	/* ensure no open queue remains */
523	list_for_each_entry_safe(q, next_q, &uacce->queues, list) {
524		/*
525		 * Taking q->mutex ensures that fops do not use the defunct
526		 * uacce->ops after the queue is disabled.
527		 */
528		mutex_lock(&q->mutex);
529		uacce_put_queue(q);
530		mutex_unlock(&q->mutex);
531		uacce_unbind_queue(q);
532	}
533
534	/* disable sva now since no opened queues */
535	if (uacce->flags & UACCE_DEV_SVA)
536		iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA);
537
538	if (uacce->cdev)
539		cdev_device_del(uacce->cdev, &uacce->dev);
540	xa_erase(&uacce_xa, uacce->dev_id);
541	/*
542	 * uacce exists as long as there are open fds, but ops will be freed
543	 * now. Ensure that bugs cause NULL deref rather than use-after-free.
544	 */
545	uacce->ops = NULL;
546	uacce->parent = NULL;
547	mutex_unlock(&uacce->mutex);
548	put_device(&uacce->dev);
549}
550EXPORT_SYMBOL_GPL(uacce_remove);
551
552static int __init uacce_init(void)
553{
554	int ret;
555
556	uacce_class = class_create(THIS_MODULE, UACCE_NAME);
557	if (IS_ERR(uacce_class))
558		return PTR_ERR(uacce_class);
559
560	ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME);
561	if (ret)
562		class_destroy(uacce_class);
563
564	return ret;
565}
566
567static __exit void uacce_exit(void)
568{
569	unregister_chrdev_region(uacce_devt, MINORMASK);
570	class_destroy(uacce_class);
571}
572
573subsys_initcall(uacce_init);
574module_exit(uacce_exit);
575
576MODULE_LICENSE("GPL");
577MODULE_AUTHOR("Hisilicon Tech. Co., Ltd.");
578MODULE_DESCRIPTION("Accelerator interface for Userland applications");
579