1// SPDX-License-Identifier: GPL-2.0-or-later
2#include <linux/compat.h>
3#include <linux/dma-mapping.h>
4#include <linux/iommu.h>
5#include <linux/module.h>
6#include <linux/poll.h>
7#include <linux/slab.h>
8#include <linux/uacce.h>
9
10static struct class *uacce_class;
11static dev_t uacce_devt;
12static DEFINE_XARRAY_ALLOC(uacce_xa);
13
14/*
15 * If the parent driver or the device disappears, the queue state is invalid and
16 * ops are not usable anymore.
17 */
18static bool uacce_queue_is_valid(struct uacce_queue *q)
19{
20	return q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED;
21}
22
23static int uacce_start_queue(struct uacce_queue *q)
24{
25	int ret;
26
27	if (q->state != UACCE_Q_INIT)
28		return -EINVAL;
29
30	if (q->uacce->ops->start_queue) {
31		ret = q->uacce->ops->start_queue(q);
32		if (ret < 0)
33			return ret;
34	}
35
36	q->state = UACCE_Q_STARTED;
37	return 0;
38}
39
40static int uacce_put_queue(struct uacce_queue *q)
41{
42	struct uacce_device *uacce = q->uacce;
43
44	if ((q->state == UACCE_Q_STARTED) && uacce->ops->stop_queue)
45		uacce->ops->stop_queue(q);
46
47	if ((q->state == UACCE_Q_INIT || q->state == UACCE_Q_STARTED) &&
48	     uacce->ops->put_queue)
49		uacce->ops->put_queue(q);
50
51	q->state = UACCE_Q_ZOMBIE;
52
53	return 0;
54}
55
56static long uacce_fops_unl_ioctl(struct file *filep,
57				 unsigned int cmd, unsigned long arg)
58{
59	struct uacce_queue *q = filep->private_data;
60	struct uacce_device *uacce = q->uacce;
61	long ret = -ENXIO;
62
63	/*
64	 * uacce->ops->ioctl() may take the mmap_lock when copying arg to/from
65	 * user. Avoid a circular lock dependency with uacce_fops_mmap(), which
66	 * gets called with mmap_lock held, by taking uacce->mutex instead of
67	 * q->mutex. Doing this in uacce_fops_mmap() is not possible because
68	 * uacce_fops_open() calls iommu_sva_bind_device(), which takes
69	 * mmap_lock, while holding uacce->mutex.
70	 */
71	mutex_lock(&uacce->mutex);
72	if (!uacce_queue_is_valid(q))
73		goto out_unlock;
74
75	switch (cmd) {
76	case UACCE_CMD_START_Q:
77		ret = uacce_start_queue(q);
78		break;
79	case UACCE_CMD_PUT_Q:
80		ret = uacce_put_queue(q);
81		break;
82	default:
83		if (uacce->ops->ioctl)
84			ret = uacce->ops->ioctl(q, cmd, arg);
85		else
86			ret = -EINVAL;
87	}
88out_unlock:
89	mutex_unlock(&uacce->mutex);
90	return ret;
91}
92
93#ifdef CONFIG_COMPAT
94static long uacce_fops_compat_ioctl(struct file *filep,
95				   unsigned int cmd, unsigned long arg)
96{
97	arg = (unsigned long)compat_ptr(arg);
98
99	return uacce_fops_unl_ioctl(filep, cmd, arg);
100}
101#endif
102
103static int uacce_bind_queue(struct uacce_device *uacce, struct uacce_queue *q)
104{
105	u32 pasid;
106	struct iommu_sva *handle;
107
108	if (!(uacce->flags & UACCE_DEV_SVA))
109		return 0;
110
111	handle = iommu_sva_bind_device(uacce->parent, current->mm);
112	if (IS_ERR(handle))
113		return PTR_ERR(handle);
114
115	pasid = iommu_sva_get_pasid(handle);
116	if (pasid == IOMMU_PASID_INVALID) {
117		iommu_sva_unbind_device(handle);
118		return -ENODEV;
119	}
120
121	q->handle = handle;
122	q->pasid = pasid;
123	return 0;
124}
125
126static void uacce_unbind_queue(struct uacce_queue *q)
127{
128	if (!q->handle)
129		return;
130	iommu_sva_unbind_device(q->handle);
131	q->handle = NULL;
132}
133
134static int uacce_fops_open(struct inode *inode, struct file *filep)
135{
136	struct uacce_device *uacce;
137	struct uacce_queue *q;
138	int ret;
139
140	uacce = xa_load(&uacce_xa, iminor(inode));
141	if (!uacce)
142		return -ENODEV;
143
144	q = kzalloc(sizeof(struct uacce_queue), GFP_KERNEL);
145	if (!q)
146		return -ENOMEM;
147
148	mutex_lock(&uacce->mutex);
149
150	if (!uacce->parent) {
151		ret = -EINVAL;
152		goto out_with_mem;
153	}
154
155	ret = uacce_bind_queue(uacce, q);
156	if (ret)
157		goto out_with_mem;
158
159	q->uacce = uacce;
160
161	if (uacce->ops->get_queue) {
162		ret = uacce->ops->get_queue(uacce, q->pasid, q);
163		if (ret < 0)
164			goto out_with_bond;
165	}
166
167	init_waitqueue_head(&q->wait);
168	filep->private_data = q;
169	q->state = UACCE_Q_INIT;
170	q->mapping = filep->f_mapping;
171	mutex_init(&q->mutex);
172	list_add(&q->list, &uacce->queues);
173	mutex_unlock(&uacce->mutex);
174
175	return 0;
176
177out_with_bond:
178	uacce_unbind_queue(q);
179out_with_mem:
180	kfree(q);
181	mutex_unlock(&uacce->mutex);
182	return ret;
183}
184
185static int uacce_fops_release(struct inode *inode, struct file *filep)
186{
187	struct uacce_queue *q = filep->private_data;
188	struct uacce_device *uacce = q->uacce;
189
190	mutex_lock(&uacce->mutex);
191	uacce_put_queue(q);
192	uacce_unbind_queue(q);
193	list_del(&q->list);
194	mutex_unlock(&uacce->mutex);
195	kfree(q);
196
197	return 0;
198}
199
200static void uacce_vma_close(struct vm_area_struct *vma)
201{
202	struct uacce_queue *q = vma->vm_private_data;
203
204	if (vma->vm_pgoff < UACCE_MAX_REGION) {
205		struct uacce_qfile_region *qfr = q->qfrs[vma->vm_pgoff];
206
207		mutex_lock(&q->mutex);
208		q->qfrs[vma->vm_pgoff] = NULL;
209		mutex_unlock(&q->mutex);
210		kfree(qfr);
211	}
212}
213
214static const struct vm_operations_struct uacce_vm_ops = {
215	.close = uacce_vma_close,
216};
217
218static int uacce_fops_mmap(struct file *filep, struct vm_area_struct *vma)
219{
220	struct uacce_queue *q = filep->private_data;
221	struct uacce_device *uacce = q->uacce;
222	struct uacce_qfile_region *qfr;
223	enum uacce_qfrt type = UACCE_MAX_REGION;
224	int ret = 0;
225
226	if (vma->vm_pgoff < UACCE_MAX_REGION)
227		type = vma->vm_pgoff;
228	else
229		return -EINVAL;
230
231	qfr = kzalloc(sizeof(*qfr), GFP_KERNEL);
232	if (!qfr)
233		return -ENOMEM;
234
235	vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND | VM_WIPEONFORK);
236	vma->vm_ops = &uacce_vm_ops;
237	vma->vm_private_data = q;
238	qfr->type = type;
239
240	mutex_lock(&q->mutex);
241	if (!uacce_queue_is_valid(q)) {
242		ret = -ENXIO;
243		goto out_with_lock;
244	}
245
246	if (q->qfrs[type]) {
247		ret = -EEXIST;
248		goto out_with_lock;
249	}
250
251	switch (type) {
252	case UACCE_QFRT_MMIO:
253	case UACCE_QFRT_DUS:
254		if (!uacce->ops->mmap) {
255			ret = -EINVAL;
256			goto out_with_lock;
257		}
258
259		ret = uacce->ops->mmap(q, vma, qfr);
260		if (ret)
261			goto out_with_lock;
262		break;
263
264	default:
265		ret = -EINVAL;
266		goto out_with_lock;
267	}
268
269	q->qfrs[type] = qfr;
270	mutex_unlock(&q->mutex);
271
272	return ret;
273
274out_with_lock:
275	mutex_unlock(&q->mutex);
276	kfree(qfr);
277	return ret;
278}
279
280static __poll_t uacce_fops_poll(struct file *file, poll_table *wait)
281{
282	struct uacce_queue *q = file->private_data;
283	struct uacce_device *uacce = q->uacce;
284	__poll_t ret = 0;
285
286	mutex_lock(&q->mutex);
287	if (!uacce_queue_is_valid(q))
288		goto out_unlock;
289
290	poll_wait(file, &q->wait, wait);
291
292	if (uacce->ops->is_q_updated && uacce->ops->is_q_updated(q))
293		ret = EPOLLIN | EPOLLRDNORM;
294
295out_unlock:
296	mutex_unlock(&q->mutex);
297	return ret;
298}
299
300static const struct file_operations uacce_fops = {
301	.owner		= THIS_MODULE,
302	.open		= uacce_fops_open,
303	.release	= uacce_fops_release,
304	.unlocked_ioctl	= uacce_fops_unl_ioctl,
305#ifdef CONFIG_COMPAT
306	.compat_ioctl	= uacce_fops_compat_ioctl,
307#endif
308	.mmap		= uacce_fops_mmap,
309	.poll		= uacce_fops_poll,
310};
311
312#define to_uacce_device(dev) container_of(dev, struct uacce_device, dev)
313
314static ssize_t api_show(struct device *dev,
315			struct device_attribute *attr, char *buf)
316{
317	struct uacce_device *uacce = to_uacce_device(dev);
318
319	return sysfs_emit(buf, "%s\n", uacce->api_ver);
320}
321
322static ssize_t flags_show(struct device *dev,
323			  struct device_attribute *attr, char *buf)
324{
325	struct uacce_device *uacce = to_uacce_device(dev);
326
327	return sysfs_emit(buf, "%u\n", uacce->flags);
328}
329
330static ssize_t available_instances_show(struct device *dev,
331					struct device_attribute *attr,
332					char *buf)
333{
334	struct uacce_device *uacce = to_uacce_device(dev);
335
336	if (!uacce->ops->get_available_instances)
337		return -ENODEV;
338
339	return sysfs_emit(buf, "%d\n",
340		       uacce->ops->get_available_instances(uacce));
341}
342
343static ssize_t algorithms_show(struct device *dev,
344			       struct device_attribute *attr, char *buf)
345{
346	struct uacce_device *uacce = to_uacce_device(dev);
347
348	return sysfs_emit(buf, "%s\n", uacce->algs);
349}
350
351static ssize_t region_mmio_size_show(struct device *dev,
352				     struct device_attribute *attr, char *buf)
353{
354	struct uacce_device *uacce = to_uacce_device(dev);
355
356	return sysfs_emit(buf, "%lu\n",
357		       uacce->qf_pg_num[UACCE_QFRT_MMIO] << PAGE_SHIFT);
358}
359
360static ssize_t region_dus_size_show(struct device *dev,
361				    struct device_attribute *attr, char *buf)
362{
363	struct uacce_device *uacce = to_uacce_device(dev);
364
365	return sysfs_emit(buf, "%lu\n",
366		       uacce->qf_pg_num[UACCE_QFRT_DUS] << PAGE_SHIFT);
367}
368
369static ssize_t isolate_show(struct device *dev,
370			    struct device_attribute *attr, char *buf)
371{
372	struct uacce_device *uacce = to_uacce_device(dev);
373
374	return sysfs_emit(buf, "%d\n", uacce->ops->get_isolate_state(uacce));
375}
376
377static ssize_t isolate_strategy_show(struct device *dev, struct device_attribute *attr, char *buf)
378{
379	struct uacce_device *uacce = to_uacce_device(dev);
380	u32 val;
381
382	val = uacce->ops->isolate_err_threshold_read(uacce);
383
384	return sysfs_emit(buf, "%u\n", val);
385}
386
387static ssize_t isolate_strategy_store(struct device *dev, struct device_attribute *attr,
388				   const char *buf, size_t count)
389{
390	struct uacce_device *uacce = to_uacce_device(dev);
391	unsigned long val;
392	int ret;
393
394	if (kstrtoul(buf, 0, &val) < 0)
395		return -EINVAL;
396
397	if (val > UACCE_MAX_ERR_THRESHOLD)
398		return -EINVAL;
399
400	ret = uacce->ops->isolate_err_threshold_write(uacce, val);
401	if (ret)
402		return ret;
403
404	return count;
405}
406
407static DEVICE_ATTR_RO(api);
408static DEVICE_ATTR_RO(flags);
409static DEVICE_ATTR_RO(available_instances);
410static DEVICE_ATTR_RO(algorithms);
411static DEVICE_ATTR_RO(region_mmio_size);
412static DEVICE_ATTR_RO(region_dus_size);
413static DEVICE_ATTR_RO(isolate);
414static DEVICE_ATTR_RW(isolate_strategy);
415
416static struct attribute *uacce_dev_attrs[] = {
417	&dev_attr_api.attr,
418	&dev_attr_flags.attr,
419	&dev_attr_available_instances.attr,
420	&dev_attr_algorithms.attr,
421	&dev_attr_region_mmio_size.attr,
422	&dev_attr_region_dus_size.attr,
423	&dev_attr_isolate.attr,
424	&dev_attr_isolate_strategy.attr,
425	NULL,
426};
427
428static umode_t uacce_dev_is_visible(struct kobject *kobj,
429				    struct attribute *attr, int n)
430{
431	struct device *dev = kobj_to_dev(kobj);
432	struct uacce_device *uacce = to_uacce_device(dev);
433
434	if (((attr == &dev_attr_region_mmio_size.attr) &&
435	    (!uacce->qf_pg_num[UACCE_QFRT_MMIO])) ||
436	    ((attr == &dev_attr_region_dus_size.attr) &&
437	    (!uacce->qf_pg_num[UACCE_QFRT_DUS])))
438		return 0;
439
440	if (attr == &dev_attr_isolate_strategy.attr &&
441	    (!uacce->ops->isolate_err_threshold_read &&
442	     !uacce->ops->isolate_err_threshold_write))
443		return 0;
444
445	if (attr == &dev_attr_isolate.attr && !uacce->ops->get_isolate_state)
446		return 0;
447
448	return attr->mode;
449}
450
451static struct attribute_group uacce_dev_group = {
452	.is_visible	= uacce_dev_is_visible,
453	.attrs		= uacce_dev_attrs,
454};
455
456__ATTRIBUTE_GROUPS(uacce_dev);
457
458static void uacce_release(struct device *dev)
459{
460	struct uacce_device *uacce = to_uacce_device(dev);
461
462	kfree(uacce);
463}
464
465static unsigned int uacce_enable_sva(struct device *parent, unsigned int flags)
466{
467	int ret;
468
469	if (!(flags & UACCE_DEV_SVA))
470		return flags;
471
472	flags &= ~UACCE_DEV_SVA;
473
474	ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_IOPF);
475	if (ret) {
476		dev_err(parent, "failed to enable IOPF feature! ret = %pe\n", ERR_PTR(ret));
477		return flags;
478	}
479
480	ret = iommu_dev_enable_feature(parent, IOMMU_DEV_FEAT_SVA);
481	if (ret) {
482		dev_err(parent, "failed to enable SVA feature! ret = %pe\n", ERR_PTR(ret));
483		iommu_dev_disable_feature(parent, IOMMU_DEV_FEAT_IOPF);
484		return flags;
485	}
486
487	return flags | UACCE_DEV_SVA;
488}
489
490static void uacce_disable_sva(struct uacce_device *uacce)
491{
492	if (!(uacce->flags & UACCE_DEV_SVA))
493		return;
494
495	iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_SVA);
496	iommu_dev_disable_feature(uacce->parent, IOMMU_DEV_FEAT_IOPF);
497}
498
499/**
500 * uacce_alloc() - alloc an accelerator
501 * @parent: pointer of uacce parent device
502 * @interface: pointer of uacce_interface for register
503 *
504 * Returns uacce pointer if success and ERR_PTR if not
505 * Need check returned negotiated uacce->flags
506 */
507struct uacce_device *uacce_alloc(struct device *parent,
508				 struct uacce_interface *interface)
509{
510	unsigned int flags = interface->flags;
511	struct uacce_device *uacce;
512	int ret;
513
514	uacce = kzalloc(sizeof(struct uacce_device), GFP_KERNEL);
515	if (!uacce)
516		return ERR_PTR(-ENOMEM);
517
518	flags = uacce_enable_sva(parent, flags);
519
520	uacce->parent = parent;
521	uacce->flags = flags;
522	uacce->ops = interface->ops;
523
524	ret = xa_alloc(&uacce_xa, &uacce->dev_id, uacce, xa_limit_32b,
525		       GFP_KERNEL);
526	if (ret < 0)
527		goto err_with_uacce;
528
529	INIT_LIST_HEAD(&uacce->queues);
530	mutex_init(&uacce->mutex);
531	device_initialize(&uacce->dev);
532	uacce->dev.devt = MKDEV(MAJOR(uacce_devt), uacce->dev_id);
533	uacce->dev.class = uacce_class;
534	uacce->dev.groups = uacce_dev_groups;
535	uacce->dev.parent = uacce->parent;
536	uacce->dev.release = uacce_release;
537	dev_set_name(&uacce->dev, "%s-%d", interface->name, uacce->dev_id);
538
539	return uacce;
540
541err_with_uacce:
542	uacce_disable_sva(uacce);
543	kfree(uacce);
544	return ERR_PTR(ret);
545}
546EXPORT_SYMBOL_GPL(uacce_alloc);
547
548/**
549 * uacce_register() - add the accelerator to cdev and export to user space
550 * @uacce: The initialized uacce device
551 *
552 * Return 0 if register succeeded, or an error.
553 */
554int uacce_register(struct uacce_device *uacce)
555{
556	if (!uacce)
557		return -ENODEV;
558
559	uacce->cdev = cdev_alloc();
560	if (!uacce->cdev)
561		return -ENOMEM;
562
563	uacce->cdev->ops = &uacce_fops;
564	uacce->cdev->owner = THIS_MODULE;
565
566	return cdev_device_add(uacce->cdev, &uacce->dev);
567}
568EXPORT_SYMBOL_GPL(uacce_register);
569
570/**
571 * uacce_remove() - remove the accelerator
572 * @uacce: the accelerator to remove
573 */
574void uacce_remove(struct uacce_device *uacce)
575{
576	struct uacce_queue *q, *next_q;
577
578	if (!uacce)
579		return;
580
581	/*
582	 * uacce_fops_open() may be running concurrently, even after we remove
583	 * the cdev. Holding uacce->mutex ensures that open() does not obtain a
584	 * removed uacce device.
585	 */
586	mutex_lock(&uacce->mutex);
587	/* ensure no open queue remains */
588	list_for_each_entry_safe(q, next_q, &uacce->queues, list) {
589		/*
590		 * Taking q->mutex ensures that fops do not use the defunct
591		 * uacce->ops after the queue is disabled.
592		 */
593		mutex_lock(&q->mutex);
594		uacce_put_queue(q);
595		mutex_unlock(&q->mutex);
596		uacce_unbind_queue(q);
597
598		/*
599		 * unmap remaining mapping from user space, preventing user still
600		 * access the mmaped area while parent device is already removed
601		 */
602		unmap_mapping_range(q->mapping, 0, 0, 1);
603	}
604
605	/* disable sva now since no opened queues */
606	uacce_disable_sva(uacce);
607
608	if (uacce->cdev)
609		cdev_device_del(uacce->cdev, &uacce->dev);
610	xa_erase(&uacce_xa, uacce->dev_id);
611	/*
612	 * uacce exists as long as there are open fds, but ops will be freed
613	 * now. Ensure that bugs cause NULL deref rather than use-after-free.
614	 */
615	uacce->ops = NULL;
616	uacce->parent = NULL;
617	mutex_unlock(&uacce->mutex);
618	put_device(&uacce->dev);
619}
620EXPORT_SYMBOL_GPL(uacce_remove);
621
622static int __init uacce_init(void)
623{
624	int ret;
625
626	uacce_class = class_create(UACCE_NAME);
627	if (IS_ERR(uacce_class))
628		return PTR_ERR(uacce_class);
629
630	ret = alloc_chrdev_region(&uacce_devt, 0, MINORMASK, UACCE_NAME);
631	if (ret)
632		class_destroy(uacce_class);
633
634	return ret;
635}
636
637static __exit void uacce_exit(void)
638{
639	unregister_chrdev_region(uacce_devt, MINORMASK);
640	class_destroy(uacce_class);
641}
642
643subsys_initcall(uacce_init);
644module_exit(uacce_exit);
645
646MODULE_LICENSE("GPL");
647MODULE_AUTHOR("HiSilicon Tech. Co., Ltd.");
648MODULE_DESCRIPTION("Accelerator interface for Userland applications");
649