xref: /kernel/linux/linux-5.10/drivers/xen/evtchn.c (revision 8c2ecf20) 
1/******************************************************************************
2 * evtchn.c
3 *
4 * Driver for receiving and demuxing event-channel signals.
5 *
6 * Copyright (c) 2004-2005, K A Fraser
7 * Multi-process extensions Copyright (c) 2004, Steven Smith
8 *
9 * This program is free software; you can redistribute it and/or
10 * modify it under the terms of the GNU General Public License version 2
11 * as published by the Free Software Foundation; or, when distributed
12 * separately from the Linux kernel or incorporated into other
13 * software packages, subject to the following license:
14 *
15 * Permission is hereby granted, free of charge, to any person obtaining a copy
16 * of this source file (the "Software"), to deal in the Software without
17 * restriction, including without limitation the rights to use, copy, modify,
18 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
19 * and to permit persons to whom the Software is furnished to do so, subject to
20 * the following conditions:
21 *
22 * The above copyright notice and this permission notice shall be included in
23 * all copies or substantial portions of the Software.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
26 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
27 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
28 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
29 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
30 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
31 * IN THE SOFTWARE.
32 */
33
34#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
35
36#include <linux/module.h>
37#include <linux/kernel.h>
38#include <linux/sched.h>
39#include <linux/slab.h>
40#include <linux/string.h>
41#include <linux/errno.h>
42#include <linux/fs.h>
43#include <linux/miscdevice.h>
44#include <linux/major.h>
45#include <linux/proc_fs.h>
46#include <linux/stat.h>
47#include <linux/poll.h>
48#include <linux/irq.h>
49#include <linux/init.h>
50#include <linux/mutex.h>
51#include <linux/cpu.h>
52#include <linux/mm.h>
53#include <linux/vmalloc.h>
54
55#include <xen/xen.h>
56#include <xen/events.h>
57#include <xen/evtchn.h>
58#include <xen/xen-ops.h>
59#include <asm/xen/hypervisor.h>
60
61struct per_user_data {
62	struct mutex bind_mutex; /* serialize bind/unbind operations */
63	struct rb_root evtchns;
64	unsigned int nr_evtchns;
65
66	/* Notification ring, accessed via /dev/xen/evtchn. */
67	unsigned int ring_size;
68	evtchn_port_t *ring;
69	unsigned int ring_cons, ring_prod, ring_overflow;
70	struct mutex ring_cons_mutex; /* protect against concurrent readers */
71	spinlock_t ring_prod_lock; /* product against concurrent interrupts */
72
73	/* Processes wait on this queue when ring is empty. */
74	wait_queue_head_t evtchn_wait;
75	struct fasync_struct *evtchn_async_queue;
76	const char *name;
77
78	domid_t restrict_domid;
79};
80
81#define UNRESTRICTED_DOMID ((domid_t)-1)
82
83struct user_evtchn {
84	struct rb_node node;
85	struct per_user_data *user;
86	evtchn_port_t port;
87	bool enabled;
88};
89
90static void evtchn_free_ring(evtchn_port_t *ring)
91{
92	kvfree(ring);
93}
94
95static unsigned int evtchn_ring_offset(struct per_user_data *u,
96				       unsigned int idx)
97{
98	return idx & (u->ring_size - 1);
99}
100
101static evtchn_port_t *evtchn_ring_entry(struct per_user_data *u,
102					unsigned int idx)
103{
104	return u->ring + evtchn_ring_offset(u, idx);
105}
106
107static int add_evtchn(struct per_user_data *u, struct user_evtchn *evtchn)
108{
109	struct rb_node **new = &(u->evtchns.rb_node), *parent = NULL;
110
111	u->nr_evtchns++;
112
113	while (*new) {
114		struct user_evtchn *this;
115
116		this = rb_entry(*new, struct user_evtchn, node);
117
118		parent = *new;
119		if (this->port < evtchn->port)
120			new = &((*new)->rb_left);
121		else if (this->port > evtchn->port)
122			new = &((*new)->rb_right);
123		else
124			return -EEXIST;
125	}
126
127	/* Add new node and rebalance tree. */
128	rb_link_node(&evtchn->node, parent, new);
129	rb_insert_color(&evtchn->node, &u->evtchns);
130
131	return 0;
132}
133
134static void del_evtchn(struct per_user_data *u, struct user_evtchn *evtchn)
135{
136	u->nr_evtchns--;
137	rb_erase(&evtchn->node, &u->evtchns);
138	kfree(evtchn);
139}
140
141static struct user_evtchn *find_evtchn(struct per_user_data *u,
142				       evtchn_port_t port)
143{
144	struct rb_node *node = u->evtchns.rb_node;
145
146	while (node) {
147		struct user_evtchn *evtchn;
148
149		evtchn = rb_entry(node, struct user_evtchn, node);
150
151		if (evtchn->port < port)
152			node = node->rb_left;
153		else if (evtchn->port > port)
154			node = node->rb_right;
155		else
156			return evtchn;
157	}
158	return NULL;
159}
160
161static irqreturn_t evtchn_interrupt(int irq, void *data)
162{
163	struct user_evtchn *evtchn = data;
164	struct per_user_data *u = evtchn->user;
165
166	WARN(!evtchn->enabled,
167	     "Interrupt for port %u, but apparently not enabled; per-user %p\n",
168	     evtchn->port, u);
169
170	evtchn->enabled = false;
171
172	spin_lock(&u->ring_prod_lock);
173
174	if ((u->ring_prod - u->ring_cons) < u->ring_size) {
175		*evtchn_ring_entry(u, u->ring_prod) = evtchn->port;
176		wmb(); /* Ensure ring contents visible */
177		if (u->ring_cons == u->ring_prod++) {
178			wake_up_interruptible(&u->evtchn_wait);
179			kill_fasync(&u->evtchn_async_queue,
180				    SIGIO, POLL_IN);
181		}
182	} else
183		u->ring_overflow = 1;
184
185	spin_unlock(&u->ring_prod_lock);
186
187	return IRQ_HANDLED;
188}
189
190static ssize_t evtchn_read(struct file *file, char __user *buf,
191			   size_t count, loff_t *ppos)
192{
193	int rc;
194	unsigned int c, p, bytes1 = 0, bytes2 = 0;
195	struct per_user_data *u = file->private_data;
196
197	/* Whole number of ports. */
198	count &= ~(sizeof(evtchn_port_t)-1);
199
200	if (count == 0)
201		return 0;
202
203	if (count > PAGE_SIZE)
204		count = PAGE_SIZE;
205
206	for (;;) {
207		mutex_lock(&u->ring_cons_mutex);
208
209		rc = -EFBIG;
210		if (u->ring_overflow)
211			goto unlock_out;
212
213		c = u->ring_cons;
214		p = u->ring_prod;
215		if (c != p)
216			break;
217
218		mutex_unlock(&u->ring_cons_mutex);
219
220		if (file->f_flags & O_NONBLOCK)
221			return -EAGAIN;
222
223		rc = wait_event_interruptible(u->evtchn_wait,
224					      u->ring_cons != u->ring_prod);
225		if (rc)
226			return rc;
227	}
228
229	/* Byte lengths of two chunks. Chunk split (if any) is at ring wrap. */
230	if (((c ^ p) & u->ring_size) != 0) {
231		bytes1 = (u->ring_size - evtchn_ring_offset(u, c)) *
232			sizeof(evtchn_port_t);
233		bytes2 = evtchn_ring_offset(u, p) * sizeof(evtchn_port_t);
234	} else {
235		bytes1 = (p - c) * sizeof(evtchn_port_t);
236		bytes2 = 0;
237	}
238
239	/* Truncate chunks according to caller's maximum byte count. */
240	if (bytes1 > count) {
241		bytes1 = count;
242		bytes2 = 0;
243	} else if ((bytes1 + bytes2) > count) {
244		bytes2 = count - bytes1;
245	}
246
247	rc = -EFAULT;
248	rmb(); /* Ensure that we see the port before we copy it. */
249	if (copy_to_user(buf, evtchn_ring_entry(u, c), bytes1) ||
250	    ((bytes2 != 0) &&
251	     copy_to_user(&buf[bytes1], &u->ring[0], bytes2)))
252		goto unlock_out;
253
254	u->ring_cons += (bytes1 + bytes2) / sizeof(evtchn_port_t);
255	rc = bytes1 + bytes2;
256
257 unlock_out:
258	mutex_unlock(&u->ring_cons_mutex);
259	return rc;
260}
261
262static ssize_t evtchn_write(struct file *file, const char __user *buf,
263			    size_t count, loff_t *ppos)
264{
265	int rc, i;
266	evtchn_port_t *kbuf = (evtchn_port_t *)__get_free_page(GFP_KERNEL);
267	struct per_user_data *u = file->private_data;
268
269	if (kbuf == NULL)
270		return -ENOMEM;
271
272	/* Whole number of ports. */
273	count &= ~(sizeof(evtchn_port_t)-1);
274
275	rc = 0;
276	if (count == 0)
277		goto out;
278
279	if (count > PAGE_SIZE)
280		count = PAGE_SIZE;
281
282	rc = -EFAULT;
283	if (copy_from_user(kbuf, buf, count) != 0)
284		goto out;
285
286	mutex_lock(&u->bind_mutex);
287
288	for (i = 0; i < (count/sizeof(evtchn_port_t)); i++) {
289		evtchn_port_t port = kbuf[i];
290		struct user_evtchn *evtchn;
291
292		evtchn = find_evtchn(u, port);
293		if (evtchn && !evtchn->enabled) {
294			evtchn->enabled = true;
295			xen_irq_lateeoi(irq_from_evtchn(port), 0);
296		}
297	}
298
299	mutex_unlock(&u->bind_mutex);
300
301	rc = count;
302
303 out:
304	free_page((unsigned long)kbuf);
305	return rc;
306}
307
308static int evtchn_resize_ring(struct per_user_data *u)
309{
310	unsigned int new_size;
311	evtchn_port_t *new_ring, *old_ring;
312
313	/*
314	 * Ensure the ring is large enough to capture all possible
315	 * events. i.e., one free slot for each bound event.
316	 */
317	if (u->nr_evtchns <= u->ring_size)
318		return 0;
319
320	if (u->ring_size == 0)
321		new_size = 64;
322	else
323		new_size = 2 * u->ring_size;
324
325	new_ring = kvmalloc_array(new_size, sizeof(*new_ring), GFP_KERNEL);
326	if (!new_ring)
327		return -ENOMEM;
328
329	old_ring = u->ring;
330
331	/*
332	 * Access to the ring contents is serialized by either the
333	 * prod /or/ cons lock so take both when resizing.
334	 */
335	mutex_lock(&u->ring_cons_mutex);
336	spin_lock_irq(&u->ring_prod_lock);
337
338	/*
339	 * Copy the old ring contents to the new ring.
340	 *
341	 * To take care of wrapping, a full ring, and the new index
342	 * pointing into the second half, simply copy the old contents
343	 * twice.
344	 *
345	 * +---------+    +------------------+
346	 * |34567  12| -> |34567  1234567  12|
347	 * +-----p-c-+    +-------c------p---+
348	 */
349	memcpy(new_ring, old_ring, u->ring_size * sizeof(*u->ring));
350	memcpy(new_ring + u->ring_size, old_ring,
351	       u->ring_size * sizeof(*u->ring));
352
353	u->ring = new_ring;
354	u->ring_size = new_size;
355
356	spin_unlock_irq(&u->ring_prod_lock);
357	mutex_unlock(&u->ring_cons_mutex);
358
359	evtchn_free_ring(old_ring);
360
361	return 0;
362}
363
364static int evtchn_bind_to_user(struct per_user_data *u, evtchn_port_t port)
365{
366	struct user_evtchn *evtchn;
367	struct evtchn_close close;
368	int rc = 0;
369
370	/*
371	 * Ports are never reused, so every caller should pass in a
372	 * unique port.
373	 *
374	 * (Locking not necessary because we haven't registered the
375	 * interrupt handler yet, and our caller has already
376	 * serialized bind operations.)
377	 */
378
379	evtchn = kzalloc(sizeof(*evtchn), GFP_KERNEL);
380	if (!evtchn)
381		return -ENOMEM;
382
383	evtchn->user = u;
384	evtchn->port = port;
385	evtchn->enabled = true; /* start enabled */
386
387	rc = add_evtchn(u, evtchn);
388	if (rc < 0)
389		goto err;
390
391	rc = evtchn_resize_ring(u);
392	if (rc < 0)
393		goto err;
394
395	rc = bind_evtchn_to_irqhandler_lateeoi(port, evtchn_interrupt, 0,
396					       u->name, evtchn);
397	if (rc < 0)
398		goto err;
399
400	rc = evtchn_make_refcounted(port);
401	return rc;
402
403err:
404	/* bind failed, should close the port now */
405	close.port = port;
406	if (HYPERVISOR_event_channel_op(EVTCHNOP_close, &close) != 0)
407		BUG();
408	del_evtchn(u, evtchn);
409	return rc;
410}
411
412static void evtchn_unbind_from_user(struct per_user_data *u,
413				    struct user_evtchn *evtchn)
414{
415	int irq = irq_from_evtchn(evtchn->port);
416
417	BUG_ON(irq < 0);
418
419	unbind_from_irqhandler(irq, evtchn);
420
421	del_evtchn(u, evtchn);
422}
423
424static DEFINE_PER_CPU(int, bind_last_selected_cpu);
425
426static void evtchn_bind_interdom_next_vcpu(evtchn_port_t evtchn)
427{
428	unsigned int selected_cpu, irq;
429	struct irq_desc *desc;
430	unsigned long flags;
431
432	irq = irq_from_evtchn(evtchn);
433	desc = irq_to_desc(irq);
434
435	if (!desc)
436		return;
437
438	raw_spin_lock_irqsave(&desc->lock, flags);
439	selected_cpu = this_cpu_read(bind_last_selected_cpu);
440	selected_cpu = cpumask_next_and(selected_cpu,
441			desc->irq_common_data.affinity, cpu_online_mask);
442
443	if (unlikely(selected_cpu >= nr_cpu_ids))
444		selected_cpu = cpumask_first_and(desc->irq_common_data.affinity,
445				cpu_online_mask);
446
447	this_cpu_write(bind_last_selected_cpu, selected_cpu);
448
449	/* unmask expects irqs to be disabled */
450	xen_set_affinity_evtchn(desc, selected_cpu);
451	raw_spin_unlock_irqrestore(&desc->lock, flags);
452}
453
454static long evtchn_ioctl(struct file *file,
455			 unsigned int cmd, unsigned long arg)
456{
457	int rc;
458	struct per_user_data *u = file->private_data;
459	void __user *uarg = (void __user *) arg;
460
461	/* Prevent bind from racing with unbind */
462	mutex_lock(&u->bind_mutex);
463
464	switch (cmd) {
465	case IOCTL_EVTCHN_BIND_VIRQ: {
466		struct ioctl_evtchn_bind_virq bind;
467		struct evtchn_bind_virq bind_virq;
468
469		rc = -EACCES;
470		if (u->restrict_domid != UNRESTRICTED_DOMID)
471			break;
472
473		rc = -EFAULT;
474		if (copy_from_user(&bind, uarg, sizeof(bind)))
475			break;
476
477		bind_virq.virq = bind.virq;
478		bind_virq.vcpu = xen_vcpu_nr(0);
479		rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_virq,
480						 &bind_virq);
481		if (rc != 0)
482			break;
483
484		rc = evtchn_bind_to_user(u, bind_virq.port);
485		if (rc == 0)
486			rc = bind_virq.port;
487		break;
488	}
489
490	case IOCTL_EVTCHN_BIND_INTERDOMAIN: {
491		struct ioctl_evtchn_bind_interdomain bind;
492		struct evtchn_bind_interdomain bind_interdomain;
493
494		rc = -EFAULT;
495		if (copy_from_user(&bind, uarg, sizeof(bind)))
496			break;
497
498		rc = -EACCES;
499		if (u->restrict_domid != UNRESTRICTED_DOMID &&
500		    u->restrict_domid != bind.remote_domain)
501			break;
502
503		bind_interdomain.remote_dom  = bind.remote_domain;
504		bind_interdomain.remote_port = bind.remote_port;
505		rc = HYPERVISOR_event_channel_op(EVTCHNOP_bind_interdomain,
506						 &bind_interdomain);
507		if (rc != 0)
508			break;
509
510		rc = evtchn_bind_to_user(u, bind_interdomain.local_port);
511		if (rc == 0) {
512			rc = bind_interdomain.local_port;
513			evtchn_bind_interdom_next_vcpu(rc);
514		}
515		break;
516	}
517
518	case IOCTL_EVTCHN_BIND_UNBOUND_PORT: {
519		struct ioctl_evtchn_bind_unbound_port bind;
520		struct evtchn_alloc_unbound alloc_unbound;
521
522		rc = -EACCES;
523		if (u->restrict_domid != UNRESTRICTED_DOMID)
524			break;
525
526		rc = -EFAULT;
527		if (copy_from_user(&bind, uarg, sizeof(bind)))
528			break;
529
530		alloc_unbound.dom        = DOMID_SELF;
531		alloc_unbound.remote_dom = bind.remote_domain;
532		rc = HYPERVISOR_event_channel_op(EVTCHNOP_alloc_unbound,
533						 &alloc_unbound);
534		if (rc != 0)
535			break;
536
537		rc = evtchn_bind_to_user(u, alloc_unbound.port);
538		if (rc == 0)
539			rc = alloc_unbound.port;
540		break;
541	}
542
543	case IOCTL_EVTCHN_UNBIND: {
544		struct ioctl_evtchn_unbind unbind;
545		struct user_evtchn *evtchn;
546
547		rc = -EFAULT;
548		if (copy_from_user(&unbind, uarg, sizeof(unbind)))
549			break;
550
551		rc = -EINVAL;
552		if (unbind.port >= xen_evtchn_nr_channels())
553			break;
554
555		rc = -ENOTCONN;
556		evtchn = find_evtchn(u, unbind.port);
557		if (!evtchn)
558			break;
559
560		disable_irq(irq_from_evtchn(unbind.port));
561		evtchn_unbind_from_user(u, evtchn);
562		rc = 0;
563		break;
564	}
565
566	case IOCTL_EVTCHN_NOTIFY: {
567		struct ioctl_evtchn_notify notify;
568		struct user_evtchn *evtchn;
569
570		rc = -EFAULT;
571		if (copy_from_user(&notify, uarg, sizeof(notify)))
572			break;
573
574		rc = -ENOTCONN;
575		evtchn = find_evtchn(u, notify.port);
576		if (evtchn) {
577			notify_remote_via_evtchn(notify.port);
578			rc = 0;
579		}
580		break;
581	}
582
583	case IOCTL_EVTCHN_RESET: {
584		/* Initialise the ring to empty. Clear errors. */
585		mutex_lock(&u->ring_cons_mutex);
586		spin_lock_irq(&u->ring_prod_lock);
587		u->ring_cons = u->ring_prod = u->ring_overflow = 0;
588		spin_unlock_irq(&u->ring_prod_lock);
589		mutex_unlock(&u->ring_cons_mutex);
590		rc = 0;
591		break;
592	}
593
594	case IOCTL_EVTCHN_RESTRICT_DOMID: {
595		struct ioctl_evtchn_restrict_domid ierd;
596
597		rc = -EACCES;
598		if (u->restrict_domid != UNRESTRICTED_DOMID)
599			break;
600
601		rc = -EFAULT;
602		if (copy_from_user(&ierd, uarg, sizeof(ierd)))
603		    break;
604
605		rc = -EINVAL;
606		if (ierd.domid == 0 || ierd.domid >= DOMID_FIRST_RESERVED)
607			break;
608
609		u->restrict_domid = ierd.domid;
610		rc = 0;
611
612		break;
613	}
614
615	default:
616		rc = -ENOSYS;
617		break;
618	}
619	mutex_unlock(&u->bind_mutex);
620
621	return rc;
622}
623
624static __poll_t evtchn_poll(struct file *file, poll_table *wait)
625{
626	__poll_t mask = EPOLLOUT | EPOLLWRNORM;
627	struct per_user_data *u = file->private_data;
628
629	poll_wait(file, &u->evtchn_wait, wait);
630	if (u->ring_cons != u->ring_prod)
631		mask |= EPOLLIN | EPOLLRDNORM;
632	if (u->ring_overflow)
633		mask = EPOLLERR;
634	return mask;
635}
636
637static int evtchn_fasync(int fd, struct file *filp, int on)
638{
639	struct per_user_data *u = filp->private_data;
640	return fasync_helper(fd, filp, on, &u->evtchn_async_queue);
641}
642
643static int evtchn_open(struct inode *inode, struct file *filp)
644{
645	struct per_user_data *u;
646
647	u = kzalloc(sizeof(*u), GFP_KERNEL);
648	if (u == NULL)
649		return -ENOMEM;
650
651	u->name = kasprintf(GFP_KERNEL, "evtchn:%s", current->comm);
652	if (u->name == NULL) {
653		kfree(u);
654		return -ENOMEM;
655	}
656
657	init_waitqueue_head(&u->evtchn_wait);
658
659	mutex_init(&u->bind_mutex);
660	mutex_init(&u->ring_cons_mutex);
661	spin_lock_init(&u->ring_prod_lock);
662
663	u->restrict_domid = UNRESTRICTED_DOMID;
664
665	filp->private_data = u;
666
667	return stream_open(inode, filp);
668}
669
670static int evtchn_release(struct inode *inode, struct file *filp)
671{
672	struct per_user_data *u = filp->private_data;
673	struct rb_node *node;
674
675	while ((node = u->evtchns.rb_node)) {
676		struct user_evtchn *evtchn;
677
678		evtchn = rb_entry(node, struct user_evtchn, node);
679		disable_irq(irq_from_evtchn(evtchn->port));
680		evtchn_unbind_from_user(u, evtchn);
681	}
682
683	evtchn_free_ring(u->ring);
684	kfree(u->name);
685	kfree(u);
686
687	return 0;
688}
689
690static const struct file_operations evtchn_fops = {
691	.owner   = THIS_MODULE,
692	.read    = evtchn_read,
693	.write   = evtchn_write,
694	.unlocked_ioctl = evtchn_ioctl,
695	.poll    = evtchn_poll,
696	.fasync  = evtchn_fasync,
697	.open    = evtchn_open,
698	.release = evtchn_release,
699	.llseek	 = no_llseek,
700};
701
702static struct miscdevice evtchn_miscdev = {
703	.minor        = MISC_DYNAMIC_MINOR,
704	.name         = "xen/evtchn",
705	.fops         = &evtchn_fops,
706};
707static int __init evtchn_init(void)
708{
709	int err;
710
711	if (!xen_domain())
712		return -ENODEV;
713
714	/* Create '/dev/xen/evtchn'. */
715	err = misc_register(&evtchn_miscdev);
716	if (err != 0) {
717		pr_err("Could not register /dev/xen/evtchn\n");
718		return err;
719	}
720
721	pr_info("Event-channel device installed\n");
722
723	return 0;
724}
725
726static void __exit evtchn_cleanup(void)
727{
728	misc_deregister(&evtchn_miscdev);
729}
730
731module_init(evtchn_init);
732module_exit(evtchn_cleanup);
733
734MODULE_LICENSE("GPL");
735