1/****************************************************************************** 2 * gntalloc.c 3 * 4 * Device for creating grant references (in user-space) that may be shared 5 * with other domains. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 * 12 * You should have received a copy of the GNU General Public License 13 * along with this program; if not, write to the Free Software 14 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 15 */ 16 17/* 18 * This driver exists to allow userspace programs in Linux to allocate kernel 19 * memory that will later be shared with another domain. Without this device, 20 * Linux userspace programs cannot create grant references. 21 * 22 * How this stuff works: 23 * X -> granting a page to Y 24 * Y -> mapping the grant from X 25 * 26 * 1. X uses the gntalloc device to allocate a page of kernel memory, P. 27 * 2. X creates an entry in the grant table that says domid(Y) can access P. 28 * This is done without a hypercall unless the grant table needs expansion. 29 * 3. X gives the grant reference identifier, GREF, to Y. 30 * 4. Y maps the page, either directly into kernel memory for use in a backend 31 * driver, or via a the gntdev device to map into the address space of an 32 * application running in Y. This is the first point at which Xen does any 33 * tracking of the page. 34 * 5. A program in X mmap()s a segment of the gntalloc device that corresponds 35 * to the shared page, and can now communicate with Y over the shared page. 36 * 37 * 38 * NOTE TO USERSPACE LIBRARIES: 39 * The grant allocation and mmap()ing are, naturally, two separate operations. 40 * You set up the sharing by calling the create ioctl() and then the mmap(). 41 * Teardown requires munmap() and either close() or ioctl(). 42 * 43 * WARNING: Since Xen does not allow a guest to forcibly end the use of a grant 44 * reference, this device can be used to consume kernel memory by leaving grant 45 * references mapped by another domain when an application exits. Therefore, 46 * there is a global limit on the number of pages that can be allocated. When 47 * all references to the page are unmapped, it will be freed during the next 48 * grant operation. 49 */ 50 51#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt 52 53#include <linux/atomic.h> 54#include <linux/module.h> 55#include <linux/miscdevice.h> 56#include <linux/kernel.h> 57#include <linux/init.h> 58#include <linux/slab.h> 59#include <linux/fs.h> 60#include <linux/device.h> 61#include <linux/mm.h> 62#include <linux/uaccess.h> 63#include <linux/types.h> 64#include <linux/list.h> 65#include <linux/highmem.h> 66 67#include <xen/xen.h> 68#include <xen/page.h> 69#include <xen/grant_table.h> 70#include <xen/gntalloc.h> 71#include <xen/events.h> 72 73static int limit = 1024; 74module_param(limit, int, 0644); 75MODULE_PARM_DESC(limit, "Maximum number of grants that may be allocated by " 76 "the gntalloc device"); 77 78static LIST_HEAD(gref_list); 79static DEFINE_MUTEX(gref_mutex); 80static int gref_size; 81 82struct notify_info { 83 uint16_t pgoff:12; /* Bits 0-11: Offset of the byte to clear */ 84 uint16_t flags:2; /* Bits 12-13: Unmap notification flags */ 85 int event; /* Port (event channel) to notify */ 86}; 87 88/* Metadata on a grant reference. */ 89struct gntalloc_gref { 90 struct list_head next_gref; /* list entry gref_list */ 91 struct list_head next_file; /* list entry file->list, if open */ 92 struct page *page; /* The shared page */ 93 uint64_t file_index; /* File offset for mmap() */ 94 unsigned int users; /* Use count - when zero, waiting on Xen */ 95 grant_ref_t gref_id; /* The grant reference number */ 96 struct notify_info notify; /* Unmap notification */ 97}; 98 99struct gntalloc_file_private_data { 100 struct list_head list; 101 uint64_t index; 102}; 103 104struct gntalloc_vma_private_data { 105 struct gntalloc_gref *gref; 106 int users; 107 int count; 108}; 109 110static void __del_gref(struct gntalloc_gref *gref); 111 112static void do_cleanup(void) 113{ 114 struct gntalloc_gref *gref, *n; 115 list_for_each_entry_safe(gref, n, &gref_list, next_gref) { 116 if (!gref->users) 117 __del_gref(gref); 118 } 119} 120 121static int add_grefs(struct ioctl_gntalloc_alloc_gref *op, 122 uint32_t *gref_ids, struct gntalloc_file_private_data *priv) 123{ 124 int i, rc, readonly; 125 LIST_HEAD(queue_gref); 126 LIST_HEAD(queue_file); 127 struct gntalloc_gref *gref, *next; 128 129 readonly = !(op->flags & GNTALLOC_FLAG_WRITABLE); 130 for (i = 0; i < op->count; i++) { 131 gref = kzalloc(sizeof(*gref), GFP_KERNEL); 132 if (!gref) { 133 rc = -ENOMEM; 134 goto undo; 135 } 136 list_add_tail(&gref->next_gref, &queue_gref); 137 list_add_tail(&gref->next_file, &queue_file); 138 gref->users = 1; 139 gref->file_index = op->index + i * PAGE_SIZE; 140 gref->page = alloc_page(GFP_KERNEL|__GFP_ZERO); 141 if (!gref->page) { 142 rc = -ENOMEM; 143 goto undo; 144 } 145 146 /* Grant foreign access to the page. */ 147 rc = gnttab_grant_foreign_access(op->domid, 148 xen_page_to_gfn(gref->page), 149 readonly); 150 if (rc < 0) 151 goto undo; 152 gref_ids[i] = gref->gref_id = rc; 153 } 154 155 /* Add to gref lists. */ 156 mutex_lock(&gref_mutex); 157 list_splice_tail(&queue_gref, &gref_list); 158 list_splice_tail(&queue_file, &priv->list); 159 mutex_unlock(&gref_mutex); 160 161 return 0; 162 163undo: 164 mutex_lock(&gref_mutex); 165 gref_size -= (op->count - i); 166 167 list_for_each_entry_safe(gref, next, &queue_file, next_file) { 168 list_del(&gref->next_file); 169 __del_gref(gref); 170 } 171 172 mutex_unlock(&gref_mutex); 173 return rc; 174} 175 176static void __del_gref(struct gntalloc_gref *gref) 177{ 178 unsigned long addr; 179 180 if (gref->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { 181 uint8_t *tmp = kmap(gref->page); 182 tmp[gref->notify.pgoff] = 0; 183 kunmap(gref->page); 184 } 185 if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) { 186 notify_remote_via_evtchn(gref->notify.event); 187 evtchn_put(gref->notify.event); 188 } 189 190 gref->notify.flags = 0; 191 192 if (gref->gref_id) { 193 if (gref->page) { 194 addr = (unsigned long)page_to_virt(gref->page); 195 gnttab_end_foreign_access(gref->gref_id, 0, addr); 196 } else 197 gnttab_free_grant_reference(gref->gref_id); 198 } 199 200 gref_size--; 201 list_del(&gref->next_gref); 202 203 kfree(gref); 204} 205 206/* finds contiguous grant references in a file, returns the first */ 207static struct gntalloc_gref *find_grefs(struct gntalloc_file_private_data *priv, 208 uint64_t index, uint32_t count) 209{ 210 struct gntalloc_gref *rv = NULL, *gref; 211 list_for_each_entry(gref, &priv->list, next_file) { 212 if (gref->file_index == index && !rv) 213 rv = gref; 214 if (rv) { 215 if (gref->file_index != index) 216 return NULL; 217 index += PAGE_SIZE; 218 count--; 219 if (count == 0) 220 return rv; 221 } 222 } 223 return NULL; 224} 225 226/* 227 * ------------------------------------- 228 * File operations. 229 * ------------------------------------- 230 */ 231static int gntalloc_open(struct inode *inode, struct file *filp) 232{ 233 struct gntalloc_file_private_data *priv; 234 235 priv = kzalloc(sizeof(*priv), GFP_KERNEL); 236 if (!priv) 237 goto out_nomem; 238 INIT_LIST_HEAD(&priv->list); 239 240 filp->private_data = priv; 241 242 pr_debug("%s: priv %p\n", __func__, priv); 243 244 return 0; 245 246out_nomem: 247 return -ENOMEM; 248} 249 250static int gntalloc_release(struct inode *inode, struct file *filp) 251{ 252 struct gntalloc_file_private_data *priv = filp->private_data; 253 struct gntalloc_gref *gref; 254 255 pr_debug("%s: priv %p\n", __func__, priv); 256 257 mutex_lock(&gref_mutex); 258 while (!list_empty(&priv->list)) { 259 gref = list_entry(priv->list.next, 260 struct gntalloc_gref, next_file); 261 list_del(&gref->next_file); 262 gref->users--; 263 if (gref->users == 0) 264 __del_gref(gref); 265 } 266 kfree(priv); 267 mutex_unlock(&gref_mutex); 268 269 return 0; 270} 271 272static long gntalloc_ioctl_alloc(struct gntalloc_file_private_data *priv, 273 struct ioctl_gntalloc_alloc_gref __user *arg) 274{ 275 int rc = 0; 276 struct ioctl_gntalloc_alloc_gref op; 277 uint32_t *gref_ids; 278 279 pr_debug("%s: priv %p\n", __func__, priv); 280 281 if (copy_from_user(&op, arg, sizeof(op))) { 282 rc = -EFAULT; 283 goto out; 284 } 285 286 gref_ids = kcalloc(op.count, sizeof(gref_ids[0]), GFP_KERNEL); 287 if (!gref_ids) { 288 rc = -ENOMEM; 289 goto out; 290 } 291 292 mutex_lock(&gref_mutex); 293 /* Clean up pages that were at zero (local) users but were still mapped 294 * by remote domains. Since those pages count towards the limit that we 295 * are about to enforce, removing them here is a good idea. 296 */ 297 do_cleanup(); 298 if (gref_size + op.count > limit) { 299 mutex_unlock(&gref_mutex); 300 rc = -ENOSPC; 301 goto out_free; 302 } 303 gref_size += op.count; 304 op.index = priv->index; 305 priv->index += op.count * PAGE_SIZE; 306 mutex_unlock(&gref_mutex); 307 308 rc = add_grefs(&op, gref_ids, priv); 309 if (rc < 0) 310 goto out_free; 311 312 /* Once we finish add_grefs, it is unsafe to touch the new reference, 313 * since it is possible for a concurrent ioctl to remove it (by guessing 314 * its index). If the userspace application doesn't provide valid memory 315 * to write the IDs to, then it will need to close the file in order to 316 * release - which it will do by segfaulting when it tries to access the 317 * IDs to close them. 318 */ 319 if (copy_to_user(arg, &op, sizeof(op))) { 320 rc = -EFAULT; 321 goto out_free; 322 } 323 if (copy_to_user(arg->gref_ids, gref_ids, 324 sizeof(gref_ids[0]) * op.count)) { 325 rc = -EFAULT; 326 goto out_free; 327 } 328 329out_free: 330 kfree(gref_ids); 331out: 332 return rc; 333} 334 335static long gntalloc_ioctl_dealloc(struct gntalloc_file_private_data *priv, 336 void __user *arg) 337{ 338 int i, rc = 0; 339 struct ioctl_gntalloc_dealloc_gref op; 340 struct gntalloc_gref *gref, *n; 341 342 pr_debug("%s: priv %p\n", __func__, priv); 343 344 if (copy_from_user(&op, arg, sizeof(op))) { 345 rc = -EFAULT; 346 goto dealloc_grant_out; 347 } 348 349 mutex_lock(&gref_mutex); 350 gref = find_grefs(priv, op.index, op.count); 351 if (gref) { 352 /* Remove from the file list only, and decrease reference count. 353 * The later call to do_cleanup() will remove from gref_list and 354 * free the memory if the pages aren't mapped anywhere. 355 */ 356 for (i = 0; i < op.count; i++) { 357 n = list_entry(gref->next_file.next, 358 struct gntalloc_gref, next_file); 359 list_del(&gref->next_file); 360 gref->users--; 361 gref = n; 362 } 363 } else { 364 rc = -EINVAL; 365 } 366 367 do_cleanup(); 368 369 mutex_unlock(&gref_mutex); 370dealloc_grant_out: 371 return rc; 372} 373 374static long gntalloc_ioctl_unmap_notify(struct gntalloc_file_private_data *priv, 375 void __user *arg) 376{ 377 struct ioctl_gntalloc_unmap_notify op; 378 struct gntalloc_gref *gref; 379 uint64_t index; 380 int pgoff; 381 int rc; 382 383 if (copy_from_user(&op, arg, sizeof(op))) 384 return -EFAULT; 385 386 index = op.index & ~(PAGE_SIZE - 1); 387 pgoff = op.index & (PAGE_SIZE - 1); 388 389 mutex_lock(&gref_mutex); 390 391 gref = find_grefs(priv, index, 1); 392 if (!gref) { 393 rc = -ENOENT; 394 goto unlock_out; 395 } 396 397 if (op.action & ~(UNMAP_NOTIFY_CLEAR_BYTE|UNMAP_NOTIFY_SEND_EVENT)) { 398 rc = -EINVAL; 399 goto unlock_out; 400 } 401 402 /* We need to grab a reference to the event channel we are going to use 403 * to send the notify before releasing the reference we may already have 404 * (if someone has called this ioctl twice). This is required so that 405 * it is possible to change the clear_byte part of the notification 406 * without disturbing the event channel part, which may now be the last 407 * reference to that event channel. 408 */ 409 if (op.action & UNMAP_NOTIFY_SEND_EVENT) { 410 if (evtchn_get(op.event_channel_port)) { 411 rc = -EINVAL; 412 goto unlock_out; 413 } 414 } 415 416 if (gref->notify.flags & UNMAP_NOTIFY_SEND_EVENT) 417 evtchn_put(gref->notify.event); 418 419 gref->notify.flags = op.action; 420 gref->notify.pgoff = pgoff; 421 gref->notify.event = op.event_channel_port; 422 rc = 0; 423 424 unlock_out: 425 mutex_unlock(&gref_mutex); 426 return rc; 427} 428 429static long gntalloc_ioctl(struct file *filp, unsigned int cmd, 430 unsigned long arg) 431{ 432 struct gntalloc_file_private_data *priv = filp->private_data; 433 434 switch (cmd) { 435 case IOCTL_GNTALLOC_ALLOC_GREF: 436 return gntalloc_ioctl_alloc(priv, (void __user *)arg); 437 438 case IOCTL_GNTALLOC_DEALLOC_GREF: 439 return gntalloc_ioctl_dealloc(priv, (void __user *)arg); 440 441 case IOCTL_GNTALLOC_SET_UNMAP_NOTIFY: 442 return gntalloc_ioctl_unmap_notify(priv, (void __user *)arg); 443 444 default: 445 return -ENOIOCTLCMD; 446 } 447 448 return 0; 449} 450 451static void gntalloc_vma_open(struct vm_area_struct *vma) 452{ 453 struct gntalloc_vma_private_data *priv = vma->vm_private_data; 454 455 if (!priv) 456 return; 457 458 mutex_lock(&gref_mutex); 459 priv->users++; 460 mutex_unlock(&gref_mutex); 461} 462 463static void gntalloc_vma_close(struct vm_area_struct *vma) 464{ 465 struct gntalloc_vma_private_data *priv = vma->vm_private_data; 466 struct gntalloc_gref *gref, *next; 467 int i; 468 469 if (!priv) 470 return; 471 472 mutex_lock(&gref_mutex); 473 priv->users--; 474 if (priv->users == 0) { 475 gref = priv->gref; 476 for (i = 0; i < priv->count; i++) { 477 gref->users--; 478 next = list_entry(gref->next_gref.next, 479 struct gntalloc_gref, next_gref); 480 if (gref->users == 0) 481 __del_gref(gref); 482 gref = next; 483 } 484 kfree(priv); 485 } 486 mutex_unlock(&gref_mutex); 487} 488 489static const struct vm_operations_struct gntalloc_vmops = { 490 .open = gntalloc_vma_open, 491 .close = gntalloc_vma_close, 492}; 493 494static int gntalloc_mmap(struct file *filp, struct vm_area_struct *vma) 495{ 496 struct gntalloc_file_private_data *priv = filp->private_data; 497 struct gntalloc_vma_private_data *vm_priv; 498 struct gntalloc_gref *gref; 499 int count = vma_pages(vma); 500 int rv, i; 501 502 if (!(vma->vm_flags & VM_SHARED)) { 503 pr_err("%s: Mapping must be shared\n", __func__); 504 return -EINVAL; 505 } 506 507 vm_priv = kmalloc(sizeof(*vm_priv), GFP_KERNEL); 508 if (!vm_priv) 509 return -ENOMEM; 510 511 mutex_lock(&gref_mutex); 512 513 pr_debug("%s: priv %p,%p, page %lu+%d\n", __func__, 514 priv, vm_priv, vma->vm_pgoff, count); 515 516 gref = find_grefs(priv, vma->vm_pgoff << PAGE_SHIFT, count); 517 if (gref == NULL) { 518 rv = -ENOENT; 519 pr_debug("%s: Could not find grant reference", 520 __func__); 521 kfree(vm_priv); 522 goto out_unlock; 523 } 524 525 vm_priv->gref = gref; 526 vm_priv->users = 1; 527 vm_priv->count = count; 528 529 vma->vm_private_data = vm_priv; 530 531 vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP; 532 533 vma->vm_ops = &gntalloc_vmops; 534 535 for (i = 0; i < count; i++) { 536 gref->users++; 537 rv = vm_insert_page(vma, vma->vm_start + i * PAGE_SIZE, 538 gref->page); 539 if (rv) 540 goto out_unlock; 541 542 gref = list_entry(gref->next_file.next, 543 struct gntalloc_gref, next_file); 544 } 545 rv = 0; 546 547out_unlock: 548 mutex_unlock(&gref_mutex); 549 return rv; 550} 551 552static const struct file_operations gntalloc_fops = { 553 .owner = THIS_MODULE, 554 .open = gntalloc_open, 555 .release = gntalloc_release, 556 .unlocked_ioctl = gntalloc_ioctl, 557 .mmap = gntalloc_mmap 558}; 559 560/* 561 * ------------------------------------- 562 * Module creation/destruction. 563 * ------------------------------------- 564 */ 565static struct miscdevice gntalloc_miscdev = { 566 .minor = MISC_DYNAMIC_MINOR, 567 .name = "xen/gntalloc", 568 .fops = &gntalloc_fops, 569}; 570 571static int __init gntalloc_init(void) 572{ 573 int err; 574 575 if (!xen_domain()) 576 return -ENODEV; 577 578 err = misc_register(&gntalloc_miscdev); 579 if (err != 0) { 580 pr_err("Could not register misc gntalloc device\n"); 581 return err; 582 } 583 584 pr_debug("Created grant allocation device at %d,%d\n", 585 MISC_MAJOR, gntalloc_miscdev.minor); 586 587 return 0; 588} 589 590static void __exit gntalloc_exit(void) 591{ 592 misc_deregister(&gntalloc_miscdev); 593} 594 595module_init(gntalloc_init); 596module_exit(gntalloc_exit); 597 598MODULE_LICENSE("GPL"); 599MODULE_AUTHOR("Carter Weatherly <carter.weatherly@jhuapl.edu>, " 600 "Daniel De Graaf <dgdegra@tycho.nsa.gov>"); 601MODULE_DESCRIPTION("User-space grant reference allocator driver"); 602