1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3#include <linux/init.h> 4#include <linux/kernel.h> 5#include <linux/module.h> 6#include <linux/pci.h> 7#include <linux/device.h> 8#include <linux/sched/task.h> 9#include <linux/io-64-nonatomic-lo-hi.h> 10#include <linux/cdev.h> 11#include <linux/fs.h> 12#include <linux/poll.h> 13#include <linux/iommu.h> 14#include <linux/highmem.h> 15#include <uapi/linux/idxd.h> 16#include <linux/xarray.h> 17#include "registers.h" 18#include "idxd.h" 19 20struct idxd_cdev_context { 21 const char *name; 22 dev_t devt; 23 struct ida minor_ida; 24}; 25 26/* 27 * Since user file names are global in DSA devices, define their ida's as 28 * global to avoid conflict file names. 29 */ 30static DEFINE_IDA(file_ida); 31static DEFINE_MUTEX(ida_lock); 32 33/* 34 * ictx is an array based off of accelerator types. enum idxd_type 35 * is used as index 36 */ 37static struct idxd_cdev_context ictx[IDXD_TYPE_MAX] = { 38 { .name = "dsa" }, 39 { .name = "iax" } 40}; 41 42struct idxd_user_context { 43 struct idxd_wq *wq; 44 struct task_struct *task; 45 unsigned int pasid; 46 struct mm_struct *mm; 47 unsigned int flags; 48 struct iommu_sva *sva; 49 struct idxd_dev idxd_dev; 50 u64 counters[COUNTER_MAX]; 51 int id; 52 pid_t pid; 53}; 54 55static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid); 56static void idxd_xa_pasid_remove(struct idxd_user_context *ctx); 57 58static inline struct idxd_user_context *dev_to_uctx(struct device *dev) 59{ 60 struct idxd_dev *idxd_dev = confdev_to_idxd_dev(dev); 61 62 return container_of(idxd_dev, struct idxd_user_context, idxd_dev); 63} 64 65static ssize_t cr_faults_show(struct device *dev, struct device_attribute *attr, char *buf) 66{ 67 struct idxd_user_context *ctx = dev_to_uctx(dev); 68 69 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULTS]); 70} 71static DEVICE_ATTR_RO(cr_faults); 72 73static ssize_t cr_fault_failures_show(struct device *dev, 74 struct device_attribute *attr, char *buf) 75{ 76 struct idxd_user_context *ctx = dev_to_uctx(dev); 77 78 return sysfs_emit(buf, "%llu\n", ctx->counters[COUNTER_FAULT_FAILS]); 79} 80static DEVICE_ATTR_RO(cr_fault_failures); 81 82static ssize_t pid_show(struct device *dev, struct device_attribute *attr, char *buf) 83{ 84 struct idxd_user_context *ctx = dev_to_uctx(dev); 85 86 return sysfs_emit(buf, "%u\n", ctx->pid); 87} 88static DEVICE_ATTR_RO(pid); 89 90static struct attribute *cdev_file_attributes[] = { 91 &dev_attr_cr_faults.attr, 92 &dev_attr_cr_fault_failures.attr, 93 &dev_attr_pid.attr, 94 NULL 95}; 96 97static umode_t cdev_file_attr_visible(struct kobject *kobj, struct attribute *a, int n) 98{ 99 struct device *dev = container_of(kobj, typeof(*dev), kobj); 100 struct idxd_user_context *ctx = dev_to_uctx(dev); 101 struct idxd_wq *wq = ctx->wq; 102 103 if (!wq_pasid_enabled(wq)) 104 return 0; 105 106 return a->mode; 107} 108 109static const struct attribute_group cdev_file_attribute_group = { 110 .attrs = cdev_file_attributes, 111 .is_visible = cdev_file_attr_visible, 112}; 113 114static const struct attribute_group *cdev_file_attribute_groups[] = { 115 &cdev_file_attribute_group, 116 NULL 117}; 118 119static void idxd_file_dev_release(struct device *dev) 120{ 121 struct idxd_user_context *ctx = dev_to_uctx(dev); 122 struct idxd_wq *wq = ctx->wq; 123 struct idxd_device *idxd = wq->idxd; 124 int rc; 125 126 mutex_lock(&ida_lock); 127 ida_free(&file_ida, ctx->id); 128 mutex_unlock(&ida_lock); 129 130 /* Wait for in-flight operations to complete. */ 131 if (wq_shared(wq)) { 132 idxd_device_drain_pasid(idxd, ctx->pasid); 133 } else { 134 if (device_user_pasid_enabled(idxd)) { 135 /* The wq disable in the disable pasid function will drain the wq */ 136 rc = idxd_wq_disable_pasid(wq); 137 if (rc < 0) 138 dev_err(dev, "wq disable pasid failed.\n"); 139 } else { 140 idxd_wq_drain(wq); 141 } 142 } 143 144 if (ctx->sva) { 145 idxd_cdev_evl_drain_pasid(wq, ctx->pasid); 146 iommu_sva_unbind_device(ctx->sva); 147 idxd_xa_pasid_remove(ctx); 148 } 149 kfree(ctx); 150 mutex_lock(&wq->wq_lock); 151 idxd_wq_put(wq); 152 mutex_unlock(&wq->wq_lock); 153} 154 155static struct device_type idxd_cdev_file_type = { 156 .name = "idxd_file", 157 .release = idxd_file_dev_release, 158 .groups = cdev_file_attribute_groups, 159}; 160 161static void idxd_cdev_dev_release(struct device *dev) 162{ 163 struct idxd_cdev *idxd_cdev = dev_to_cdev(dev); 164 struct idxd_cdev_context *cdev_ctx; 165 struct idxd_wq *wq = idxd_cdev->wq; 166 167 cdev_ctx = &ictx[wq->idxd->data->type]; 168 ida_simple_remove(&cdev_ctx->minor_ida, idxd_cdev->minor); 169 kfree(idxd_cdev); 170} 171 172static struct device_type idxd_cdev_device_type = { 173 .name = "idxd_cdev", 174 .release = idxd_cdev_dev_release, 175}; 176 177static inline struct idxd_cdev *inode_idxd_cdev(struct inode *inode) 178{ 179 struct cdev *cdev = inode->i_cdev; 180 181 return container_of(cdev, struct idxd_cdev, cdev); 182} 183 184static inline struct idxd_wq *inode_wq(struct inode *inode) 185{ 186 struct idxd_cdev *idxd_cdev = inode_idxd_cdev(inode); 187 188 return idxd_cdev->wq; 189} 190 191static void idxd_xa_pasid_remove(struct idxd_user_context *ctx) 192{ 193 struct idxd_wq *wq = ctx->wq; 194 void *ptr; 195 196 mutex_lock(&wq->uc_lock); 197 ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL); 198 if (ptr != (void *)ctx) 199 dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n", 200 ctx->pasid); 201 mutex_unlock(&wq->uc_lock); 202} 203 204void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index) 205{ 206 struct idxd_user_context *ctx; 207 208 if (index >= COUNTER_MAX) 209 return; 210 211 mutex_lock(&wq->uc_lock); 212 ctx = xa_load(&wq->upasid_xa, pasid); 213 if (!ctx) { 214 mutex_unlock(&wq->uc_lock); 215 return; 216 } 217 ctx->counters[index]++; 218 mutex_unlock(&wq->uc_lock); 219} 220 221static int idxd_cdev_open(struct inode *inode, struct file *filp) 222{ 223 struct idxd_user_context *ctx; 224 struct idxd_device *idxd; 225 struct idxd_wq *wq; 226 struct device *dev, *fdev; 227 int rc = 0; 228 struct iommu_sva *sva; 229 unsigned int pasid; 230 struct idxd_cdev *idxd_cdev; 231 232 wq = inode_wq(inode); 233 idxd = wq->idxd; 234 dev = &idxd->pdev->dev; 235 236 dev_dbg(dev, "%s called: %d\n", __func__, idxd_wq_refcount(wq)); 237 238 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 239 if (!ctx) 240 return -ENOMEM; 241 242 mutex_lock(&wq->wq_lock); 243 244 if (idxd_wq_refcount(wq) > 0 && wq_dedicated(wq)) { 245 rc = -EBUSY; 246 goto failed; 247 } 248 249 ctx->wq = wq; 250 filp->private_data = ctx; 251 ctx->pid = current->pid; 252 253 if (device_user_pasid_enabled(idxd)) { 254 sva = iommu_sva_bind_device(dev, current->mm); 255 if (IS_ERR(sva)) { 256 rc = PTR_ERR(sva); 257 dev_err(dev, "pasid allocation failed: %d\n", rc); 258 goto failed; 259 } 260 261 pasid = iommu_sva_get_pasid(sva); 262 if (pasid == IOMMU_PASID_INVALID) { 263 rc = -EINVAL; 264 goto failed_get_pasid; 265 } 266 267 ctx->sva = sva; 268 ctx->pasid = pasid; 269 ctx->mm = current->mm; 270 271 mutex_lock(&wq->uc_lock); 272 rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL); 273 mutex_unlock(&wq->uc_lock); 274 if (rc < 0) 275 dev_warn(dev, "PASID entry already exist in xarray.\n"); 276 277 if (wq_dedicated(wq)) { 278 rc = idxd_wq_set_pasid(wq, pasid); 279 if (rc < 0) { 280 dev_err(dev, "wq set pasid failed: %d\n", rc); 281 goto failed_set_pasid; 282 } 283 } 284 } 285 286 idxd_cdev = wq->idxd_cdev; 287 mutex_lock(&ida_lock); 288 ctx->id = ida_alloc(&file_ida, GFP_KERNEL); 289 mutex_unlock(&ida_lock); 290 if (ctx->id < 0) { 291 dev_warn(dev, "ida alloc failure\n"); 292 goto failed_ida; 293 } 294 ctx->idxd_dev.type = IDXD_DEV_CDEV_FILE; 295 fdev = user_ctx_dev(ctx); 296 device_initialize(fdev); 297 fdev->parent = cdev_dev(idxd_cdev); 298 fdev->bus = &dsa_bus_type; 299 fdev->type = &idxd_cdev_file_type; 300 301 rc = dev_set_name(fdev, "file%d", ctx->id); 302 if (rc < 0) { 303 dev_warn(dev, "set name failure\n"); 304 goto failed_dev_name; 305 } 306 307 rc = device_add(fdev); 308 if (rc < 0) { 309 dev_warn(dev, "file device add failure\n"); 310 goto failed_dev_add; 311 } 312 313 idxd_wq_get(wq); 314 mutex_unlock(&wq->wq_lock); 315 return 0; 316 317failed_dev_add: 318failed_dev_name: 319 put_device(fdev); 320failed_ida: 321failed_set_pasid: 322 if (device_user_pasid_enabled(idxd)) 323 idxd_xa_pasid_remove(ctx); 324failed_get_pasid: 325 if (device_user_pasid_enabled(idxd)) 326 iommu_sva_unbind_device(sva); 327failed: 328 mutex_unlock(&wq->wq_lock); 329 kfree(ctx); 330 return rc; 331} 332 333static void idxd_cdev_evl_drain_pasid(struct idxd_wq *wq, u32 pasid) 334{ 335 struct idxd_device *idxd = wq->idxd; 336 struct idxd_evl *evl = idxd->evl; 337 union evl_status_reg status; 338 u16 h, t, size; 339 int ent_size = evl_ent_size(idxd); 340 struct __evl_entry *entry_head; 341 342 if (!evl) 343 return; 344 345 spin_lock(&evl->lock); 346 status.bits = ioread64(idxd->reg_base + IDXD_EVLSTATUS_OFFSET); 347 t = status.tail; 348 h = status.head; 349 size = evl->size; 350 351 while (h != t) { 352 entry_head = (struct __evl_entry *)(evl->log + (h * ent_size)); 353 if (entry_head->pasid == pasid && entry_head->wq_idx == wq->id) 354 set_bit(h, evl->bmap); 355 h = (h + 1) % size; 356 } 357 spin_unlock(&evl->lock); 358 359 drain_workqueue(wq->wq); 360} 361 362static int idxd_cdev_release(struct inode *node, struct file *filep) 363{ 364 struct idxd_user_context *ctx = filep->private_data; 365 struct idxd_wq *wq = ctx->wq; 366 struct idxd_device *idxd = wq->idxd; 367 struct device *dev = &idxd->pdev->dev; 368 369 dev_dbg(dev, "%s called\n", __func__); 370 filep->private_data = NULL; 371 372 device_unregister(user_ctx_dev(ctx)); 373 374 return 0; 375} 376 377static int check_vma(struct idxd_wq *wq, struct vm_area_struct *vma, 378 const char *func) 379{ 380 struct device *dev = &wq->idxd->pdev->dev; 381 382 if ((vma->vm_end - vma->vm_start) > PAGE_SIZE) { 383 dev_info_ratelimited(dev, 384 "%s: %s: mapping too large: %lu\n", 385 current->comm, func, 386 vma->vm_end - vma->vm_start); 387 return -EINVAL; 388 } 389 390 return 0; 391} 392 393static int idxd_cdev_mmap(struct file *filp, struct vm_area_struct *vma) 394{ 395 struct idxd_user_context *ctx = filp->private_data; 396 struct idxd_wq *wq = ctx->wq; 397 struct idxd_device *idxd = wq->idxd; 398 struct pci_dev *pdev = idxd->pdev; 399 phys_addr_t base = pci_resource_start(pdev, IDXD_WQ_BAR); 400 unsigned long pfn; 401 int rc; 402 403 dev_dbg(&pdev->dev, "%s called\n", __func__); 404 rc = check_vma(wq, vma, __func__); 405 if (rc < 0) 406 return rc; 407 408 vm_flags_set(vma, VM_DONTCOPY); 409 pfn = (base + idxd_get_wq_portal_full_offset(wq->id, 410 IDXD_PORTAL_LIMITED)) >> PAGE_SHIFT; 411 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 412 vma->vm_private_data = ctx; 413 414 return io_remap_pfn_range(vma, vma->vm_start, pfn, PAGE_SIZE, 415 vma->vm_page_prot); 416} 417 418static __poll_t idxd_cdev_poll(struct file *filp, 419 struct poll_table_struct *wait) 420{ 421 struct idxd_user_context *ctx = filp->private_data; 422 struct idxd_wq *wq = ctx->wq; 423 struct idxd_device *idxd = wq->idxd; 424 __poll_t out = 0; 425 426 poll_wait(filp, &wq->err_queue, wait); 427 spin_lock(&idxd->dev_lock); 428 if (idxd->sw_err.valid) 429 out = EPOLLIN | EPOLLRDNORM; 430 spin_unlock(&idxd->dev_lock); 431 432 return out; 433} 434 435static const struct file_operations idxd_cdev_fops = { 436 .owner = THIS_MODULE, 437 .open = idxd_cdev_open, 438 .release = idxd_cdev_release, 439 .mmap = idxd_cdev_mmap, 440 .poll = idxd_cdev_poll, 441}; 442 443int idxd_cdev_get_major(struct idxd_device *idxd) 444{ 445 return MAJOR(ictx[idxd->data->type].devt); 446} 447 448int idxd_wq_add_cdev(struct idxd_wq *wq) 449{ 450 struct idxd_device *idxd = wq->idxd; 451 struct idxd_cdev *idxd_cdev; 452 struct cdev *cdev; 453 struct device *dev; 454 struct idxd_cdev_context *cdev_ctx; 455 int rc, minor; 456 457 idxd_cdev = kzalloc(sizeof(*idxd_cdev), GFP_KERNEL); 458 if (!idxd_cdev) 459 return -ENOMEM; 460 461 idxd_cdev->idxd_dev.type = IDXD_DEV_CDEV; 462 idxd_cdev->wq = wq; 463 cdev = &idxd_cdev->cdev; 464 dev = cdev_dev(idxd_cdev); 465 cdev_ctx = &ictx[wq->idxd->data->type]; 466 minor = ida_simple_get(&cdev_ctx->minor_ida, 0, MINORMASK, GFP_KERNEL); 467 if (minor < 0) { 468 kfree(idxd_cdev); 469 return minor; 470 } 471 idxd_cdev->minor = minor; 472 473 device_initialize(dev); 474 dev->parent = wq_confdev(wq); 475 dev->bus = &dsa_bus_type; 476 dev->type = &idxd_cdev_device_type; 477 dev->devt = MKDEV(MAJOR(cdev_ctx->devt), minor); 478 479 rc = dev_set_name(dev, "%s/wq%u.%u", idxd->data->name_prefix, idxd->id, wq->id); 480 if (rc < 0) 481 goto err; 482 483 wq->idxd_cdev = idxd_cdev; 484 cdev_init(cdev, &idxd_cdev_fops); 485 rc = cdev_device_add(cdev, dev); 486 if (rc) { 487 dev_dbg(&wq->idxd->pdev->dev, "cdev_add failed: %d\n", rc); 488 goto err; 489 } 490 491 return 0; 492 493 err: 494 put_device(dev); 495 wq->idxd_cdev = NULL; 496 return rc; 497} 498 499void idxd_wq_del_cdev(struct idxd_wq *wq) 500{ 501 struct idxd_cdev *idxd_cdev; 502 503 idxd_cdev = wq->idxd_cdev; 504 ida_destroy(&file_ida); 505 wq->idxd_cdev = NULL; 506 cdev_device_del(&idxd_cdev->cdev, cdev_dev(idxd_cdev)); 507 put_device(cdev_dev(idxd_cdev)); 508} 509 510static int idxd_user_drv_probe(struct idxd_dev *idxd_dev) 511{ 512 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 513 struct idxd_device *idxd = wq->idxd; 514 int rc; 515 516 if (idxd->state != IDXD_DEV_ENABLED) 517 return -ENXIO; 518 519 /* 520 * User type WQ is enabled only when SVA is enabled for two reasons: 521 * - If no IOMMU or IOMMU Passthrough without SVA, userspace 522 * can directly access physical address through the WQ. 523 * - The IDXD cdev driver does not provide any ways to pin 524 * user pages and translate the address from user VA to IOVA or 525 * PA without IOMMU SVA. Therefore the application has no way 526 * to instruct the device to perform DMA function. This makes 527 * the cdev not usable for normal application usage. 528 */ 529 if (!device_user_pasid_enabled(idxd)) { 530 idxd->cmd_status = IDXD_SCMD_WQ_USER_NO_IOMMU; 531 dev_dbg(&idxd->pdev->dev, 532 "User type WQ cannot be enabled without SVA.\n"); 533 534 return -EOPNOTSUPP; 535 } 536 537 mutex_lock(&wq->wq_lock); 538 539 wq->wq = create_workqueue(dev_name(wq_confdev(wq))); 540 if (!wq->wq) { 541 rc = -ENOMEM; 542 goto wq_err; 543 } 544 545 wq->type = IDXD_WQT_USER; 546 rc = drv_enable_wq(wq); 547 if (rc < 0) 548 goto err; 549 550 rc = idxd_wq_add_cdev(wq); 551 if (rc < 0) { 552 idxd->cmd_status = IDXD_SCMD_CDEV_ERR; 553 goto err_cdev; 554 } 555 556 idxd->cmd_status = 0; 557 mutex_unlock(&wq->wq_lock); 558 return 0; 559 560err_cdev: 561 drv_disable_wq(wq); 562err: 563 destroy_workqueue(wq->wq); 564 wq->type = IDXD_WQT_NONE; 565wq_err: 566 mutex_unlock(&wq->wq_lock); 567 return rc; 568} 569 570static void idxd_user_drv_remove(struct idxd_dev *idxd_dev) 571{ 572 struct idxd_wq *wq = idxd_dev_to_wq(idxd_dev); 573 574 mutex_lock(&wq->wq_lock); 575 idxd_wq_del_cdev(wq); 576 drv_disable_wq(wq); 577 wq->type = IDXD_WQT_NONE; 578 destroy_workqueue(wq->wq); 579 wq->wq = NULL; 580 mutex_unlock(&wq->wq_lock); 581} 582 583static enum idxd_dev_type dev_types[] = { 584 IDXD_DEV_WQ, 585 IDXD_DEV_NONE, 586}; 587 588struct idxd_device_driver idxd_user_drv = { 589 .probe = idxd_user_drv_probe, 590 .remove = idxd_user_drv_remove, 591 .name = "user", 592 .type = dev_types, 593}; 594EXPORT_SYMBOL_GPL(idxd_user_drv); 595 596int idxd_cdev_register(void) 597{ 598 int rc, i; 599 600 for (i = 0; i < IDXD_TYPE_MAX; i++) { 601 ida_init(&ictx[i].minor_ida); 602 rc = alloc_chrdev_region(&ictx[i].devt, 0, MINORMASK, 603 ictx[i].name); 604 if (rc) 605 goto err_free_chrdev_region; 606 } 607 608 return 0; 609 610err_free_chrdev_region: 611 for (i--; i >= 0; i--) 612 unregister_chrdev_region(ictx[i].devt, MINORMASK); 613 614 return rc; 615} 616 617void idxd_cdev_remove(void) 618{ 619 int i; 620 621 for (i = 0; i < IDXD_TYPE_MAX; i++) { 622 unregister_chrdev_region(ictx[i].devt, MINORMASK); 623 ida_destroy(&ictx[i].minor_ida); 624 } 625} 626 627/** 628 * idxd_copy_cr - copy completion record to user address space found by wq and 629 * PASID 630 * @wq: work queue 631 * @pasid: PASID 632 * @addr: user fault address to write 633 * @cr: completion record 634 * @len: number of bytes to copy 635 * 636 * This is called by a work that handles completion record fault. 637 * 638 * Return: number of bytes copied. 639 */ 640int idxd_copy_cr(struct idxd_wq *wq, ioasid_t pasid, unsigned long addr, 641 void *cr, int len) 642{ 643 struct device *dev = &wq->idxd->pdev->dev; 644 int left = len, status_size = 1; 645 struct idxd_user_context *ctx; 646 struct mm_struct *mm; 647 648 mutex_lock(&wq->uc_lock); 649 650 ctx = xa_load(&wq->upasid_xa, pasid); 651 if (!ctx) { 652 dev_warn(dev, "No user context\n"); 653 goto out; 654 } 655 656 mm = ctx->mm; 657 /* 658 * The completion record fault handling work is running in kernel 659 * thread context. It temporarily switches to the mm to copy cr 660 * to addr in the mm. 661 */ 662 kthread_use_mm(mm); 663 left = copy_to_user((void __user *)addr + status_size, cr + status_size, 664 len - status_size); 665 /* 666 * Copy status only after the rest of completion record is copied 667 * successfully so that the user gets the complete completion record 668 * when a non-zero status is polled. 669 */ 670 if (!left) { 671 u8 status; 672 673 /* 674 * Ensure that the completion record's status field is written 675 * after the rest of the completion record has been written. 676 * This ensures that the user receives the correct completion 677 * record information once polling for a non-zero status. 678 */ 679 wmb(); 680 status = *(u8 *)cr; 681 if (put_user(status, (u8 __user *)addr)) 682 left += status_size; 683 } else { 684 left += status_size; 685 } 686 kthread_unuse_mm(mm); 687 688out: 689 mutex_unlock(&wq->uc_lock); 690 691 return len - left; 692} 693