1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2012 Red Hat, Inc. All rights reserved. 4 * Author: Alex Williamson <alex.williamson@redhat.com> 5 * 6 * Derived from original vfio: 7 * Copyright 2010 Cisco Systems, Inc. All rights reserved. 8 * Author: Tom Lyon, pugs@cisco.com 9 */ 10 11#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 12 13#include <linux/device.h> 14#include <linux/eventfd.h> 15#include <linux/file.h> 16#include <linux/interrupt.h> 17#include <linux/iommu.h> 18#include <linux/module.h> 19#include <linux/mutex.h> 20#include <linux/notifier.h> 21#include <linux/pci.h> 22#include <linux/pm_runtime.h> 23#include <linux/slab.h> 24#include <linux/types.h> 25#include <linux/uaccess.h> 26#include <linux/vfio.h> 27#include <linux/vgaarb.h> 28#include <linux/nospec.h> 29#include <linux/sched/mm.h> 30 31#include "vfio_pci_private.h" 32 33#define DRIVER_VERSION "0.2" 34#define DRIVER_AUTHOR "Alex Williamson <alex.williamson@redhat.com>" 35#define DRIVER_DESC "VFIO PCI - User Level meta-driver" 36 37static char ids[1024] __initdata; 38module_param_string(ids, ids, sizeof(ids), 0); 39MODULE_PARM_DESC(ids, "Initial PCI IDs to add to the vfio driver, format is \"vendor:device[:subvendor[:subdevice[:class[:class_mask]]]]\" and multiple comma separated entries can be specified"); 40 41static bool nointxmask; 42module_param_named(nointxmask, nointxmask, bool, S_IRUGO | S_IWUSR); 43MODULE_PARM_DESC(nointxmask, 44 "Disable support for PCI 2.3 style INTx masking. If this resolves problems for specific devices, report lspci -vvvxxx to linux-pci@vger.kernel.org so the device can be fixed automatically via the broken_intx_masking flag."); 45 46#ifdef CONFIG_VFIO_PCI_VGA 47static bool disable_vga; 48module_param(disable_vga, bool, S_IRUGO); 49MODULE_PARM_DESC(disable_vga, "Disable VGA resource access through vfio-pci"); 50#endif 51 52static bool disable_idle_d3; 53module_param(disable_idle_d3, bool, S_IRUGO | S_IWUSR); 54MODULE_PARM_DESC(disable_idle_d3, 55 "Disable using the PCI D3 low power state for idle, unused devices"); 56 57static bool enable_sriov; 58#ifdef CONFIG_PCI_IOV 59module_param(enable_sriov, bool, 0644); 60MODULE_PARM_DESC(enable_sriov, "Enable support for SR-IOV configuration. Enabling SR-IOV on a PF typically requires support of the userspace PF driver, enabling VFs without such support may result in non-functional VFs or PF."); 61#endif 62 63static bool disable_denylist; 64module_param(disable_denylist, bool, 0444); 65MODULE_PARM_DESC(disable_denylist, "Disable use of device denylist. Disabling the denylist allows binding to devices with known errata that may lead to exploitable stability or security issues when accessed by untrusted users."); 66 67static inline bool vfio_vga_disabled(void) 68{ 69#ifdef CONFIG_VFIO_PCI_VGA 70 return disable_vga; 71#else 72 return true; 73#endif 74} 75 76static bool vfio_pci_dev_in_denylist(struct pci_dev *pdev) 77{ 78 switch (pdev->vendor) { 79 case PCI_VENDOR_ID_INTEL: 80 switch (pdev->device) { 81 case PCI_DEVICE_ID_INTEL_QAT_C3XXX: 82 case PCI_DEVICE_ID_INTEL_QAT_C3XXX_VF: 83 case PCI_DEVICE_ID_INTEL_QAT_C62X: 84 case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: 85 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC: 86 case PCI_DEVICE_ID_INTEL_QAT_DH895XCC_VF: 87 return true; 88 default: 89 return false; 90 } 91 } 92 93 return false; 94} 95 96static bool vfio_pci_is_denylisted(struct pci_dev *pdev) 97{ 98 if (!vfio_pci_dev_in_denylist(pdev)) 99 return false; 100 101 if (disable_denylist) { 102 pci_warn(pdev, 103 "device denylist disabled - allowing device %04x:%04x.\n", 104 pdev->vendor, pdev->device); 105 return false; 106 } 107 108 pci_warn(pdev, "%04x:%04x exists in vfio-pci device denylist, driver probing disallowed.\n", 109 pdev->vendor, pdev->device); 110 111 return true; 112} 113 114/* 115 * Our VGA arbiter participation is limited since we don't know anything 116 * about the device itself. However, if the device is the only VGA device 117 * downstream of a bridge and VFIO VGA support is disabled, then we can 118 * safely return legacy VGA IO and memory as not decoded since the user 119 * has no way to get to it and routing can be disabled externally at the 120 * bridge. 121 */ 122static unsigned int vfio_pci_set_vga_decode(void *opaque, bool single_vga) 123{ 124 struct vfio_pci_device *vdev = opaque; 125 struct pci_dev *tmp = NULL, *pdev = vdev->pdev; 126 unsigned char max_busnr; 127 unsigned int decodes; 128 129 if (single_vga || !vfio_vga_disabled() || pci_is_root_bus(pdev->bus)) 130 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 131 VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 132 133 max_busnr = pci_bus_max_busnr(pdev->bus); 134 decodes = VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM; 135 136 while ((tmp = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, tmp)) != NULL) { 137 if (tmp == pdev || 138 pci_domain_nr(tmp->bus) != pci_domain_nr(pdev->bus) || 139 pci_is_root_bus(tmp->bus)) 140 continue; 141 142 if (tmp->bus->number >= pdev->bus->number && 143 tmp->bus->number <= max_busnr) { 144 pci_dev_put(tmp); 145 decodes |= VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM; 146 break; 147 } 148 } 149 150 return decodes; 151} 152 153static inline bool vfio_pci_is_vga(struct pci_dev *pdev) 154{ 155 return (pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA; 156} 157 158static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev) 159{ 160 struct resource *res; 161 int i; 162 struct vfio_pci_dummy_resource *dummy_res; 163 164 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 165 int bar = i + PCI_STD_RESOURCES; 166 167 res = &vdev->pdev->resource[bar]; 168 169 if (!IS_ENABLED(CONFIG_VFIO_PCI_MMAP)) 170 goto no_mmap; 171 172 if (!(res->flags & IORESOURCE_MEM)) 173 goto no_mmap; 174 175 /* 176 * The PCI core shouldn't set up a resource with a 177 * type but zero size. But there may be bugs that 178 * cause us to do that. 179 */ 180 if (!resource_size(res)) 181 goto no_mmap; 182 183 if (resource_size(res) >= PAGE_SIZE) { 184 vdev->bar_mmap_supported[bar] = true; 185 continue; 186 } 187 188 if (!(res->start & ~PAGE_MASK)) { 189 /* 190 * Add a dummy resource to reserve the remainder 191 * of the exclusive page in case that hot-add 192 * device's bar is assigned into it. 193 */ 194 dummy_res = kzalloc(sizeof(*dummy_res), GFP_KERNEL); 195 if (dummy_res == NULL) 196 goto no_mmap; 197 198 dummy_res->resource.name = "vfio sub-page reserved"; 199 dummy_res->resource.start = res->end + 1; 200 dummy_res->resource.end = res->start + PAGE_SIZE - 1; 201 dummy_res->resource.flags = res->flags; 202 if (request_resource(res->parent, 203 &dummy_res->resource)) { 204 kfree(dummy_res); 205 goto no_mmap; 206 } 207 dummy_res->index = bar; 208 list_add(&dummy_res->res_next, 209 &vdev->dummy_resources_list); 210 vdev->bar_mmap_supported[bar] = true; 211 continue; 212 } 213 /* 214 * Here we don't handle the case when the BAR is not page 215 * aligned because we can't expect the BAR will be 216 * assigned into the same location in a page in guest 217 * when we passthrough the BAR. And it's hard to access 218 * this BAR in userspace because we have no way to get 219 * the BAR's location in a page. 220 */ 221no_mmap: 222 vdev->bar_mmap_supported[bar] = false; 223 } 224} 225 226static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev); 227static void vfio_pci_disable(struct vfio_pci_device *vdev); 228static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data); 229 230/* 231 * INTx masking requires the ability to disable INTx signaling via PCI_COMMAND 232 * _and_ the ability detect when the device is asserting INTx via PCI_STATUS. 233 * If a device implements the former but not the latter we would typically 234 * expect broken_intx_masking be set and require an exclusive interrupt. 235 * However since we do have control of the device's ability to assert INTx, 236 * we can instead pretend that the device does not implement INTx, virtualizing 237 * the pin register to report zero and maintaining DisINTx set on the host. 238 */ 239static bool vfio_pci_nointx(struct pci_dev *pdev) 240{ 241 switch (pdev->vendor) { 242 case PCI_VENDOR_ID_INTEL: 243 switch (pdev->device) { 244 /* All i40e (XL710/X710/XXV710) 10/20/25/40GbE NICs */ 245 case 0x1572: 246 case 0x1574: 247 case 0x1580 ... 0x1581: 248 case 0x1583 ... 0x158b: 249 case 0x37d0 ... 0x37d2: 250 /* X550 */ 251 case 0x1563: 252 return true; 253 default: 254 return false; 255 } 256 } 257 258 return false; 259} 260 261static void vfio_pci_probe_power_state(struct vfio_pci_device *vdev) 262{ 263 struct pci_dev *pdev = vdev->pdev; 264 u16 pmcsr; 265 266 if (!pdev->pm_cap) 267 return; 268 269 pci_read_config_word(pdev, pdev->pm_cap + PCI_PM_CTRL, &pmcsr); 270 271 vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET); 272} 273 274/* 275 * pci_set_power_state() wrapper handling devices which perform a soft reset on 276 * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev, 277 * restore when returned to D0. Saved separately from pci_saved_state for use 278 * by PM capability emulation and separately from pci_dev internal saved state 279 * to avoid it being overwritten and consumed around other resets. 280 */ 281int vfio_pci_set_power_state(struct vfio_pci_device *vdev, pci_power_t state) 282{ 283 struct pci_dev *pdev = vdev->pdev; 284 bool needs_restore = false, needs_save = false; 285 int ret; 286 287 if (vdev->needs_pm_restore) { 288 if (pdev->current_state < PCI_D3hot && state >= PCI_D3hot) { 289 pci_save_state(pdev); 290 needs_save = true; 291 } 292 293 if (pdev->current_state >= PCI_D3hot && state <= PCI_D0) 294 needs_restore = true; 295 } 296 297 ret = pci_set_power_state(pdev, state); 298 299 if (!ret) { 300 /* D3 might be unsupported via quirk, skip unless in D3 */ 301 if (needs_save && pdev->current_state >= PCI_D3hot) { 302 vdev->pm_save = pci_store_saved_state(pdev); 303 } else if (needs_restore) { 304 pci_load_and_free_saved_state(pdev, &vdev->pm_save); 305 pci_restore_state(pdev); 306 } 307 } 308 309 return ret; 310} 311 312static int vfio_pci_enable(struct vfio_pci_device *vdev) 313{ 314 struct pci_dev *pdev = vdev->pdev; 315 int ret; 316 u16 cmd; 317 u8 msix_pos; 318 319 vfio_pci_set_power_state(vdev, PCI_D0); 320 321 /* Don't allow our initial saved state to include busmaster */ 322 pci_clear_master(pdev); 323 324 ret = pci_enable_device(pdev); 325 if (ret) 326 return ret; 327 328 /* If reset fails because of the device lock, fail this path entirely */ 329 ret = pci_try_reset_function(pdev); 330 if (ret == -EAGAIN) { 331 pci_disable_device(pdev); 332 return ret; 333 } 334 335 vdev->reset_works = !ret; 336 pci_save_state(pdev); 337 vdev->pci_saved_state = pci_store_saved_state(pdev); 338 if (!vdev->pci_saved_state) 339 pci_dbg(pdev, "%s: Couldn't store saved state\n", __func__); 340 341 if (likely(!nointxmask)) { 342 if (vfio_pci_nointx(pdev)) { 343 pci_info(pdev, "Masking broken INTx support\n"); 344 vdev->nointx = true; 345 pci_intx(pdev, 0); 346 } else 347 vdev->pci_2_3 = pci_intx_mask_supported(pdev); 348 } 349 350 pci_read_config_word(pdev, PCI_COMMAND, &cmd); 351 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) { 352 cmd &= ~PCI_COMMAND_INTX_DISABLE; 353 pci_write_config_word(pdev, PCI_COMMAND, cmd); 354 } 355 356 ret = vfio_config_init(vdev); 357 if (ret) { 358 kfree(vdev->pci_saved_state); 359 vdev->pci_saved_state = NULL; 360 pci_disable_device(pdev); 361 return ret; 362 } 363 364 msix_pos = pdev->msix_cap; 365 if (msix_pos) { 366 u16 flags; 367 u32 table; 368 369 pci_read_config_word(pdev, msix_pos + PCI_MSIX_FLAGS, &flags); 370 pci_read_config_dword(pdev, msix_pos + PCI_MSIX_TABLE, &table); 371 372 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR; 373 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET; 374 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16; 375 } else 376 vdev->msix_bar = 0xFF; 377 378 if (!vfio_vga_disabled() && vfio_pci_is_vga(pdev)) 379 vdev->has_vga = true; 380 381 382 if (vfio_pci_is_vga(pdev) && 383 pdev->vendor == PCI_VENDOR_ID_INTEL && 384 IS_ENABLED(CONFIG_VFIO_PCI_IGD)) { 385 ret = vfio_pci_igd_init(vdev); 386 if (ret && ret != -ENODEV) { 387 pci_warn(pdev, "Failed to setup Intel IGD regions\n"); 388 goto disable_exit; 389 } 390 } 391 392 if (pdev->vendor == PCI_VENDOR_ID_NVIDIA && 393 IS_ENABLED(CONFIG_VFIO_PCI_NVLINK2)) { 394 ret = vfio_pci_nvdia_v100_nvlink2_init(vdev); 395 if (ret && ret != -ENODEV) { 396 pci_warn(pdev, "Failed to setup NVIDIA NV2 RAM region\n"); 397 goto disable_exit; 398 } 399 } 400 401 if (pdev->vendor == PCI_VENDOR_ID_IBM && 402 IS_ENABLED(CONFIG_VFIO_PCI_NVLINK2)) { 403 ret = vfio_pci_ibm_npu2_init(vdev); 404 if (ret && ret != -ENODEV) { 405 pci_warn(pdev, "Failed to setup NVIDIA NV2 ATSD region\n"); 406 goto disable_exit; 407 } 408 } 409 410 vfio_pci_probe_mmaps(vdev); 411 412 return 0; 413 414disable_exit: 415 vfio_pci_disable(vdev); 416 return ret; 417} 418 419static void vfio_pci_disable(struct vfio_pci_device *vdev) 420{ 421 struct pci_dev *pdev = vdev->pdev; 422 struct vfio_pci_dummy_resource *dummy_res, *tmp; 423 struct vfio_pci_ioeventfd *ioeventfd, *ioeventfd_tmp; 424 int i, bar; 425 426 /* Stop the device from further DMA */ 427 pci_clear_master(pdev); 428 429 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE | 430 VFIO_IRQ_SET_ACTION_TRIGGER, 431 vdev->irq_type, 0, 0, NULL); 432 433 /* Device closed, don't need mutex here */ 434 list_for_each_entry_safe(ioeventfd, ioeventfd_tmp, 435 &vdev->ioeventfds_list, next) { 436 vfio_virqfd_disable(&ioeventfd->virqfd); 437 list_del(&ioeventfd->next); 438 kfree(ioeventfd); 439 } 440 vdev->ioeventfds_nr = 0; 441 442 vdev->virq_disabled = false; 443 444 for (i = 0; i < vdev->num_regions; i++) 445 vdev->region[i].ops->release(vdev, &vdev->region[i]); 446 447 vdev->num_regions = 0; 448 kfree(vdev->region); 449 vdev->region = NULL; /* don't krealloc a freed pointer */ 450 451 vfio_config_free(vdev); 452 453 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 454 bar = i + PCI_STD_RESOURCES; 455 if (!vdev->barmap[bar]) 456 continue; 457 pci_iounmap(pdev, vdev->barmap[bar]); 458 pci_release_selected_regions(pdev, 1 << bar); 459 vdev->barmap[bar] = NULL; 460 } 461 462 list_for_each_entry_safe(dummy_res, tmp, 463 &vdev->dummy_resources_list, res_next) { 464 list_del(&dummy_res->res_next); 465 release_resource(&dummy_res->resource); 466 kfree(dummy_res); 467 } 468 469 vdev->needs_reset = true; 470 471 /* 472 * If we have saved state, restore it. If we can reset the device, 473 * even better. Resetting with current state seems better than 474 * nothing, but saving and restoring current state without reset 475 * is just busy work. 476 */ 477 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) { 478 pci_info(pdev, "%s: Couldn't reload saved state\n", __func__); 479 480 if (!vdev->reset_works) 481 goto out; 482 483 pci_save_state(pdev); 484 } 485 486 /* 487 * Disable INTx and MSI, presumably to avoid spurious interrupts 488 * during reset. Stolen from pci_reset_function() 489 */ 490 pci_write_config_word(pdev, PCI_COMMAND, PCI_COMMAND_INTX_DISABLE); 491 492 /* 493 * Try to get the locks ourselves to prevent a deadlock. The 494 * success of this is dependent on being able to lock the device, 495 * which is not always possible. 496 * We can not use the "try" reset interface here, which will 497 * overwrite the previously restored configuration information. 498 */ 499 if (vdev->reset_works && pci_cfg_access_trylock(pdev)) { 500 if (device_trylock(&pdev->dev)) { 501 if (!__pci_reset_function_locked(pdev)) 502 vdev->needs_reset = false; 503 device_unlock(&pdev->dev); 504 } 505 pci_cfg_access_unlock(pdev); 506 } 507 508 pci_restore_state(pdev); 509out: 510 pci_disable_device(pdev); 511 512 vfio_pci_try_bus_reset(vdev); 513 514 if (!disable_idle_d3) 515 vfio_pci_set_power_state(vdev, PCI_D3hot); 516} 517 518static struct pci_driver vfio_pci_driver; 519 520static struct vfio_pci_device *get_pf_vdev(struct vfio_pci_device *vdev, 521 struct vfio_device **pf_dev) 522{ 523 struct pci_dev *physfn = pci_physfn(vdev->pdev); 524 525 if (!vdev->pdev->is_virtfn) 526 return NULL; 527 528 *pf_dev = vfio_device_get_from_dev(&physfn->dev); 529 if (!*pf_dev) 530 return NULL; 531 532 if (pci_dev_driver(physfn) != &vfio_pci_driver) { 533 vfio_device_put(*pf_dev); 534 return NULL; 535 } 536 537 return vfio_device_data(*pf_dev); 538} 539 540static void vfio_pci_vf_token_user_add(struct vfio_pci_device *vdev, int val) 541{ 542 struct vfio_device *pf_dev; 543 struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev); 544 545 if (!pf_vdev) 546 return; 547 548 mutex_lock(&pf_vdev->vf_token->lock); 549 pf_vdev->vf_token->users += val; 550 WARN_ON(pf_vdev->vf_token->users < 0); 551 mutex_unlock(&pf_vdev->vf_token->lock); 552 553 vfio_device_put(pf_dev); 554} 555 556static void vfio_pci_release(void *device_data) 557{ 558 struct vfio_pci_device *vdev = device_data; 559 560 mutex_lock(&vdev->reflck->lock); 561 562 if (!(--vdev->refcnt)) { 563 vfio_pci_vf_token_user_add(vdev, -1); 564 vfio_spapr_pci_eeh_release(vdev->pdev); 565 vfio_pci_disable(vdev); 566 567 mutex_lock(&vdev->igate); 568 if (vdev->err_trigger) { 569 eventfd_ctx_put(vdev->err_trigger); 570 vdev->err_trigger = NULL; 571 } 572 if (vdev->req_trigger) { 573 eventfd_ctx_put(vdev->req_trigger); 574 vdev->req_trigger = NULL; 575 } 576 mutex_unlock(&vdev->igate); 577 } 578 579 mutex_unlock(&vdev->reflck->lock); 580 581 module_put(THIS_MODULE); 582} 583 584static int vfio_pci_open(void *device_data) 585{ 586 struct vfio_pci_device *vdev = device_data; 587 int ret = 0; 588 589 if (!try_module_get(THIS_MODULE)) 590 return -ENODEV; 591 592 mutex_lock(&vdev->reflck->lock); 593 594 if (!vdev->refcnt) { 595 ret = vfio_pci_enable(vdev); 596 if (ret) 597 goto error; 598 599 vfio_spapr_pci_eeh_open(vdev->pdev); 600 vfio_pci_vf_token_user_add(vdev, 1); 601 } 602 vdev->refcnt++; 603error: 604 mutex_unlock(&vdev->reflck->lock); 605 if (ret) 606 module_put(THIS_MODULE); 607 return ret; 608} 609 610static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type) 611{ 612 if (irq_type == VFIO_PCI_INTX_IRQ_INDEX) { 613 u8 pin; 614 615 if (!IS_ENABLED(CONFIG_VFIO_PCI_INTX) || 616 vdev->nointx || vdev->pdev->is_virtfn) 617 return 0; 618 619 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin); 620 621 return pin ? 1 : 0; 622 } else if (irq_type == VFIO_PCI_MSI_IRQ_INDEX) { 623 u8 pos; 624 u16 flags; 625 626 pos = vdev->pdev->msi_cap; 627 if (pos) { 628 pci_read_config_word(vdev->pdev, 629 pos + PCI_MSI_FLAGS, &flags); 630 return 1 << ((flags & PCI_MSI_FLAGS_QMASK) >> 1); 631 } 632 } else if (irq_type == VFIO_PCI_MSIX_IRQ_INDEX) { 633 u8 pos; 634 u16 flags; 635 636 pos = vdev->pdev->msix_cap; 637 if (pos) { 638 pci_read_config_word(vdev->pdev, 639 pos + PCI_MSIX_FLAGS, &flags); 640 641 return (flags & PCI_MSIX_FLAGS_QSIZE) + 1; 642 } 643 } else if (irq_type == VFIO_PCI_ERR_IRQ_INDEX) { 644 if (pci_is_pcie(vdev->pdev)) 645 return 1; 646 } else if (irq_type == VFIO_PCI_REQ_IRQ_INDEX) { 647 return 1; 648 } 649 650 return 0; 651} 652 653static int vfio_pci_count_devs(struct pci_dev *pdev, void *data) 654{ 655 (*(int *)data)++; 656 return 0; 657} 658 659struct vfio_pci_fill_info { 660 int max; 661 int cur; 662 struct vfio_pci_dependent_device *devices; 663}; 664 665static int vfio_pci_fill_devs(struct pci_dev *pdev, void *data) 666{ 667 struct vfio_pci_fill_info *fill = data; 668 struct iommu_group *iommu_group; 669 670 if (fill->cur == fill->max) 671 return -EAGAIN; /* Something changed, try again */ 672 673 iommu_group = iommu_group_get(&pdev->dev); 674 if (!iommu_group) 675 return -EPERM; /* Cannot reset non-isolated devices */ 676 677 fill->devices[fill->cur].group_id = iommu_group_id(iommu_group); 678 fill->devices[fill->cur].segment = pci_domain_nr(pdev->bus); 679 fill->devices[fill->cur].bus = pdev->bus->number; 680 fill->devices[fill->cur].devfn = pdev->devfn; 681 fill->cur++; 682 iommu_group_put(iommu_group); 683 return 0; 684} 685 686struct vfio_pci_group_entry { 687 struct vfio_group *group; 688 int id; 689}; 690 691struct vfio_pci_group_info { 692 int count; 693 struct vfio_pci_group_entry *groups; 694}; 695 696static int vfio_pci_validate_devs(struct pci_dev *pdev, void *data) 697{ 698 struct vfio_pci_group_info *info = data; 699 struct iommu_group *group; 700 int id, i; 701 702 group = iommu_group_get(&pdev->dev); 703 if (!group) 704 return -EPERM; 705 706 id = iommu_group_id(group); 707 708 for (i = 0; i < info->count; i++) 709 if (info->groups[i].id == id) 710 break; 711 712 iommu_group_put(group); 713 714 return (i == info->count) ? -EINVAL : 0; 715} 716 717static bool vfio_pci_dev_below_slot(struct pci_dev *pdev, struct pci_slot *slot) 718{ 719 for (; pdev; pdev = pdev->bus->self) 720 if (pdev->bus == slot->bus) 721 return (pdev->slot == slot); 722 return false; 723} 724 725struct vfio_pci_walk_info { 726 int (*fn)(struct pci_dev *, void *data); 727 void *data; 728 struct pci_dev *pdev; 729 bool slot; 730 int ret; 731}; 732 733static int vfio_pci_walk_wrapper(struct pci_dev *pdev, void *data) 734{ 735 struct vfio_pci_walk_info *walk = data; 736 737 if (!walk->slot || vfio_pci_dev_below_slot(pdev, walk->pdev->slot)) 738 walk->ret = walk->fn(pdev, walk->data); 739 740 return walk->ret; 741} 742 743static int vfio_pci_for_each_slot_or_bus(struct pci_dev *pdev, 744 int (*fn)(struct pci_dev *, 745 void *data), void *data, 746 bool slot) 747{ 748 struct vfio_pci_walk_info walk = { 749 .fn = fn, .data = data, .pdev = pdev, .slot = slot, .ret = 0, 750 }; 751 752 pci_walk_bus(pdev->bus, vfio_pci_walk_wrapper, &walk); 753 754 return walk.ret; 755} 756 757static int msix_mmappable_cap(struct vfio_pci_device *vdev, 758 struct vfio_info_cap *caps) 759{ 760 struct vfio_info_cap_header header = { 761 .id = VFIO_REGION_INFO_CAP_MSIX_MAPPABLE, 762 .version = 1 763 }; 764 765 return vfio_info_add_capability(caps, &header, sizeof(header)); 766} 767 768int vfio_pci_register_dev_region(struct vfio_pci_device *vdev, 769 unsigned int type, unsigned int subtype, 770 const struct vfio_pci_regops *ops, 771 size_t size, u32 flags, void *data) 772{ 773 struct vfio_pci_region *region; 774 775 region = krealloc(vdev->region, 776 (vdev->num_regions + 1) * sizeof(*region), 777 GFP_KERNEL); 778 if (!region) 779 return -ENOMEM; 780 781 vdev->region = region; 782 vdev->region[vdev->num_regions].type = type; 783 vdev->region[vdev->num_regions].subtype = subtype; 784 vdev->region[vdev->num_regions].ops = ops; 785 vdev->region[vdev->num_regions].size = size; 786 vdev->region[vdev->num_regions].flags = flags; 787 vdev->region[vdev->num_regions].data = data; 788 789 vdev->num_regions++; 790 791 return 0; 792} 793 794struct vfio_devices { 795 struct vfio_device **devices; 796 int cur_index; 797 int max_index; 798}; 799 800static long vfio_pci_ioctl(void *device_data, 801 unsigned int cmd, unsigned long arg) 802{ 803 struct vfio_pci_device *vdev = device_data; 804 unsigned long minsz; 805 806 if (cmd == VFIO_DEVICE_GET_INFO) { 807 struct vfio_device_info info; 808 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 809 unsigned long capsz; 810 811 minsz = offsetofend(struct vfio_device_info, num_irqs); 812 813 /* For backward compatibility, cannot require this */ 814 capsz = offsetofend(struct vfio_iommu_type1_info, cap_offset); 815 816 if (copy_from_user(&info, (void __user *)arg, minsz)) 817 return -EFAULT; 818 819 if (info.argsz < minsz) 820 return -EINVAL; 821 822 if (info.argsz >= capsz) { 823 minsz = capsz; 824 info.cap_offset = 0; 825 } 826 827 info.flags = VFIO_DEVICE_FLAGS_PCI; 828 829 if (vdev->reset_works) 830 info.flags |= VFIO_DEVICE_FLAGS_RESET; 831 832 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions; 833 info.num_irqs = VFIO_PCI_NUM_IRQS; 834 835 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV)) { 836 int ret = vfio_pci_info_zdev_add_caps(vdev, &caps); 837 838 if (ret && ret != -ENODEV) { 839 pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n"); 840 return ret; 841 } 842 } 843 844 if (caps.size) { 845 info.flags |= VFIO_DEVICE_FLAGS_CAPS; 846 if (info.argsz < sizeof(info) + caps.size) { 847 info.argsz = sizeof(info) + caps.size; 848 } else { 849 vfio_info_cap_shift(&caps, sizeof(info)); 850 if (copy_to_user((void __user *)arg + 851 sizeof(info), caps.buf, 852 caps.size)) { 853 kfree(caps.buf); 854 return -EFAULT; 855 } 856 info.cap_offset = sizeof(info); 857 } 858 859 kfree(caps.buf); 860 } 861 862 return copy_to_user((void __user *)arg, &info, minsz) ? 863 -EFAULT : 0; 864 865 } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { 866 struct pci_dev *pdev = vdev->pdev; 867 struct vfio_region_info info; 868 struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; 869 int i, ret; 870 871 minsz = offsetofend(struct vfio_region_info, offset); 872 873 if (copy_from_user(&info, (void __user *)arg, minsz)) 874 return -EFAULT; 875 876 if (info.argsz < minsz) 877 return -EINVAL; 878 879 switch (info.index) { 880 case VFIO_PCI_CONFIG_REGION_INDEX: 881 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 882 info.size = pdev->cfg_size; 883 info.flags = VFIO_REGION_INFO_FLAG_READ | 884 VFIO_REGION_INFO_FLAG_WRITE; 885 break; 886 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 887 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 888 info.size = pci_resource_len(pdev, info.index); 889 if (!info.size) { 890 info.flags = 0; 891 break; 892 } 893 894 info.flags = VFIO_REGION_INFO_FLAG_READ | 895 VFIO_REGION_INFO_FLAG_WRITE; 896 if (vdev->bar_mmap_supported[info.index]) { 897 info.flags |= VFIO_REGION_INFO_FLAG_MMAP; 898 if (info.index == vdev->msix_bar) { 899 ret = msix_mmappable_cap(vdev, &caps); 900 if (ret) 901 return ret; 902 } 903 } 904 905 break; 906 case VFIO_PCI_ROM_REGION_INDEX: 907 { 908 void __iomem *io; 909 size_t size; 910 u16 cmd; 911 912 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 913 info.flags = 0; 914 915 /* Report the BAR size, not the ROM size */ 916 info.size = pci_resource_len(pdev, info.index); 917 if (!info.size) { 918 /* Shadow ROMs appear as PCI option ROMs */ 919 if (pdev->resource[PCI_ROM_RESOURCE].flags & 920 IORESOURCE_ROM_SHADOW) 921 info.size = 0x20000; 922 else 923 break; 924 } 925 926 /* 927 * Is it really there? Enable memory decode for 928 * implicit access in pci_map_rom(). 929 */ 930 cmd = vfio_pci_memory_lock_and_enable(vdev); 931 io = pci_map_rom(pdev, &size); 932 if (io) { 933 info.flags = VFIO_REGION_INFO_FLAG_READ; 934 pci_unmap_rom(pdev, io); 935 } else { 936 info.size = 0; 937 } 938 vfio_pci_memory_unlock_and_restore(vdev, cmd); 939 940 break; 941 } 942 case VFIO_PCI_VGA_REGION_INDEX: 943 if (!vdev->has_vga) 944 return -EINVAL; 945 946 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 947 info.size = 0xc0000; 948 info.flags = VFIO_REGION_INFO_FLAG_READ | 949 VFIO_REGION_INFO_FLAG_WRITE; 950 951 break; 952 default: 953 { 954 struct vfio_region_info_cap_type cap_type = { 955 .header.id = VFIO_REGION_INFO_CAP_TYPE, 956 .header.version = 1 }; 957 958 if (info.index >= 959 VFIO_PCI_NUM_REGIONS + vdev->num_regions) 960 return -EINVAL; 961 info.index = array_index_nospec(info.index, 962 VFIO_PCI_NUM_REGIONS + 963 vdev->num_regions); 964 965 i = info.index - VFIO_PCI_NUM_REGIONS; 966 967 info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index); 968 info.size = vdev->region[i].size; 969 info.flags = vdev->region[i].flags; 970 971 cap_type.type = vdev->region[i].type; 972 cap_type.subtype = vdev->region[i].subtype; 973 974 ret = vfio_info_add_capability(&caps, &cap_type.header, 975 sizeof(cap_type)); 976 if (ret) 977 return ret; 978 979 if (vdev->region[i].ops->add_capability) { 980 ret = vdev->region[i].ops->add_capability(vdev, 981 &vdev->region[i], &caps); 982 if (ret) 983 return ret; 984 } 985 } 986 } 987 988 if (caps.size) { 989 info.flags |= VFIO_REGION_INFO_FLAG_CAPS; 990 if (info.argsz < sizeof(info) + caps.size) { 991 info.argsz = sizeof(info) + caps.size; 992 info.cap_offset = 0; 993 } else { 994 vfio_info_cap_shift(&caps, sizeof(info)); 995 if (copy_to_user((void __user *)arg + 996 sizeof(info), caps.buf, 997 caps.size)) { 998 kfree(caps.buf); 999 return -EFAULT; 1000 } 1001 info.cap_offset = sizeof(info); 1002 } 1003 1004 kfree(caps.buf); 1005 } 1006 1007 return copy_to_user((void __user *)arg, &info, minsz) ? 1008 -EFAULT : 0; 1009 1010 } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { 1011 struct vfio_irq_info info; 1012 1013 minsz = offsetofend(struct vfio_irq_info, count); 1014 1015 if (copy_from_user(&info, (void __user *)arg, minsz)) 1016 return -EFAULT; 1017 1018 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS) 1019 return -EINVAL; 1020 1021 switch (info.index) { 1022 case VFIO_PCI_INTX_IRQ_INDEX ... VFIO_PCI_MSIX_IRQ_INDEX: 1023 case VFIO_PCI_REQ_IRQ_INDEX: 1024 break; 1025 case VFIO_PCI_ERR_IRQ_INDEX: 1026 if (pci_is_pcie(vdev->pdev)) 1027 break; 1028 fallthrough; 1029 default: 1030 return -EINVAL; 1031 } 1032 1033 info.flags = VFIO_IRQ_INFO_EVENTFD; 1034 1035 info.count = vfio_pci_get_irq_count(vdev, info.index); 1036 1037 if (info.index == VFIO_PCI_INTX_IRQ_INDEX) 1038 info.flags |= (VFIO_IRQ_INFO_MASKABLE | 1039 VFIO_IRQ_INFO_AUTOMASKED); 1040 else 1041 info.flags |= VFIO_IRQ_INFO_NORESIZE; 1042 1043 return copy_to_user((void __user *)arg, &info, minsz) ? 1044 -EFAULT : 0; 1045 1046 } else if (cmd == VFIO_DEVICE_SET_IRQS) { 1047 struct vfio_irq_set hdr; 1048 u8 *data = NULL; 1049 int max, ret = 0; 1050 size_t data_size = 0; 1051 1052 minsz = offsetofend(struct vfio_irq_set, count); 1053 1054 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1055 return -EFAULT; 1056 1057 max = vfio_pci_get_irq_count(vdev, hdr.index); 1058 1059 ret = vfio_set_irqs_validate_and_prepare(&hdr, max, 1060 VFIO_PCI_NUM_IRQS, &data_size); 1061 if (ret) 1062 return ret; 1063 1064 if (data_size) { 1065 data = memdup_user((void __user *)(arg + minsz), 1066 data_size); 1067 if (IS_ERR(data)) 1068 return PTR_ERR(data); 1069 } 1070 1071 mutex_lock(&vdev->igate); 1072 1073 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index, 1074 hdr.start, hdr.count, data); 1075 1076 mutex_unlock(&vdev->igate); 1077 kfree(data); 1078 1079 return ret; 1080 1081 } else if (cmd == VFIO_DEVICE_RESET) { 1082 int ret; 1083 1084 if (!vdev->reset_works) 1085 return -EINVAL; 1086 1087 vfio_pci_zap_and_down_write_memory_lock(vdev); 1088 ret = pci_try_reset_function(vdev->pdev); 1089 up_write(&vdev->memory_lock); 1090 1091 return ret; 1092 1093 } else if (cmd == VFIO_DEVICE_GET_PCI_HOT_RESET_INFO) { 1094 struct vfio_pci_hot_reset_info hdr; 1095 struct vfio_pci_fill_info fill = { 0 }; 1096 struct vfio_pci_dependent_device *devices = NULL; 1097 bool slot = false; 1098 int ret = 0; 1099 1100 minsz = offsetofend(struct vfio_pci_hot_reset_info, count); 1101 1102 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1103 return -EFAULT; 1104 1105 if (hdr.argsz < minsz) 1106 return -EINVAL; 1107 1108 hdr.flags = 0; 1109 1110 /* Can we do a slot or bus reset or neither? */ 1111 if (!pci_probe_reset_slot(vdev->pdev->slot)) 1112 slot = true; 1113 else if (pci_probe_reset_bus(vdev->pdev->bus)) 1114 return -ENODEV; 1115 1116 /* How many devices are affected? */ 1117 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1118 vfio_pci_count_devs, 1119 &fill.max, slot); 1120 if (ret) 1121 return ret; 1122 1123 WARN_ON(!fill.max); /* Should always be at least one */ 1124 1125 /* 1126 * If there's enough space, fill it now, otherwise return 1127 * -ENOSPC and the number of devices affected. 1128 */ 1129 if (hdr.argsz < sizeof(hdr) + (fill.max * sizeof(*devices))) { 1130 ret = -ENOSPC; 1131 hdr.count = fill.max; 1132 goto reset_info_exit; 1133 } 1134 1135 devices = kcalloc(fill.max, sizeof(*devices), GFP_KERNEL); 1136 if (!devices) 1137 return -ENOMEM; 1138 1139 fill.devices = devices; 1140 1141 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1142 vfio_pci_fill_devs, 1143 &fill, slot); 1144 1145 /* 1146 * If a device was removed between counting and filling, 1147 * we may come up short of fill.max. If a device was 1148 * added, we'll have a return of -EAGAIN above. 1149 */ 1150 if (!ret) 1151 hdr.count = fill.cur; 1152 1153reset_info_exit: 1154 if (copy_to_user((void __user *)arg, &hdr, minsz)) 1155 ret = -EFAULT; 1156 1157 if (!ret) { 1158 if (copy_to_user((void __user *)(arg + minsz), devices, 1159 hdr.count * sizeof(*devices))) 1160 ret = -EFAULT; 1161 } 1162 1163 kfree(devices); 1164 return ret; 1165 1166 } else if (cmd == VFIO_DEVICE_PCI_HOT_RESET) { 1167 struct vfio_pci_hot_reset hdr; 1168 int32_t *group_fds; 1169 struct vfio_pci_group_entry *groups; 1170 struct vfio_pci_group_info info; 1171 struct vfio_devices devs = { .cur_index = 0 }; 1172 bool slot = false; 1173 int i, group_idx, mem_idx = 0, count = 0, ret = 0; 1174 1175 minsz = offsetofend(struct vfio_pci_hot_reset, count); 1176 1177 if (copy_from_user(&hdr, (void __user *)arg, minsz)) 1178 return -EFAULT; 1179 1180 if (hdr.argsz < minsz || hdr.flags) 1181 return -EINVAL; 1182 1183 /* Can we do a slot or bus reset or neither? */ 1184 if (!pci_probe_reset_slot(vdev->pdev->slot)) 1185 slot = true; 1186 else if (pci_probe_reset_bus(vdev->pdev->bus)) 1187 return -ENODEV; 1188 1189 /* 1190 * We can't let userspace give us an arbitrarily large 1191 * buffer to copy, so verify how many we think there 1192 * could be. Note groups can have multiple devices so 1193 * one group per device is the max. 1194 */ 1195 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1196 vfio_pci_count_devs, 1197 &count, slot); 1198 if (ret) 1199 return ret; 1200 1201 /* Somewhere between 1 and count is OK */ 1202 if (!hdr.count || hdr.count > count) 1203 return -EINVAL; 1204 1205 group_fds = kcalloc(hdr.count, sizeof(*group_fds), GFP_KERNEL); 1206 groups = kcalloc(hdr.count, sizeof(*groups), GFP_KERNEL); 1207 if (!group_fds || !groups) { 1208 kfree(group_fds); 1209 kfree(groups); 1210 return -ENOMEM; 1211 } 1212 1213 if (copy_from_user(group_fds, (void __user *)(arg + minsz), 1214 hdr.count * sizeof(*group_fds))) { 1215 kfree(group_fds); 1216 kfree(groups); 1217 return -EFAULT; 1218 } 1219 1220 /* 1221 * For each group_fd, get the group through the vfio external 1222 * user interface and store the group and iommu ID. This 1223 * ensures the group is held across the reset. 1224 */ 1225 for (group_idx = 0; group_idx < hdr.count; group_idx++) { 1226 struct vfio_group *group; 1227 struct fd f = fdget(group_fds[group_idx]); 1228 if (!f.file) { 1229 ret = -EBADF; 1230 break; 1231 } 1232 1233 group = vfio_group_get_external_user(f.file); 1234 fdput(f); 1235 if (IS_ERR(group)) { 1236 ret = PTR_ERR(group); 1237 break; 1238 } 1239 1240 groups[group_idx].group = group; 1241 groups[group_idx].id = 1242 vfio_external_user_iommu_id(group); 1243 } 1244 1245 kfree(group_fds); 1246 1247 /* release reference to groups on error */ 1248 if (ret) 1249 goto hot_reset_release; 1250 1251 info.count = hdr.count; 1252 info.groups = groups; 1253 1254 /* 1255 * Test whether all the affected devices are contained 1256 * by the set of groups provided by the user. 1257 */ 1258 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1259 vfio_pci_validate_devs, 1260 &info, slot); 1261 if (ret) 1262 goto hot_reset_release; 1263 1264 devs.max_index = count; 1265 devs.devices = kcalloc(count, sizeof(struct vfio_device *), 1266 GFP_KERNEL); 1267 if (!devs.devices) { 1268 ret = -ENOMEM; 1269 goto hot_reset_release; 1270 } 1271 1272 /* 1273 * We need to get memory_lock for each device, but devices 1274 * can share mmap_lock, therefore we need to zap and hold 1275 * the vma_lock for each device, and only then get each 1276 * memory_lock. 1277 */ 1278 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev, 1279 vfio_pci_try_zap_and_vma_lock_cb, 1280 &devs, slot); 1281 if (ret) 1282 goto hot_reset_release; 1283 1284 for (; mem_idx < devs.cur_index; mem_idx++) { 1285 struct vfio_pci_device *tmp; 1286 1287 tmp = vfio_device_data(devs.devices[mem_idx]); 1288 1289 ret = down_write_trylock(&tmp->memory_lock); 1290 if (!ret) { 1291 ret = -EBUSY; 1292 goto hot_reset_release; 1293 } 1294 mutex_unlock(&tmp->vma_lock); 1295 } 1296 1297 /* User has access, do the reset */ 1298 ret = pci_reset_bus(vdev->pdev); 1299 1300hot_reset_release: 1301 for (i = 0; i < devs.cur_index; i++) { 1302 struct vfio_device *device; 1303 struct vfio_pci_device *tmp; 1304 1305 device = devs.devices[i]; 1306 tmp = vfio_device_data(device); 1307 1308 if (i < mem_idx) 1309 up_write(&tmp->memory_lock); 1310 else 1311 mutex_unlock(&tmp->vma_lock); 1312 vfio_device_put(device); 1313 } 1314 kfree(devs.devices); 1315 1316 for (group_idx--; group_idx >= 0; group_idx--) 1317 vfio_group_put_external_user(groups[group_idx].group); 1318 1319 kfree(groups); 1320 return ret; 1321 } else if (cmd == VFIO_DEVICE_IOEVENTFD) { 1322 struct vfio_device_ioeventfd ioeventfd; 1323 int count; 1324 1325 minsz = offsetofend(struct vfio_device_ioeventfd, fd); 1326 1327 if (copy_from_user(&ioeventfd, (void __user *)arg, minsz)) 1328 return -EFAULT; 1329 1330 if (ioeventfd.argsz < minsz) 1331 return -EINVAL; 1332 1333 if (ioeventfd.flags & ~VFIO_DEVICE_IOEVENTFD_SIZE_MASK) 1334 return -EINVAL; 1335 1336 count = ioeventfd.flags & VFIO_DEVICE_IOEVENTFD_SIZE_MASK; 1337 1338 if (hweight8(count) != 1 || ioeventfd.fd < -1) 1339 return -EINVAL; 1340 1341 return vfio_pci_ioeventfd(vdev, ioeventfd.offset, 1342 ioeventfd.data, count, ioeventfd.fd); 1343 } else if (cmd == VFIO_DEVICE_FEATURE) { 1344 struct vfio_device_feature feature; 1345 uuid_t uuid; 1346 1347 minsz = offsetofend(struct vfio_device_feature, flags); 1348 1349 if (copy_from_user(&feature, (void __user *)arg, minsz)) 1350 return -EFAULT; 1351 1352 if (feature.argsz < minsz) 1353 return -EINVAL; 1354 1355 /* Check unknown flags */ 1356 if (feature.flags & ~(VFIO_DEVICE_FEATURE_MASK | 1357 VFIO_DEVICE_FEATURE_SET | 1358 VFIO_DEVICE_FEATURE_GET | 1359 VFIO_DEVICE_FEATURE_PROBE)) 1360 return -EINVAL; 1361 1362 /* GET & SET are mutually exclusive except with PROBE */ 1363 if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) && 1364 (feature.flags & VFIO_DEVICE_FEATURE_SET) && 1365 (feature.flags & VFIO_DEVICE_FEATURE_GET)) 1366 return -EINVAL; 1367 1368 switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) { 1369 case VFIO_DEVICE_FEATURE_PCI_VF_TOKEN: 1370 if (!vdev->vf_token) 1371 return -ENOTTY; 1372 1373 /* 1374 * We do not support GET of the VF Token UUID as this 1375 * could expose the token of the previous device user. 1376 */ 1377 if (feature.flags & VFIO_DEVICE_FEATURE_GET) 1378 return -EINVAL; 1379 1380 if (feature.flags & VFIO_DEVICE_FEATURE_PROBE) 1381 return 0; 1382 1383 /* Don't SET unless told to do so */ 1384 if (!(feature.flags & VFIO_DEVICE_FEATURE_SET)) 1385 return -EINVAL; 1386 1387 if (feature.argsz < minsz + sizeof(uuid)) 1388 return -EINVAL; 1389 1390 if (copy_from_user(&uuid, (void __user *)(arg + minsz), 1391 sizeof(uuid))) 1392 return -EFAULT; 1393 1394 mutex_lock(&vdev->vf_token->lock); 1395 uuid_copy(&vdev->vf_token->uuid, &uuid); 1396 mutex_unlock(&vdev->vf_token->lock); 1397 1398 return 0; 1399 default: 1400 return -ENOTTY; 1401 } 1402 } 1403 1404 return -ENOTTY; 1405} 1406 1407static ssize_t vfio_pci_rw(void *device_data, char __user *buf, 1408 size_t count, loff_t *ppos, bool iswrite) 1409{ 1410 unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos); 1411 struct vfio_pci_device *vdev = device_data; 1412 1413 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1414 return -EINVAL; 1415 1416 switch (index) { 1417 case VFIO_PCI_CONFIG_REGION_INDEX: 1418 return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite); 1419 1420 case VFIO_PCI_ROM_REGION_INDEX: 1421 if (iswrite) 1422 return -EINVAL; 1423 return vfio_pci_bar_rw(vdev, buf, count, ppos, false); 1424 1425 case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX: 1426 return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite); 1427 1428 case VFIO_PCI_VGA_REGION_INDEX: 1429 return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite); 1430 default: 1431 index -= VFIO_PCI_NUM_REGIONS; 1432 return vdev->region[index].ops->rw(vdev, buf, 1433 count, ppos, iswrite); 1434 } 1435 1436 return -EINVAL; 1437} 1438 1439static ssize_t vfio_pci_read(void *device_data, char __user *buf, 1440 size_t count, loff_t *ppos) 1441{ 1442 if (!count) 1443 return 0; 1444 1445 return vfio_pci_rw(device_data, buf, count, ppos, false); 1446} 1447 1448static ssize_t vfio_pci_write(void *device_data, const char __user *buf, 1449 size_t count, loff_t *ppos) 1450{ 1451 if (!count) 1452 return 0; 1453 1454 return vfio_pci_rw(device_data, (char __user *)buf, count, ppos, true); 1455} 1456 1457/* Return 1 on zap and vma_lock acquired, 0 on contention (only with @try) */ 1458static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try) 1459{ 1460 struct vfio_pci_mmap_vma *mmap_vma, *tmp; 1461 1462 /* 1463 * Lock ordering: 1464 * vma_lock is nested under mmap_lock for vm_ops callback paths. 1465 * The memory_lock semaphore is used by both code paths calling 1466 * into this function to zap vmas and the vm_ops.fault callback 1467 * to protect the memory enable state of the device. 1468 * 1469 * When zapping vmas we need to maintain the mmap_lock => vma_lock 1470 * ordering, which requires using vma_lock to walk vma_list to 1471 * acquire an mm, then dropping vma_lock to get the mmap_lock and 1472 * reacquiring vma_lock. This logic is derived from similar 1473 * requirements in uverbs_user_mmap_disassociate(). 1474 * 1475 * mmap_lock must always be the top-level lock when it is taken. 1476 * Therefore we can only hold the memory_lock write lock when 1477 * vma_list is empty, as we'd need to take mmap_lock to clear 1478 * entries. vma_list can only be guaranteed empty when holding 1479 * vma_lock, thus memory_lock is nested under vma_lock. 1480 * 1481 * This enables the vm_ops.fault callback to acquire vma_lock, 1482 * followed by memory_lock read lock, while already holding 1483 * mmap_lock without risk of deadlock. 1484 */ 1485 while (1) { 1486 struct mm_struct *mm = NULL; 1487 1488 if (try) { 1489 if (!mutex_trylock(&vdev->vma_lock)) 1490 return 0; 1491 } else { 1492 mutex_lock(&vdev->vma_lock); 1493 } 1494 while (!list_empty(&vdev->vma_list)) { 1495 mmap_vma = list_first_entry(&vdev->vma_list, 1496 struct vfio_pci_mmap_vma, 1497 vma_next); 1498 mm = mmap_vma->vma->vm_mm; 1499 if (mmget_not_zero(mm)) 1500 break; 1501 1502 list_del(&mmap_vma->vma_next); 1503 kfree(mmap_vma); 1504 mm = NULL; 1505 } 1506 if (!mm) 1507 return 1; 1508 mutex_unlock(&vdev->vma_lock); 1509 1510 if (try) { 1511 if (!mmap_read_trylock(mm)) { 1512 mmput(mm); 1513 return 0; 1514 } 1515 } else { 1516 mmap_read_lock(mm); 1517 } 1518 if (try) { 1519 if (!mutex_trylock(&vdev->vma_lock)) { 1520 mmap_read_unlock(mm); 1521 mmput(mm); 1522 return 0; 1523 } 1524 } else { 1525 mutex_lock(&vdev->vma_lock); 1526 } 1527 list_for_each_entry_safe(mmap_vma, tmp, 1528 &vdev->vma_list, vma_next) { 1529 struct vm_area_struct *vma = mmap_vma->vma; 1530 1531 if (vma->vm_mm != mm) 1532 continue; 1533 1534 list_del(&mmap_vma->vma_next); 1535 kfree(mmap_vma); 1536 1537 zap_vma_ptes(vma, vma->vm_start, 1538 vma->vm_end - vma->vm_start); 1539 } 1540 mutex_unlock(&vdev->vma_lock); 1541 mmap_read_unlock(mm); 1542 mmput(mm); 1543 } 1544} 1545 1546void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device *vdev) 1547{ 1548 vfio_pci_zap_and_vma_lock(vdev, false); 1549 down_write(&vdev->memory_lock); 1550 mutex_unlock(&vdev->vma_lock); 1551} 1552 1553u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev) 1554{ 1555 u16 cmd; 1556 1557 down_write(&vdev->memory_lock); 1558 pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd); 1559 if (!(cmd & PCI_COMMAND_MEMORY)) 1560 pci_write_config_word(vdev->pdev, PCI_COMMAND, 1561 cmd | PCI_COMMAND_MEMORY); 1562 1563 return cmd; 1564} 1565 1566void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev, u16 cmd) 1567{ 1568 pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd); 1569 up_write(&vdev->memory_lock); 1570} 1571 1572/* Caller holds vma_lock */ 1573static int __vfio_pci_add_vma(struct vfio_pci_device *vdev, 1574 struct vm_area_struct *vma) 1575{ 1576 struct vfio_pci_mmap_vma *mmap_vma; 1577 1578 mmap_vma = kmalloc(sizeof(*mmap_vma), GFP_KERNEL); 1579 if (!mmap_vma) 1580 return -ENOMEM; 1581 1582 mmap_vma->vma = vma; 1583 list_add(&mmap_vma->vma_next, &vdev->vma_list); 1584 1585 return 0; 1586} 1587 1588/* 1589 * Zap mmaps on open so that we can fault them in on access and therefore 1590 * our vma_list only tracks mappings accessed since last zap. 1591 */ 1592static void vfio_pci_mmap_open(struct vm_area_struct *vma) 1593{ 1594 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1595} 1596 1597static void vfio_pci_mmap_close(struct vm_area_struct *vma) 1598{ 1599 struct vfio_pci_device *vdev = vma->vm_private_data; 1600 struct vfio_pci_mmap_vma *mmap_vma; 1601 1602 mutex_lock(&vdev->vma_lock); 1603 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) { 1604 if (mmap_vma->vma == vma) { 1605 list_del(&mmap_vma->vma_next); 1606 kfree(mmap_vma); 1607 break; 1608 } 1609 } 1610 mutex_unlock(&vdev->vma_lock); 1611} 1612 1613static vm_fault_t vfio_pci_mmap_fault(struct vm_fault *vmf) 1614{ 1615 struct vm_area_struct *vma = vmf->vma; 1616 struct vfio_pci_device *vdev = vma->vm_private_data; 1617 struct vfio_pci_mmap_vma *mmap_vma; 1618 vm_fault_t ret = VM_FAULT_NOPAGE; 1619 1620 mutex_lock(&vdev->vma_lock); 1621 down_read(&vdev->memory_lock); 1622 1623 if (!__vfio_pci_memory_enabled(vdev)) { 1624 ret = VM_FAULT_SIGBUS; 1625 goto up_out; 1626 } 1627 1628 /* 1629 * We populate the whole vma on fault, so we need to test whether 1630 * the vma has already been mapped, such as for concurrent faults 1631 * to the same vma. io_remap_pfn_range() will trigger a BUG_ON if 1632 * we ask it to fill the same range again. 1633 */ 1634 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) { 1635 if (mmap_vma->vma == vma) 1636 goto up_out; 1637 } 1638 1639 if (io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, 1640 vma->vm_end - vma->vm_start, 1641 vma->vm_page_prot)) { 1642 ret = VM_FAULT_SIGBUS; 1643 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1644 goto up_out; 1645 } 1646 1647 if (__vfio_pci_add_vma(vdev, vma)) { 1648 ret = VM_FAULT_OOM; 1649 zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start); 1650 } 1651 1652up_out: 1653 up_read(&vdev->memory_lock); 1654 mutex_unlock(&vdev->vma_lock); 1655 return ret; 1656} 1657 1658static const struct vm_operations_struct vfio_pci_mmap_ops = { 1659 .open = vfio_pci_mmap_open, 1660 .close = vfio_pci_mmap_close, 1661 .fault = vfio_pci_mmap_fault, 1662}; 1663 1664static int vfio_pci_mmap(void *device_data, struct vm_area_struct *vma) 1665{ 1666 struct vfio_pci_device *vdev = device_data; 1667 struct pci_dev *pdev = vdev->pdev; 1668 unsigned int index; 1669 u64 phys_len, req_len, pgoff, req_start; 1670 int ret; 1671 1672 index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT); 1673 1674 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions) 1675 return -EINVAL; 1676 if (vma->vm_end < vma->vm_start) 1677 return -EINVAL; 1678 if ((vma->vm_flags & VM_SHARED) == 0) 1679 return -EINVAL; 1680 if (index >= VFIO_PCI_NUM_REGIONS) { 1681 int regnum = index - VFIO_PCI_NUM_REGIONS; 1682 struct vfio_pci_region *region = vdev->region + regnum; 1683 1684 if (region->ops && region->ops->mmap && 1685 (region->flags & VFIO_REGION_INFO_FLAG_MMAP)) 1686 return region->ops->mmap(vdev, region, vma); 1687 return -EINVAL; 1688 } 1689 if (index >= VFIO_PCI_ROM_REGION_INDEX) 1690 return -EINVAL; 1691 if (!vdev->bar_mmap_supported[index]) 1692 return -EINVAL; 1693 1694 phys_len = PAGE_ALIGN(pci_resource_len(pdev, index)); 1695 req_len = vma->vm_end - vma->vm_start; 1696 pgoff = vma->vm_pgoff & 1697 ((1U << (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT)) - 1); 1698 req_start = pgoff << PAGE_SHIFT; 1699 1700 if (req_start + req_len > phys_len) 1701 return -EINVAL; 1702 1703 /* 1704 * Even though we don't make use of the barmap for the mmap, 1705 * we need to request the region and the barmap tracks that. 1706 */ 1707 if (!vdev->barmap[index]) { 1708 ret = pci_request_selected_regions(pdev, 1709 1 << index, "vfio-pci"); 1710 if (ret) 1711 return ret; 1712 1713 vdev->barmap[index] = pci_iomap(pdev, index, 0); 1714 if (!vdev->barmap[index]) { 1715 pci_release_selected_regions(pdev, 1 << index); 1716 return -ENOMEM; 1717 } 1718 } 1719 1720 vma->vm_private_data = vdev; 1721 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); 1722 vma->vm_pgoff = (pci_resource_start(pdev, index) >> PAGE_SHIFT) + pgoff; 1723 1724 /* 1725 * See remap_pfn_range(), called from vfio_pci_fault() but we can't 1726 * change vm_flags within the fault handler. Set them now. 1727 */ 1728 vma->vm_flags |= VM_IO | VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP; 1729 vma->vm_ops = &vfio_pci_mmap_ops; 1730 1731 return 0; 1732} 1733 1734static void vfio_pci_request(void *device_data, unsigned int count) 1735{ 1736 struct vfio_pci_device *vdev = device_data; 1737 struct pci_dev *pdev = vdev->pdev; 1738 1739 mutex_lock(&vdev->igate); 1740 1741 if (vdev->req_trigger) { 1742 if (!(count % 10)) 1743 pci_notice_ratelimited(pdev, 1744 "Relaying device request to user (#%u)\n", 1745 count); 1746 eventfd_signal(vdev->req_trigger, 1); 1747 } else if (count == 0) { 1748 pci_warn(pdev, 1749 "No device request channel registered, blocked until released by user\n"); 1750 } 1751 1752 mutex_unlock(&vdev->igate); 1753} 1754 1755static int vfio_pci_validate_vf_token(struct vfio_pci_device *vdev, 1756 bool vf_token, uuid_t *uuid) 1757{ 1758 /* 1759 * There's always some degree of trust or collaboration between SR-IOV 1760 * PF and VFs, even if just that the PF hosts the SR-IOV capability and 1761 * can disrupt VFs with a reset, but often the PF has more explicit 1762 * access to deny service to the VF or access data passed through the 1763 * VF. We therefore require an opt-in via a shared VF token (UUID) to 1764 * represent this trust. This both prevents that a VF driver might 1765 * assume the PF driver is a trusted, in-kernel driver, and also that 1766 * a PF driver might be replaced with a rogue driver, unknown to in-use 1767 * VF drivers. 1768 * 1769 * Therefore when presented with a VF, if the PF is a vfio device and 1770 * it is bound to the vfio-pci driver, the user needs to provide a VF 1771 * token to access the device, in the form of appending a vf_token to 1772 * the device name, for example: 1773 * 1774 * "0000:04:10.0 vf_token=bd8d9d2b-5a5f-4f5a-a211-f591514ba1f3" 1775 * 1776 * When presented with a PF which has VFs in use, the user must also 1777 * provide the current VF token to prove collaboration with existing 1778 * VF users. If VFs are not in use, the VF token provided for the PF 1779 * device will act to set the VF token. 1780 * 1781 * If the VF token is provided but unused, an error is generated. 1782 */ 1783 if (!vdev->pdev->is_virtfn && !vdev->vf_token && !vf_token) 1784 return 0; /* No VF token provided or required */ 1785 1786 if (vdev->pdev->is_virtfn) { 1787 struct vfio_device *pf_dev; 1788 struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev); 1789 bool match; 1790 1791 if (!pf_vdev) { 1792 if (!vf_token) 1793 return 0; /* PF is not vfio-pci, no VF token */ 1794 1795 pci_info_ratelimited(vdev->pdev, 1796 "VF token incorrectly provided, PF not bound to vfio-pci\n"); 1797 return -EINVAL; 1798 } 1799 1800 if (!vf_token) { 1801 vfio_device_put(pf_dev); 1802 pci_info_ratelimited(vdev->pdev, 1803 "VF token required to access device\n"); 1804 return -EACCES; 1805 } 1806 1807 mutex_lock(&pf_vdev->vf_token->lock); 1808 match = uuid_equal(uuid, &pf_vdev->vf_token->uuid); 1809 mutex_unlock(&pf_vdev->vf_token->lock); 1810 1811 vfio_device_put(pf_dev); 1812 1813 if (!match) { 1814 pci_info_ratelimited(vdev->pdev, 1815 "Incorrect VF token provided for device\n"); 1816 return -EACCES; 1817 } 1818 } else if (vdev->vf_token) { 1819 mutex_lock(&vdev->vf_token->lock); 1820 if (vdev->vf_token->users) { 1821 if (!vf_token) { 1822 mutex_unlock(&vdev->vf_token->lock); 1823 pci_info_ratelimited(vdev->pdev, 1824 "VF token required to access device\n"); 1825 return -EACCES; 1826 } 1827 1828 if (!uuid_equal(uuid, &vdev->vf_token->uuid)) { 1829 mutex_unlock(&vdev->vf_token->lock); 1830 pci_info_ratelimited(vdev->pdev, 1831 "Incorrect VF token provided for device\n"); 1832 return -EACCES; 1833 } 1834 } else if (vf_token) { 1835 uuid_copy(&vdev->vf_token->uuid, uuid); 1836 } 1837 1838 mutex_unlock(&vdev->vf_token->lock); 1839 } else if (vf_token) { 1840 pci_info_ratelimited(vdev->pdev, 1841 "VF token incorrectly provided, not a PF or VF\n"); 1842 return -EINVAL; 1843 } 1844 1845 return 0; 1846} 1847 1848#define VF_TOKEN_ARG "vf_token=" 1849 1850static int vfio_pci_match(void *device_data, char *buf) 1851{ 1852 struct vfio_pci_device *vdev = device_data; 1853 bool vf_token = false; 1854 uuid_t uuid; 1855 int ret; 1856 1857 if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev)))) 1858 return 0; /* No match */ 1859 1860 if (strlen(buf) > strlen(pci_name(vdev->pdev))) { 1861 buf += strlen(pci_name(vdev->pdev)); 1862 1863 if (*buf != ' ') 1864 return 0; /* No match: non-whitespace after name */ 1865 1866 while (*buf) { 1867 if (*buf == ' ') { 1868 buf++; 1869 continue; 1870 } 1871 1872 if (!vf_token && !strncmp(buf, VF_TOKEN_ARG, 1873 strlen(VF_TOKEN_ARG))) { 1874 buf += strlen(VF_TOKEN_ARG); 1875 1876 if (strlen(buf) < UUID_STRING_LEN) 1877 return -EINVAL; 1878 1879 ret = uuid_parse(buf, &uuid); 1880 if (ret) 1881 return ret; 1882 1883 vf_token = true; 1884 buf += UUID_STRING_LEN; 1885 } else { 1886 /* Unknown/duplicate option */ 1887 return -EINVAL; 1888 } 1889 } 1890 } 1891 1892 ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid); 1893 if (ret) 1894 return ret; 1895 1896 return 1; /* Match */ 1897} 1898 1899static const struct vfio_device_ops vfio_pci_ops = { 1900 .name = "vfio-pci", 1901 .open = vfio_pci_open, 1902 .release = vfio_pci_release, 1903 .ioctl = vfio_pci_ioctl, 1904 .read = vfio_pci_read, 1905 .write = vfio_pci_write, 1906 .mmap = vfio_pci_mmap, 1907 .request = vfio_pci_request, 1908 .match = vfio_pci_match, 1909}; 1910 1911static int vfio_pci_reflck_attach(struct vfio_pci_device *vdev); 1912static void vfio_pci_reflck_put(struct vfio_pci_reflck *reflck); 1913 1914static int vfio_pci_bus_notifier(struct notifier_block *nb, 1915 unsigned long action, void *data) 1916{ 1917 struct vfio_pci_device *vdev = container_of(nb, 1918 struct vfio_pci_device, nb); 1919 struct device *dev = data; 1920 struct pci_dev *pdev = to_pci_dev(dev); 1921 struct pci_dev *physfn = pci_physfn(pdev); 1922 1923 if (action == BUS_NOTIFY_ADD_DEVICE && 1924 pdev->is_virtfn && physfn == vdev->pdev) { 1925 pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n", 1926 pci_name(pdev)); 1927 pdev->driver_override = kasprintf(GFP_KERNEL, "%s", 1928 vfio_pci_ops.name); 1929 } else if (action == BUS_NOTIFY_BOUND_DRIVER && 1930 pdev->is_virtfn && physfn == vdev->pdev) { 1931 struct pci_driver *drv = pci_dev_driver(pdev); 1932 1933 if (drv && drv != &vfio_pci_driver) 1934 pci_warn(vdev->pdev, 1935 "VF %s bound to driver %s while PF bound to vfio-pci\n", 1936 pci_name(pdev), drv->name); 1937 } 1938 1939 return 0; 1940} 1941 1942static int vfio_pci_vf_init(struct vfio_pci_device *vdev) 1943{ 1944 struct pci_dev *pdev = vdev->pdev; 1945 int ret; 1946 1947 if (!pdev->is_physfn) 1948 return 0; 1949 1950 vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL); 1951 if (!vdev->vf_token) 1952 return -ENOMEM; 1953 1954 mutex_init(&vdev->vf_token->lock); 1955 uuid_gen(&vdev->vf_token->uuid); 1956 1957 vdev->nb.notifier_call = vfio_pci_bus_notifier; 1958 ret = bus_register_notifier(&pci_bus_type, &vdev->nb); 1959 if (ret) { 1960 kfree(vdev->vf_token); 1961 return ret; 1962 } 1963 return 0; 1964} 1965 1966static void vfio_pci_vf_uninit(struct vfio_pci_device *vdev) 1967{ 1968 if (!vdev->vf_token) 1969 return; 1970 1971 bus_unregister_notifier(&pci_bus_type, &vdev->nb); 1972 WARN_ON(vdev->vf_token->users); 1973 mutex_destroy(&vdev->vf_token->lock); 1974 kfree(vdev->vf_token); 1975} 1976 1977static int vfio_pci_vga_init(struct vfio_pci_device *vdev) 1978{ 1979 struct pci_dev *pdev = vdev->pdev; 1980 int ret; 1981 1982 if (!vfio_pci_is_vga(pdev)) 1983 return 0; 1984 1985 ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode); 1986 if (ret) 1987 return ret; 1988 vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false)); 1989 return 0; 1990} 1991 1992static void vfio_pci_vga_uninit(struct vfio_pci_device *vdev) 1993{ 1994 struct pci_dev *pdev = vdev->pdev; 1995 1996 if (!vfio_pci_is_vga(pdev)) 1997 return; 1998 vga_client_register(pdev, NULL, NULL, NULL); 1999 vga_set_legacy_decoding(pdev, VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM | 2000 VGA_RSRC_LEGACY_IO | 2001 VGA_RSRC_LEGACY_MEM); 2002} 2003 2004static int vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 2005{ 2006 struct vfio_pci_device *vdev; 2007 struct iommu_group *group; 2008 int ret; 2009 2010 if (vfio_pci_is_denylisted(pdev)) 2011 return -EINVAL; 2012 2013 if (pdev->hdr_type != PCI_HEADER_TYPE_NORMAL) 2014 return -EINVAL; 2015 2016 /* 2017 * Prevent binding to PFs with VFs enabled, the VFs might be in use 2018 * by the host or other users. We cannot capture the VFs if they 2019 * already exist, nor can we track VF users. Disabling SR-IOV here 2020 * would initiate removing the VFs, which would unbind the driver, 2021 * which is prone to blocking if that VF is also in use by vfio-pci. 2022 * Just reject these PFs and let the user sort it out. 2023 */ 2024 if (pci_num_vf(pdev)) { 2025 pci_warn(pdev, "Cannot bind to PF with SR-IOV enabled\n"); 2026 return -EBUSY; 2027 } 2028 2029 group = vfio_iommu_group_get(&pdev->dev); 2030 if (!group) 2031 return -EINVAL; 2032 2033 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL); 2034 if (!vdev) { 2035 ret = -ENOMEM; 2036 goto out_group_put; 2037 } 2038 2039 vdev->pdev = pdev; 2040 vdev->irq_type = VFIO_PCI_NUM_IRQS; 2041 mutex_init(&vdev->igate); 2042 spin_lock_init(&vdev->irqlock); 2043 mutex_init(&vdev->ioeventfds_lock); 2044 INIT_LIST_HEAD(&vdev->dummy_resources_list); 2045 INIT_LIST_HEAD(&vdev->ioeventfds_list); 2046 mutex_init(&vdev->vma_lock); 2047 INIT_LIST_HEAD(&vdev->vma_list); 2048 init_rwsem(&vdev->memory_lock); 2049 2050 ret = vfio_pci_reflck_attach(vdev); 2051 if (ret) 2052 goto out_free; 2053 ret = vfio_pci_vf_init(vdev); 2054 if (ret) 2055 goto out_reflck; 2056 ret = vfio_pci_vga_init(vdev); 2057 if (ret) 2058 goto out_vf; 2059 2060 vfio_pci_probe_power_state(vdev); 2061 2062 if (!disable_idle_d3) { 2063 /* 2064 * pci-core sets the device power state to an unknown value at 2065 * bootup and after being removed from a driver. The only 2066 * transition it allows from this unknown state is to D0, which 2067 * typically happens when a driver calls pci_enable_device(). 2068 * We're not ready to enable the device yet, but we do want to 2069 * be able to get to D3. Therefore first do a D0 transition 2070 * before going to D3. 2071 */ 2072 vfio_pci_set_power_state(vdev, PCI_D0); 2073 vfio_pci_set_power_state(vdev, PCI_D3hot); 2074 } 2075 2076 ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev); 2077 if (ret) 2078 goto out_power; 2079 return 0; 2080 2081out_power: 2082 if (!disable_idle_d3) 2083 vfio_pci_set_power_state(vdev, PCI_D0); 2084out_vf: 2085 vfio_pci_vf_uninit(vdev); 2086out_reflck: 2087 vfio_pci_reflck_put(vdev->reflck); 2088out_free: 2089 kfree(vdev->pm_save); 2090 kfree(vdev); 2091out_group_put: 2092 vfio_iommu_group_put(group, &pdev->dev); 2093 return ret; 2094} 2095 2096static void vfio_pci_remove(struct pci_dev *pdev) 2097{ 2098 struct vfio_pci_device *vdev; 2099 2100 pci_disable_sriov(pdev); 2101 2102 vdev = vfio_del_group_dev(&pdev->dev); 2103 if (!vdev) 2104 return; 2105 2106 vfio_pci_vf_uninit(vdev); 2107 vfio_pci_reflck_put(vdev->reflck); 2108 vfio_pci_vga_uninit(vdev); 2109 2110 vfio_iommu_group_put(pdev->dev.iommu_group, &pdev->dev); 2111 2112 if (!disable_idle_d3) 2113 vfio_pci_set_power_state(vdev, PCI_D0); 2114 2115 mutex_destroy(&vdev->ioeventfds_lock); 2116 kfree(vdev->region); 2117 kfree(vdev->pm_save); 2118 kfree(vdev); 2119} 2120 2121static pci_ers_result_t vfio_pci_aer_err_detected(struct pci_dev *pdev, 2122 pci_channel_state_t state) 2123{ 2124 struct vfio_pci_device *vdev; 2125 struct vfio_device *device; 2126 2127 device = vfio_device_get_from_dev(&pdev->dev); 2128 if (device == NULL) 2129 return PCI_ERS_RESULT_DISCONNECT; 2130 2131 vdev = vfio_device_data(device); 2132 if (vdev == NULL) { 2133 vfio_device_put(device); 2134 return PCI_ERS_RESULT_DISCONNECT; 2135 } 2136 2137 mutex_lock(&vdev->igate); 2138 2139 if (vdev->err_trigger) 2140 eventfd_signal(vdev->err_trigger, 1); 2141 2142 mutex_unlock(&vdev->igate); 2143 2144 vfio_device_put(device); 2145 2146 return PCI_ERS_RESULT_CAN_RECOVER; 2147} 2148 2149static int vfio_pci_sriov_configure(struct pci_dev *pdev, int nr_virtfn) 2150{ 2151 struct vfio_pci_device *vdev; 2152 struct vfio_device *device; 2153 int ret = 0; 2154 2155 might_sleep(); 2156 2157 if (!enable_sriov) 2158 return -ENOENT; 2159 2160 device = vfio_device_get_from_dev(&pdev->dev); 2161 if (!device) 2162 return -ENODEV; 2163 2164 vdev = vfio_device_data(device); 2165 if (!vdev) { 2166 vfio_device_put(device); 2167 return -ENODEV; 2168 } 2169 2170 if (nr_virtfn == 0) 2171 pci_disable_sriov(pdev); 2172 else 2173 ret = pci_enable_sriov(pdev, nr_virtfn); 2174 2175 vfio_device_put(device); 2176 2177 return ret < 0 ? ret : nr_virtfn; 2178} 2179 2180static const struct pci_error_handlers vfio_err_handlers = { 2181 .error_detected = vfio_pci_aer_err_detected, 2182}; 2183 2184static struct pci_driver vfio_pci_driver = { 2185 .name = "vfio-pci", 2186 .id_table = NULL, /* only dynamic ids */ 2187 .probe = vfio_pci_probe, 2188 .remove = vfio_pci_remove, 2189 .sriov_configure = vfio_pci_sriov_configure, 2190 .err_handler = &vfio_err_handlers, 2191}; 2192 2193static DEFINE_MUTEX(reflck_lock); 2194 2195static struct vfio_pci_reflck *vfio_pci_reflck_alloc(void) 2196{ 2197 struct vfio_pci_reflck *reflck; 2198 2199 reflck = kzalloc(sizeof(*reflck), GFP_KERNEL); 2200 if (!reflck) 2201 return ERR_PTR(-ENOMEM); 2202 2203 kref_init(&reflck->kref); 2204 mutex_init(&reflck->lock); 2205 2206 return reflck; 2207} 2208 2209static void vfio_pci_reflck_get(struct vfio_pci_reflck *reflck) 2210{ 2211 kref_get(&reflck->kref); 2212} 2213 2214static int vfio_pci_reflck_find(struct pci_dev *pdev, void *data) 2215{ 2216 struct vfio_pci_reflck **preflck = data; 2217 struct vfio_device *device; 2218 struct vfio_pci_device *vdev; 2219 2220 device = vfio_device_get_from_dev(&pdev->dev); 2221 if (!device) 2222 return 0; 2223 2224 if (pci_dev_driver(pdev) != &vfio_pci_driver) { 2225 vfio_device_put(device); 2226 return 0; 2227 } 2228 2229 vdev = vfio_device_data(device); 2230 2231 if (vdev->reflck) { 2232 vfio_pci_reflck_get(vdev->reflck); 2233 *preflck = vdev->reflck; 2234 vfio_device_put(device); 2235 return 1; 2236 } 2237 2238 vfio_device_put(device); 2239 return 0; 2240} 2241 2242static int vfio_pci_reflck_attach(struct vfio_pci_device *vdev) 2243{ 2244 bool slot = !pci_probe_reset_slot(vdev->pdev->slot); 2245 2246 mutex_lock(&reflck_lock); 2247 2248 if (pci_is_root_bus(vdev->pdev->bus) || 2249 vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_reflck_find, 2250 &vdev->reflck, slot) <= 0) 2251 vdev->reflck = vfio_pci_reflck_alloc(); 2252 2253 mutex_unlock(&reflck_lock); 2254 2255 return PTR_ERR_OR_ZERO(vdev->reflck); 2256} 2257 2258static void vfio_pci_reflck_release(struct kref *kref) 2259{ 2260 struct vfio_pci_reflck *reflck = container_of(kref, 2261 struct vfio_pci_reflck, 2262 kref); 2263 2264 kfree(reflck); 2265 mutex_unlock(&reflck_lock); 2266} 2267 2268static void vfio_pci_reflck_put(struct vfio_pci_reflck *reflck) 2269{ 2270 kref_put_mutex(&reflck->kref, vfio_pci_reflck_release, &reflck_lock); 2271} 2272 2273static int vfio_pci_get_unused_devs(struct pci_dev *pdev, void *data) 2274{ 2275 struct vfio_devices *devs = data; 2276 struct vfio_device *device; 2277 struct vfio_pci_device *vdev; 2278 2279 if (devs->cur_index == devs->max_index) 2280 return -ENOSPC; 2281 2282 device = vfio_device_get_from_dev(&pdev->dev); 2283 if (!device) 2284 return -EINVAL; 2285 2286 if (pci_dev_driver(pdev) != &vfio_pci_driver) { 2287 vfio_device_put(device); 2288 return -EBUSY; 2289 } 2290 2291 vdev = vfio_device_data(device); 2292 2293 /* Fault if the device is not unused */ 2294 if (vdev->refcnt) { 2295 vfio_device_put(device); 2296 return -EBUSY; 2297 } 2298 2299 devs->devices[devs->cur_index++] = device; 2300 return 0; 2301} 2302 2303static int vfio_pci_try_zap_and_vma_lock_cb(struct pci_dev *pdev, void *data) 2304{ 2305 struct vfio_devices *devs = data; 2306 struct vfio_device *device; 2307 struct vfio_pci_device *vdev; 2308 2309 if (devs->cur_index == devs->max_index) 2310 return -ENOSPC; 2311 2312 device = vfio_device_get_from_dev(&pdev->dev); 2313 if (!device) 2314 return -EINVAL; 2315 2316 if (pci_dev_driver(pdev) != &vfio_pci_driver) { 2317 vfio_device_put(device); 2318 return -EBUSY; 2319 } 2320 2321 vdev = vfio_device_data(device); 2322 2323 /* 2324 * Locking multiple devices is prone to deadlock, runaway and 2325 * unwind if we hit contention. 2326 */ 2327 if (!vfio_pci_zap_and_vma_lock(vdev, true)) { 2328 vfio_device_put(device); 2329 return -EBUSY; 2330 } 2331 2332 devs->devices[devs->cur_index++] = device; 2333 return 0; 2334} 2335 2336/* 2337 * If a bus or slot reset is available for the provided device and: 2338 * - All of the devices affected by that bus or slot reset are unused 2339 * (!refcnt) 2340 * - At least one of the affected devices is marked dirty via 2341 * needs_reset (such as by lack of FLR support) 2342 * Then attempt to perform that bus or slot reset. Callers are required 2343 * to hold vdev->reflck->lock, protecting the bus/slot reset group from 2344 * concurrent opens. A vfio_device reference is acquired for each device 2345 * to prevent unbinds during the reset operation. 2346 * 2347 * NB: vfio-core considers a group to be viable even if some devices are 2348 * bound to drivers like pci-stub or pcieport. Here we require all devices 2349 * to be bound to vfio_pci since that's the only way we can be sure they 2350 * stay put. 2351 */ 2352static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev) 2353{ 2354 struct vfio_devices devs = { .cur_index = 0 }; 2355 int i = 0, ret = -EINVAL; 2356 bool slot = false; 2357 struct vfio_pci_device *tmp; 2358 2359 if (!pci_probe_reset_slot(vdev->pdev->slot)) 2360 slot = true; 2361 else if (pci_probe_reset_bus(vdev->pdev->bus)) 2362 return; 2363 2364 if (vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs, 2365 &i, slot) || !i) 2366 return; 2367 2368 devs.max_index = i; 2369 devs.devices = kcalloc(i, sizeof(struct vfio_device *), GFP_KERNEL); 2370 if (!devs.devices) 2371 return; 2372 2373 if (vfio_pci_for_each_slot_or_bus(vdev->pdev, 2374 vfio_pci_get_unused_devs, 2375 &devs, slot)) 2376 goto put_devs; 2377 2378 /* Does at least one need a reset? */ 2379 for (i = 0; i < devs.cur_index; i++) { 2380 tmp = vfio_device_data(devs.devices[i]); 2381 if (tmp->needs_reset) { 2382 ret = pci_reset_bus(vdev->pdev); 2383 break; 2384 } 2385 } 2386 2387put_devs: 2388 for (i = 0; i < devs.cur_index; i++) { 2389 tmp = vfio_device_data(devs.devices[i]); 2390 2391 /* 2392 * If reset was successful, affected devices no longer need 2393 * a reset and we should return all the collateral devices 2394 * to low power. If not successful, we either didn't reset 2395 * the bus or timed out waiting for it, so let's not touch 2396 * the power state. 2397 */ 2398 if (!ret) { 2399 tmp->needs_reset = false; 2400 2401 if (tmp != vdev && !disable_idle_d3) 2402 vfio_pci_set_power_state(tmp, PCI_D3hot); 2403 } 2404 2405 vfio_device_put(devs.devices[i]); 2406 } 2407 2408 kfree(devs.devices); 2409} 2410 2411static void __exit vfio_pci_cleanup(void) 2412{ 2413 pci_unregister_driver(&vfio_pci_driver); 2414 vfio_pci_uninit_perm_bits(); 2415} 2416 2417static void __init vfio_pci_fill_ids(void) 2418{ 2419 char *p, *id; 2420 int rc; 2421 2422 /* no ids passed actually */ 2423 if (ids[0] == '\0') 2424 return; 2425 2426 /* add ids specified in the module parameter */ 2427 p = ids; 2428 while ((id = strsep(&p, ","))) { 2429 unsigned int vendor, device, subvendor = PCI_ANY_ID, 2430 subdevice = PCI_ANY_ID, class = 0, class_mask = 0; 2431 int fields; 2432 2433 if (!strlen(id)) 2434 continue; 2435 2436 fields = sscanf(id, "%x:%x:%x:%x:%x:%x", 2437 &vendor, &device, &subvendor, &subdevice, 2438 &class, &class_mask); 2439 2440 if (fields < 2) { 2441 pr_warn("invalid id string \"%s\"\n", id); 2442 continue; 2443 } 2444 2445 rc = pci_add_dynid(&vfio_pci_driver, vendor, device, 2446 subvendor, subdevice, class, class_mask, 0); 2447 if (rc) 2448 pr_warn("failed to add dynamic id [%04x:%04x[%04x:%04x]] class %#08x/%08x (%d)\n", 2449 vendor, device, subvendor, subdevice, 2450 class, class_mask, rc); 2451 else 2452 pr_info("add [%04x:%04x[%04x:%04x]] class %#08x/%08x\n", 2453 vendor, device, subvendor, subdevice, 2454 class, class_mask); 2455 } 2456} 2457 2458static int __init vfio_pci_init(void) 2459{ 2460 int ret; 2461 2462 /* Allocate shared config space permision data used by all devices */ 2463 ret = vfio_pci_init_perm_bits(); 2464 if (ret) 2465 return ret; 2466 2467 /* Register and scan for devices */ 2468 ret = pci_register_driver(&vfio_pci_driver); 2469 if (ret) 2470 goto out_driver; 2471 2472 vfio_pci_fill_ids(); 2473 2474 if (disable_denylist) 2475 pr_warn("device denylist disabled.\n"); 2476 2477 return 0; 2478 2479out_driver: 2480 vfio_pci_uninit_perm_bits(); 2481 return ret; 2482} 2483 2484module_init(vfio_pci_init); 2485module_exit(vfio_pci_cleanup); 2486 2487MODULE_VERSION(DRIVER_VERSION); 2488MODULE_LICENSE("GPL v2"); 2489MODULE_AUTHOR(DRIVER_AUTHOR); 2490MODULE_DESCRIPTION(DRIVER_DESC); 2491