1// SPDX-License-Identifier: GPL-2.0 2/* 3 * PCI Express I/O Virtualization (IOV) support 4 * Single Root IOV 1.0 5 * Address Translation Service 1.0 6 * 7 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 8 */ 9 10#include <linux/pci.h> 11#include <linux/slab.h> 12#include <linux/export.h> 13#include <linux/string.h> 14#include <linux/delay.h> 15#include "pci.h" 16 17#define VIRTFN_ID_LEN 17 /* "virtfn%u\0" for 2^32 - 1 */ 18 19int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id) 20{ 21 if (!dev->is_physfn) 22 return -EINVAL; 23 return dev->bus->number + ((dev->devfn + dev->sriov->offset + 24 dev->sriov->stride * vf_id) >> 8); 25} 26 27int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id) 28{ 29 if (!dev->is_physfn) 30 return -EINVAL; 31 return (dev->devfn + dev->sriov->offset + 32 dev->sriov->stride * vf_id) & 0xff; 33} 34EXPORT_SYMBOL_GPL(pci_iov_virtfn_devfn); 35 36int pci_iov_vf_id(struct pci_dev *dev) 37{ 38 struct pci_dev *pf; 39 40 if (!dev->is_virtfn) 41 return -EINVAL; 42 43 pf = pci_physfn(dev); 44 return (pci_dev_id(dev) - (pci_dev_id(pf) + pf->sriov->offset)) / 45 pf->sriov->stride; 46} 47EXPORT_SYMBOL_GPL(pci_iov_vf_id); 48 49/** 50 * pci_iov_get_pf_drvdata - Return the drvdata of a PF 51 * @dev: VF pci_dev 52 * @pf_driver: Device driver required to own the PF 53 * 54 * This must be called from a context that ensures that a VF driver is attached. 55 * The value returned is invalid once the VF driver completes its remove() 56 * callback. 57 * 58 * Locking is achieved by the driver core. A VF driver cannot be probed until 59 * pci_enable_sriov() is called and pci_disable_sriov() does not return until 60 * all VF drivers have completed their remove(). 61 * 62 * The PF driver must call pci_disable_sriov() before it begins to destroy the 63 * drvdata. 64 */ 65void *pci_iov_get_pf_drvdata(struct pci_dev *dev, struct pci_driver *pf_driver) 66{ 67 struct pci_dev *pf_dev; 68 69 if (!dev->is_virtfn) 70 return ERR_PTR(-EINVAL); 71 pf_dev = dev->physfn; 72 if (pf_dev->driver != pf_driver) 73 return ERR_PTR(-EINVAL); 74 return pci_get_drvdata(pf_dev); 75} 76EXPORT_SYMBOL_GPL(pci_iov_get_pf_drvdata); 77 78/* 79 * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may 80 * change when NumVFs changes. 81 * 82 * Update iov->offset and iov->stride when NumVFs is written. 83 */ 84static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn) 85{ 86 struct pci_sriov *iov = dev->sriov; 87 88 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn); 89 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset); 90 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride); 91} 92 93/* 94 * The PF consumes one bus number. NumVFs, First VF Offset, and VF Stride 95 * determine how many additional bus numbers will be consumed by VFs. 96 * 97 * Iterate over all valid NumVFs, validate offset and stride, and calculate 98 * the maximum number of bus numbers that could ever be required. 99 */ 100static int compute_max_vf_buses(struct pci_dev *dev) 101{ 102 struct pci_sriov *iov = dev->sriov; 103 int nr_virtfn, busnr, rc = 0; 104 105 for (nr_virtfn = iov->total_VFs; nr_virtfn; nr_virtfn--) { 106 pci_iov_set_numvfs(dev, nr_virtfn); 107 if (!iov->offset || (nr_virtfn > 1 && !iov->stride)) { 108 rc = -EIO; 109 goto out; 110 } 111 112 busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 113 if (busnr > iov->max_VF_buses) 114 iov->max_VF_buses = busnr; 115 } 116 117out: 118 pci_iov_set_numvfs(dev, 0); 119 return rc; 120} 121 122static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr) 123{ 124 struct pci_bus *child; 125 126 if (bus->number == busnr) 127 return bus; 128 129 child = pci_find_bus(pci_domain_nr(bus), busnr); 130 if (child) 131 return child; 132 133 child = pci_add_new_bus(bus, NULL, busnr); 134 if (!child) 135 return NULL; 136 137 pci_bus_insert_busn_res(child, busnr, busnr); 138 139 return child; 140} 141 142static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus) 143{ 144 if (physbus != virtbus && list_empty(&virtbus->devices)) 145 pci_remove_bus(virtbus); 146} 147 148resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno) 149{ 150 if (!dev->is_physfn) 151 return 0; 152 153 return dev->sriov->barsz[resno - PCI_IOV_RESOURCES]; 154} 155 156static void pci_read_vf_config_common(struct pci_dev *virtfn) 157{ 158 struct pci_dev *physfn = virtfn->physfn; 159 160 /* 161 * Some config registers are the same across all associated VFs. 162 * Read them once from VF0 so we can skip reading them from the 163 * other VFs. 164 * 165 * PCIe r4.0, sec 9.3.4.1, technically doesn't require all VFs to 166 * have the same Revision ID and Subsystem ID, but we assume they 167 * do. 168 */ 169 pci_read_config_dword(virtfn, PCI_CLASS_REVISION, 170 &physfn->sriov->class); 171 pci_read_config_byte(virtfn, PCI_HEADER_TYPE, 172 &physfn->sriov->hdr_type); 173 pci_read_config_word(virtfn, PCI_SUBSYSTEM_VENDOR_ID, 174 &physfn->sriov->subsystem_vendor); 175 pci_read_config_word(virtfn, PCI_SUBSYSTEM_ID, 176 &physfn->sriov->subsystem_device); 177} 178 179int pci_iov_sysfs_link(struct pci_dev *dev, 180 struct pci_dev *virtfn, int id) 181{ 182 char buf[VIRTFN_ID_LEN]; 183 int rc; 184 185 sprintf(buf, "virtfn%u", id); 186 rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf); 187 if (rc) 188 goto failed; 189 rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn"); 190 if (rc) 191 goto failed1; 192 193 kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE); 194 195 return 0; 196 197failed1: 198 sysfs_remove_link(&dev->dev.kobj, buf); 199failed: 200 return rc; 201} 202 203#ifdef CONFIG_PCI_MSI 204static ssize_t sriov_vf_total_msix_show(struct device *dev, 205 struct device_attribute *attr, 206 char *buf) 207{ 208 struct pci_dev *pdev = to_pci_dev(dev); 209 u32 vf_total_msix = 0; 210 211 device_lock(dev); 212 if (!pdev->driver || !pdev->driver->sriov_get_vf_total_msix) 213 goto unlock; 214 215 vf_total_msix = pdev->driver->sriov_get_vf_total_msix(pdev); 216unlock: 217 device_unlock(dev); 218 return sysfs_emit(buf, "%u\n", vf_total_msix); 219} 220static DEVICE_ATTR_RO(sriov_vf_total_msix); 221 222static ssize_t sriov_vf_msix_count_store(struct device *dev, 223 struct device_attribute *attr, 224 const char *buf, size_t count) 225{ 226 struct pci_dev *vf_dev = to_pci_dev(dev); 227 struct pci_dev *pdev = pci_physfn(vf_dev); 228 int val, ret = 0; 229 230 if (kstrtoint(buf, 0, &val) < 0) 231 return -EINVAL; 232 233 if (val < 0) 234 return -EINVAL; 235 236 device_lock(&pdev->dev); 237 if (!pdev->driver || !pdev->driver->sriov_set_msix_vec_count) { 238 ret = -EOPNOTSUPP; 239 goto err_pdev; 240 } 241 242 device_lock(&vf_dev->dev); 243 if (vf_dev->driver) { 244 /* 245 * A driver is already attached to this VF and has configured 246 * itself based on the current MSI-X vector count. Changing 247 * the vector size could mess up the driver, so block it. 248 */ 249 ret = -EBUSY; 250 goto err_dev; 251 } 252 253 ret = pdev->driver->sriov_set_msix_vec_count(vf_dev, val); 254 255err_dev: 256 device_unlock(&vf_dev->dev); 257err_pdev: 258 device_unlock(&pdev->dev); 259 return ret ? : count; 260} 261static DEVICE_ATTR_WO(sriov_vf_msix_count); 262#endif 263 264static struct attribute *sriov_vf_dev_attrs[] = { 265#ifdef CONFIG_PCI_MSI 266 &dev_attr_sriov_vf_msix_count.attr, 267#endif 268 NULL, 269}; 270 271static umode_t sriov_vf_attrs_are_visible(struct kobject *kobj, 272 struct attribute *a, int n) 273{ 274 struct device *dev = kobj_to_dev(kobj); 275 struct pci_dev *pdev = to_pci_dev(dev); 276 277 if (!pdev->is_virtfn) 278 return 0; 279 280 return a->mode; 281} 282 283const struct attribute_group sriov_vf_dev_attr_group = { 284 .attrs = sriov_vf_dev_attrs, 285 .is_visible = sriov_vf_attrs_are_visible, 286}; 287 288int pci_iov_add_virtfn(struct pci_dev *dev, int id) 289{ 290 int i; 291 int rc = -ENOMEM; 292 u64 size; 293 struct pci_dev *virtfn; 294 struct resource *res; 295 struct pci_sriov *iov = dev->sriov; 296 struct pci_bus *bus; 297 298 bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id)); 299 if (!bus) 300 goto failed; 301 302 virtfn = pci_alloc_dev(bus); 303 if (!virtfn) 304 goto failed0; 305 306 virtfn->devfn = pci_iov_virtfn_devfn(dev, id); 307 virtfn->vendor = dev->vendor; 308 virtfn->device = iov->vf_device; 309 virtfn->is_virtfn = 1; 310 virtfn->physfn = pci_dev_get(dev); 311 virtfn->no_command_memory = 1; 312 313 if (id == 0) 314 pci_read_vf_config_common(virtfn); 315 316 rc = pci_setup_device(virtfn); 317 if (rc) 318 goto failed1; 319 320 virtfn->dev.parent = dev->dev.parent; 321 virtfn->multifunction = 0; 322 323 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 324 res = &dev->resource[i + PCI_IOV_RESOURCES]; 325 if (!res->parent) 326 continue; 327 virtfn->resource[i].name = pci_name(virtfn); 328 virtfn->resource[i].flags = res->flags; 329 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES); 330 virtfn->resource[i].start = res->start + size * id; 331 virtfn->resource[i].end = virtfn->resource[i].start + size - 1; 332 rc = request_resource(res, &virtfn->resource[i]); 333 BUG_ON(rc); 334 } 335 336 pci_device_add(virtfn, virtfn->bus); 337 rc = pci_iov_sysfs_link(dev, virtfn, id); 338 if (rc) 339 goto failed1; 340 341 pci_bus_add_device(virtfn); 342 343 return 0; 344 345failed1: 346 pci_stop_and_remove_bus_device(virtfn); 347 pci_dev_put(dev); 348failed0: 349 virtfn_remove_bus(dev->bus, bus); 350failed: 351 352 return rc; 353} 354 355void pci_iov_remove_virtfn(struct pci_dev *dev, int id) 356{ 357 char buf[VIRTFN_ID_LEN]; 358 struct pci_dev *virtfn; 359 360 virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus), 361 pci_iov_virtfn_bus(dev, id), 362 pci_iov_virtfn_devfn(dev, id)); 363 if (!virtfn) 364 return; 365 366 sprintf(buf, "virtfn%u", id); 367 sysfs_remove_link(&dev->dev.kobj, buf); 368 /* 369 * pci_stop_dev() could have been called for this virtfn already, 370 * so the directory for the virtfn may have been removed before. 371 * Double check to avoid spurious sysfs warnings. 372 */ 373 if (virtfn->dev.kobj.sd) 374 sysfs_remove_link(&virtfn->dev.kobj, "physfn"); 375 376 pci_stop_and_remove_bus_device(virtfn); 377 virtfn_remove_bus(dev->bus, virtfn->bus); 378 379 /* balance pci_get_domain_bus_and_slot() */ 380 pci_dev_put(virtfn); 381 pci_dev_put(dev); 382} 383 384static ssize_t sriov_totalvfs_show(struct device *dev, 385 struct device_attribute *attr, 386 char *buf) 387{ 388 struct pci_dev *pdev = to_pci_dev(dev); 389 390 return sysfs_emit(buf, "%u\n", pci_sriov_get_totalvfs(pdev)); 391} 392 393static ssize_t sriov_numvfs_show(struct device *dev, 394 struct device_attribute *attr, 395 char *buf) 396{ 397 struct pci_dev *pdev = to_pci_dev(dev); 398 u16 num_vfs; 399 400 /* Serialize vs sriov_numvfs_store() so readers see valid num_VFs */ 401 device_lock(&pdev->dev); 402 num_vfs = pdev->sriov->num_VFs; 403 device_unlock(&pdev->dev); 404 405 return sysfs_emit(buf, "%u\n", num_vfs); 406} 407 408/* 409 * num_vfs > 0; number of VFs to enable 410 * num_vfs = 0; disable all VFs 411 * 412 * Note: SRIOV spec does not allow partial VF 413 * disable, so it's all or none. 414 */ 415static ssize_t sriov_numvfs_store(struct device *dev, 416 struct device_attribute *attr, 417 const char *buf, size_t count) 418{ 419 struct pci_dev *pdev = to_pci_dev(dev); 420 int ret = 0; 421 u16 num_vfs; 422 423 if (kstrtou16(buf, 0, &num_vfs) < 0) 424 return -EINVAL; 425 426 if (num_vfs > pci_sriov_get_totalvfs(pdev)) 427 return -ERANGE; 428 429 device_lock(&pdev->dev); 430 431 if (num_vfs == pdev->sriov->num_VFs) 432 goto exit; 433 434 /* is PF driver loaded */ 435 if (!pdev->driver) { 436 pci_info(pdev, "no driver bound to device; cannot configure SR-IOV\n"); 437 ret = -ENOENT; 438 goto exit; 439 } 440 441 /* is PF driver loaded w/callback */ 442 if (!pdev->driver->sriov_configure) { 443 pci_info(pdev, "driver does not support SR-IOV configuration via sysfs\n"); 444 ret = -ENOENT; 445 goto exit; 446 } 447 448 if (num_vfs == 0) { 449 /* disable VFs */ 450 ret = pdev->driver->sriov_configure(pdev, 0); 451 goto exit; 452 } 453 454 /* enable VFs */ 455 if (pdev->sriov->num_VFs) { 456 pci_warn(pdev, "%d VFs already enabled. Disable before enabling %d VFs\n", 457 pdev->sriov->num_VFs, num_vfs); 458 ret = -EBUSY; 459 goto exit; 460 } 461 462 ret = pdev->driver->sriov_configure(pdev, num_vfs); 463 if (ret < 0) 464 goto exit; 465 466 if (ret != num_vfs) 467 pci_warn(pdev, "%d VFs requested; only %d enabled\n", 468 num_vfs, ret); 469 470exit: 471 device_unlock(&pdev->dev); 472 473 if (ret < 0) 474 return ret; 475 476 return count; 477} 478 479static ssize_t sriov_offset_show(struct device *dev, 480 struct device_attribute *attr, 481 char *buf) 482{ 483 struct pci_dev *pdev = to_pci_dev(dev); 484 485 return sysfs_emit(buf, "%u\n", pdev->sriov->offset); 486} 487 488static ssize_t sriov_stride_show(struct device *dev, 489 struct device_attribute *attr, 490 char *buf) 491{ 492 struct pci_dev *pdev = to_pci_dev(dev); 493 494 return sysfs_emit(buf, "%u\n", pdev->sriov->stride); 495} 496 497static ssize_t sriov_vf_device_show(struct device *dev, 498 struct device_attribute *attr, 499 char *buf) 500{ 501 struct pci_dev *pdev = to_pci_dev(dev); 502 503 return sysfs_emit(buf, "%x\n", pdev->sriov->vf_device); 504} 505 506static ssize_t sriov_drivers_autoprobe_show(struct device *dev, 507 struct device_attribute *attr, 508 char *buf) 509{ 510 struct pci_dev *pdev = to_pci_dev(dev); 511 512 return sysfs_emit(buf, "%u\n", pdev->sriov->drivers_autoprobe); 513} 514 515static ssize_t sriov_drivers_autoprobe_store(struct device *dev, 516 struct device_attribute *attr, 517 const char *buf, size_t count) 518{ 519 struct pci_dev *pdev = to_pci_dev(dev); 520 bool drivers_autoprobe; 521 522 if (kstrtobool(buf, &drivers_autoprobe) < 0) 523 return -EINVAL; 524 525 pdev->sriov->drivers_autoprobe = drivers_autoprobe; 526 527 return count; 528} 529 530static DEVICE_ATTR_RO(sriov_totalvfs); 531static DEVICE_ATTR_RW(sriov_numvfs); 532static DEVICE_ATTR_RO(sriov_offset); 533static DEVICE_ATTR_RO(sriov_stride); 534static DEVICE_ATTR_RO(sriov_vf_device); 535static DEVICE_ATTR_RW(sriov_drivers_autoprobe); 536 537static struct attribute *sriov_pf_dev_attrs[] = { 538 &dev_attr_sriov_totalvfs.attr, 539 &dev_attr_sriov_numvfs.attr, 540 &dev_attr_sriov_offset.attr, 541 &dev_attr_sriov_stride.attr, 542 &dev_attr_sriov_vf_device.attr, 543 &dev_attr_sriov_drivers_autoprobe.attr, 544#ifdef CONFIG_PCI_MSI 545 &dev_attr_sriov_vf_total_msix.attr, 546#endif 547 NULL, 548}; 549 550static umode_t sriov_pf_attrs_are_visible(struct kobject *kobj, 551 struct attribute *a, int n) 552{ 553 struct device *dev = kobj_to_dev(kobj); 554 555 if (!dev_is_pf(dev)) 556 return 0; 557 558 return a->mode; 559} 560 561const struct attribute_group sriov_pf_dev_attr_group = { 562 .attrs = sriov_pf_dev_attrs, 563 .is_visible = sriov_pf_attrs_are_visible, 564}; 565 566int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) 567{ 568 return 0; 569} 570 571int __weak pcibios_sriov_disable(struct pci_dev *pdev) 572{ 573 return 0; 574} 575 576static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) 577{ 578 unsigned int i; 579 int rc; 580 581 if (dev->no_vf_scan) 582 return 0; 583 584 for (i = 0; i < num_vfs; i++) { 585 rc = pci_iov_add_virtfn(dev, i); 586 if (rc) 587 goto failed; 588 } 589 return 0; 590failed: 591 while (i--) 592 pci_iov_remove_virtfn(dev, i); 593 594 return rc; 595} 596 597static int sriov_enable(struct pci_dev *dev, int nr_virtfn) 598{ 599 int rc; 600 int i; 601 int nres; 602 u16 initial; 603 struct resource *res; 604 struct pci_dev *pdev; 605 struct pci_sriov *iov = dev->sriov; 606 int bars = 0; 607 int bus; 608 609 if (!nr_virtfn) 610 return 0; 611 612 if (iov->num_VFs) 613 return -EINVAL; 614 615 pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial); 616 if (initial > iov->total_VFs || 617 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs))) 618 return -EIO; 619 620 if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs || 621 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial))) 622 return -EINVAL; 623 624 nres = 0; 625 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 626 bars |= (1 << (i + PCI_IOV_RESOURCES)); 627 res = &dev->resource[i + PCI_IOV_RESOURCES]; 628 if (res->parent) 629 nres++; 630 } 631 if (nres != iov->nres) { 632 pci_err(dev, "not enough MMIO resources for SR-IOV\n"); 633 return -ENOMEM; 634 } 635 636 bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 637 if (bus > dev->bus->busn_res.end) { 638 pci_err(dev, "can't enable %d VFs (bus %02x out of range of %pR)\n", 639 nr_virtfn, bus, &dev->bus->busn_res); 640 return -ENOMEM; 641 } 642 643 if (pci_enable_resources(dev, bars)) { 644 pci_err(dev, "SR-IOV: IOV BARS not allocated\n"); 645 return -ENOMEM; 646 } 647 648 if (iov->link != dev->devfn) { 649 pdev = pci_get_slot(dev->bus, iov->link); 650 if (!pdev) 651 return -ENODEV; 652 653 if (!pdev->is_physfn) { 654 pci_dev_put(pdev); 655 return -ENOSYS; 656 } 657 658 rc = sysfs_create_link(&dev->dev.kobj, 659 &pdev->dev.kobj, "dep_link"); 660 pci_dev_put(pdev); 661 if (rc) 662 return rc; 663 } 664 665 iov->initial_VFs = initial; 666 if (nr_virtfn < initial) 667 initial = nr_virtfn; 668 669 rc = pcibios_sriov_enable(dev, initial); 670 if (rc) { 671 pci_err(dev, "failure %d from pcibios_sriov_enable()\n", rc); 672 goto err_pcibios; 673 } 674 675 pci_iov_set_numvfs(dev, nr_virtfn); 676 iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; 677 pci_cfg_access_lock(dev); 678 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 679 msleep(100); 680 pci_cfg_access_unlock(dev); 681 682 rc = sriov_add_vfs(dev, initial); 683 if (rc) 684 goto err_pcibios; 685 686 kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE); 687 iov->num_VFs = nr_virtfn; 688 689 return 0; 690 691err_pcibios: 692 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 693 pci_cfg_access_lock(dev); 694 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 695 ssleep(1); 696 pci_cfg_access_unlock(dev); 697 698 pcibios_sriov_disable(dev); 699 700 if (iov->link != dev->devfn) 701 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 702 703 pci_iov_set_numvfs(dev, 0); 704 return rc; 705} 706 707static void sriov_del_vfs(struct pci_dev *dev) 708{ 709 struct pci_sriov *iov = dev->sriov; 710 int i; 711 712 for (i = 0; i < iov->num_VFs; i++) 713 pci_iov_remove_virtfn(dev, i); 714} 715 716static void sriov_disable(struct pci_dev *dev) 717{ 718 struct pci_sriov *iov = dev->sriov; 719 720 if (!iov->num_VFs) 721 return; 722 723 sriov_del_vfs(dev); 724 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 725 pci_cfg_access_lock(dev); 726 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 727 ssleep(1); 728 pci_cfg_access_unlock(dev); 729 730 pcibios_sriov_disable(dev); 731 732 if (iov->link != dev->devfn) 733 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 734 735 iov->num_VFs = 0; 736 pci_iov_set_numvfs(dev, 0); 737} 738 739static int sriov_init(struct pci_dev *dev, int pos) 740{ 741 int i, bar64; 742 int rc; 743 int nres; 744 u32 pgsz; 745 u16 ctrl, total; 746 struct pci_sriov *iov; 747 struct resource *res; 748 struct pci_dev *pdev; 749 750 pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl); 751 if (ctrl & PCI_SRIOV_CTRL_VFE) { 752 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0); 753 ssleep(1); 754 } 755 756 ctrl = 0; 757 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 758 if (pdev->is_physfn) 759 goto found; 760 761 pdev = NULL; 762 if (pci_ari_enabled(dev->bus)) 763 ctrl |= PCI_SRIOV_CTRL_ARI; 764 765found: 766 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl); 767 768 pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total); 769 if (!total) 770 return 0; 771 772 pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz); 773 i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0; 774 pgsz &= ~((1 << i) - 1); 775 if (!pgsz) 776 return -EIO; 777 778 pgsz &= ~(pgsz - 1); 779 pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz); 780 781 iov = kzalloc(sizeof(*iov), GFP_KERNEL); 782 if (!iov) 783 return -ENOMEM; 784 785 nres = 0; 786 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 787 res = &dev->resource[i + PCI_IOV_RESOURCES]; 788 /* 789 * If it is already FIXED, don't change it, something 790 * (perhaps EA or header fixups) wants it this way. 791 */ 792 if (res->flags & IORESOURCE_PCI_FIXED) 793 bar64 = (res->flags & IORESOURCE_MEM_64) ? 1 : 0; 794 else 795 bar64 = __pci_read_base(dev, pci_bar_unknown, res, 796 pos + PCI_SRIOV_BAR + i * 4); 797 if (!res->flags) 798 continue; 799 if (resource_size(res) & (PAGE_SIZE - 1)) { 800 rc = -EIO; 801 goto failed; 802 } 803 iov->barsz[i] = resource_size(res); 804 res->end = res->start + resource_size(res) * total - 1; 805 pci_info(dev, "VF(n) BAR%d space: %pR (contains BAR%d for %d VFs)\n", 806 i, res, i, total); 807 i += bar64; 808 nres++; 809 } 810 811 iov->pos = pos; 812 iov->nres = nres; 813 iov->ctrl = ctrl; 814 iov->total_VFs = total; 815 iov->driver_max_VFs = total; 816 pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device); 817 iov->pgsz = pgsz; 818 iov->self = dev; 819 iov->drivers_autoprobe = true; 820 pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap); 821 pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); 822 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) 823 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link); 824 825 if (pdev) 826 iov->dev = pci_dev_get(pdev); 827 else 828 iov->dev = dev; 829 830 dev->sriov = iov; 831 dev->is_physfn = 1; 832 rc = compute_max_vf_buses(dev); 833 if (rc) 834 goto fail_max_buses; 835 836 return 0; 837 838fail_max_buses: 839 dev->sriov = NULL; 840 dev->is_physfn = 0; 841failed: 842 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 843 res = &dev->resource[i + PCI_IOV_RESOURCES]; 844 res->flags = 0; 845 } 846 847 kfree(iov); 848 return rc; 849} 850 851static void sriov_release(struct pci_dev *dev) 852{ 853 BUG_ON(dev->sriov->num_VFs); 854 855 if (dev != dev->sriov->dev) 856 pci_dev_put(dev->sriov->dev); 857 858 kfree(dev->sriov); 859 dev->sriov = NULL; 860} 861 862static void sriov_restore_state(struct pci_dev *dev) 863{ 864 int i; 865 u16 ctrl; 866 struct pci_sriov *iov = dev->sriov; 867 868 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl); 869 if (ctrl & PCI_SRIOV_CTRL_VFE) 870 return; 871 872 /* 873 * Restore PCI_SRIOV_CTRL_ARI before pci_iov_set_numvfs() because 874 * it reads offset & stride, which depend on PCI_SRIOV_CTRL_ARI. 875 */ 876 ctrl &= ~PCI_SRIOV_CTRL_ARI; 877 ctrl |= iov->ctrl & PCI_SRIOV_CTRL_ARI; 878 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl); 879 880 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) 881 pci_update_resource(dev, i + PCI_IOV_RESOURCES); 882 883 pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz); 884 pci_iov_set_numvfs(dev, iov->num_VFs); 885 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 886 if (iov->ctrl & PCI_SRIOV_CTRL_VFE) 887 msleep(100); 888} 889 890/** 891 * pci_iov_init - initialize the IOV capability 892 * @dev: the PCI device 893 * 894 * Returns 0 on success, or negative on failure. 895 */ 896int pci_iov_init(struct pci_dev *dev) 897{ 898 int pos; 899 900 if (!pci_is_pcie(dev)) 901 return -ENODEV; 902 903 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); 904 if (pos) 905 return sriov_init(dev, pos); 906 907 return -ENODEV; 908} 909 910/** 911 * pci_iov_release - release resources used by the IOV capability 912 * @dev: the PCI device 913 */ 914void pci_iov_release(struct pci_dev *dev) 915{ 916 if (dev->is_physfn) 917 sriov_release(dev); 918} 919 920/** 921 * pci_iov_remove - clean up SR-IOV state after PF driver is detached 922 * @dev: the PCI device 923 */ 924void pci_iov_remove(struct pci_dev *dev) 925{ 926 struct pci_sriov *iov = dev->sriov; 927 928 if (!dev->is_physfn) 929 return; 930 931 iov->driver_max_VFs = iov->total_VFs; 932 if (iov->num_VFs) 933 pci_warn(dev, "driver left SR-IOV enabled after remove\n"); 934} 935 936/** 937 * pci_iov_update_resource - update a VF BAR 938 * @dev: the PCI device 939 * @resno: the resource number 940 * 941 * Update a VF BAR in the SR-IOV capability of a PF. 942 */ 943void pci_iov_update_resource(struct pci_dev *dev, int resno) 944{ 945 struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL; 946 struct resource *res = dev->resource + resno; 947 int vf_bar = resno - PCI_IOV_RESOURCES; 948 struct pci_bus_region region; 949 u16 cmd; 950 u32 new; 951 int reg; 952 953 /* 954 * The generic pci_restore_bars() path calls this for all devices, 955 * including VFs and non-SR-IOV devices. If this is not a PF, we 956 * have nothing to do. 957 */ 958 if (!iov) 959 return; 960 961 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd); 962 if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) { 963 dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n", 964 vf_bar, res); 965 return; 966 } 967 968 /* 969 * Ignore unimplemented BARs, unused resource slots for 64-bit 970 * BARs, and non-movable resources, e.g., those described via 971 * Enhanced Allocation. 972 */ 973 if (!res->flags) 974 return; 975 976 if (res->flags & IORESOURCE_UNSET) 977 return; 978 979 if (res->flags & IORESOURCE_PCI_FIXED) 980 return; 981 982 pcibios_resource_to_bus(dev->bus, ®ion, res); 983 new = region.start; 984 new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK; 985 986 reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar; 987 pci_write_config_dword(dev, reg, new); 988 if (res->flags & IORESOURCE_MEM_64) { 989 new = region.start >> 16 >> 16; 990 pci_write_config_dword(dev, reg + 4, new); 991 } 992} 993 994resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev, 995 int resno) 996{ 997 return pci_iov_resource_size(dev, resno); 998} 999 1000/** 1001 * pci_sriov_resource_alignment - get resource alignment for VF BAR 1002 * @dev: the PCI device 1003 * @resno: the resource number 1004 * 1005 * Returns the alignment of the VF BAR found in the SR-IOV capability. 1006 * This is not the same as the resource size which is defined as 1007 * the VF BAR size multiplied by the number of VFs. The alignment 1008 * is just the VF BAR size. 1009 */ 1010resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno) 1011{ 1012 return pcibios_iov_resource_alignment(dev, resno); 1013} 1014 1015/** 1016 * pci_restore_iov_state - restore the state of the IOV capability 1017 * @dev: the PCI device 1018 */ 1019void pci_restore_iov_state(struct pci_dev *dev) 1020{ 1021 if (dev->is_physfn) 1022 sriov_restore_state(dev); 1023} 1024 1025/** 1026 * pci_vf_drivers_autoprobe - set PF property drivers_autoprobe for VFs 1027 * @dev: the PCI device 1028 * @auto_probe: set VF drivers auto probe flag 1029 */ 1030void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool auto_probe) 1031{ 1032 if (dev->is_physfn) 1033 dev->sriov->drivers_autoprobe = auto_probe; 1034} 1035 1036/** 1037 * pci_iov_bus_range - find bus range used by Virtual Function 1038 * @bus: the PCI bus 1039 * 1040 * Returns max number of buses (exclude current one) used by Virtual 1041 * Functions. 1042 */ 1043int pci_iov_bus_range(struct pci_bus *bus) 1044{ 1045 int max = 0; 1046 struct pci_dev *dev; 1047 1048 list_for_each_entry(dev, &bus->devices, bus_list) { 1049 if (!dev->is_physfn) 1050 continue; 1051 if (dev->sriov->max_VF_buses > max) 1052 max = dev->sriov->max_VF_buses; 1053 } 1054 1055 return max ? max - bus->number : 0; 1056} 1057 1058/** 1059 * pci_enable_sriov - enable the SR-IOV capability 1060 * @dev: the PCI device 1061 * @nr_virtfn: number of virtual functions to enable 1062 * 1063 * Returns 0 on success, or negative on failure. 1064 */ 1065int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) 1066{ 1067 might_sleep(); 1068 1069 if (!dev->is_physfn) 1070 return -ENOSYS; 1071 1072 return sriov_enable(dev, nr_virtfn); 1073} 1074EXPORT_SYMBOL_GPL(pci_enable_sriov); 1075 1076/** 1077 * pci_disable_sriov - disable the SR-IOV capability 1078 * @dev: the PCI device 1079 */ 1080void pci_disable_sriov(struct pci_dev *dev) 1081{ 1082 might_sleep(); 1083 1084 if (!dev->is_physfn) 1085 return; 1086 1087 sriov_disable(dev); 1088} 1089EXPORT_SYMBOL_GPL(pci_disable_sriov); 1090 1091/** 1092 * pci_num_vf - return number of VFs associated with a PF device_release_driver 1093 * @dev: the PCI device 1094 * 1095 * Returns number of VFs, or 0 if SR-IOV is not enabled. 1096 */ 1097int pci_num_vf(struct pci_dev *dev) 1098{ 1099 if (!dev->is_physfn) 1100 return 0; 1101 1102 return dev->sriov->num_VFs; 1103} 1104EXPORT_SYMBOL_GPL(pci_num_vf); 1105 1106/** 1107 * pci_vfs_assigned - returns number of VFs are assigned to a guest 1108 * @dev: the PCI device 1109 * 1110 * Returns number of VFs belonging to this device that are assigned to a guest. 1111 * If device is not a physical function returns 0. 1112 */ 1113int pci_vfs_assigned(struct pci_dev *dev) 1114{ 1115 struct pci_dev *vfdev; 1116 unsigned int vfs_assigned = 0; 1117 unsigned short dev_id; 1118 1119 /* only search if we are a PF */ 1120 if (!dev->is_physfn) 1121 return 0; 1122 1123 /* 1124 * determine the device ID for the VFs, the vendor ID will be the 1125 * same as the PF so there is no need to check for that one 1126 */ 1127 dev_id = dev->sriov->vf_device; 1128 1129 /* loop through all the VFs to see if we own any that are assigned */ 1130 vfdev = pci_get_device(dev->vendor, dev_id, NULL); 1131 while (vfdev) { 1132 /* 1133 * It is considered assigned if it is a virtual function with 1134 * our dev as the physical function and the assigned bit is set 1135 */ 1136 if (vfdev->is_virtfn && (vfdev->physfn == dev) && 1137 pci_is_dev_assigned(vfdev)) 1138 vfs_assigned++; 1139 1140 vfdev = pci_get_device(dev->vendor, dev_id, vfdev); 1141 } 1142 1143 return vfs_assigned; 1144} 1145EXPORT_SYMBOL_GPL(pci_vfs_assigned); 1146 1147/** 1148 * pci_sriov_set_totalvfs -- reduce the TotalVFs available 1149 * @dev: the PCI PF device 1150 * @numvfs: number that should be used for TotalVFs supported 1151 * 1152 * Should be called from PF driver's probe routine with 1153 * device's mutex held. 1154 * 1155 * Returns 0 if PF is an SRIOV-capable device and 1156 * value of numvfs valid. If not a PF return -ENOSYS; 1157 * if numvfs is invalid return -EINVAL; 1158 * if VFs already enabled, return -EBUSY. 1159 */ 1160int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs) 1161{ 1162 if (!dev->is_physfn) 1163 return -ENOSYS; 1164 1165 if (numvfs > dev->sriov->total_VFs) 1166 return -EINVAL; 1167 1168 /* Shouldn't change if VFs already enabled */ 1169 if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE) 1170 return -EBUSY; 1171 1172 dev->sriov->driver_max_VFs = numvfs; 1173 return 0; 1174} 1175EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs); 1176 1177/** 1178 * pci_sriov_get_totalvfs -- get total VFs supported on this device 1179 * @dev: the PCI PF device 1180 * 1181 * For a PCIe device with SRIOV support, return the PCIe 1182 * SRIOV capability value of TotalVFs or the value of driver_max_VFs 1183 * if the driver reduced it. Otherwise 0. 1184 */ 1185int pci_sriov_get_totalvfs(struct pci_dev *dev) 1186{ 1187 if (!dev->is_physfn) 1188 return 0; 1189 1190 return dev->sriov->driver_max_VFs; 1191} 1192EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs); 1193 1194/** 1195 * pci_sriov_configure_simple - helper to configure SR-IOV 1196 * @dev: the PCI device 1197 * @nr_virtfn: number of virtual functions to enable, 0 to disable 1198 * 1199 * Enable or disable SR-IOV for devices that don't require any PF setup 1200 * before enabling SR-IOV. Return value is negative on error, or number of 1201 * VFs allocated on success. 1202 */ 1203int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn) 1204{ 1205 int rc; 1206 1207 might_sleep(); 1208 1209 if (!dev->is_physfn) 1210 return -ENODEV; 1211 1212 if (pci_vfs_assigned(dev)) { 1213 pci_warn(dev, "Cannot modify SR-IOV while VFs are assigned\n"); 1214 return -EPERM; 1215 } 1216 1217 if (nr_virtfn == 0) { 1218 sriov_disable(dev); 1219 return 0; 1220 } 1221 1222 rc = sriov_enable(dev, nr_virtfn); 1223 if (rc < 0) 1224 return rc; 1225 1226 return nr_virtfn; 1227} 1228EXPORT_SYMBOL_GPL(pci_sriov_configure_simple); 1229