1// SPDX-License-Identifier: GPL-2.0 2/* 3 * PCI Express I/O Virtualization (IOV) support 4 * Single Root IOV 1.0 5 * Address Translation Service 1.0 6 * 7 * Copyright (C) 2009 Intel Corporation, Yu Zhao <yu.zhao@intel.com> 8 */ 9 10#include <linux/pci.h> 11#include <linux/slab.h> 12#include <linux/export.h> 13#include <linux/string.h> 14#include <linux/delay.h> 15#include "pci.h" 16 17#define VIRTFN_ID_LEN 16 18 19int pci_iov_virtfn_bus(struct pci_dev *dev, int vf_id) 20{ 21 if (!dev->is_physfn) 22 return -EINVAL; 23 return dev->bus->number + ((dev->devfn + dev->sriov->offset + 24 dev->sriov->stride * vf_id) >> 8); 25} 26 27int pci_iov_virtfn_devfn(struct pci_dev *dev, int vf_id) 28{ 29 if (!dev->is_physfn) 30 return -EINVAL; 31 return (dev->devfn + dev->sriov->offset + 32 dev->sriov->stride * vf_id) & 0xff; 33} 34 35/* 36 * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may 37 * change when NumVFs changes. 38 * 39 * Update iov->offset and iov->stride when NumVFs is written. 40 */ 41static inline void pci_iov_set_numvfs(struct pci_dev *dev, int nr_virtfn) 42{ 43 struct pci_sriov *iov = dev->sriov; 44 45 pci_write_config_word(dev, iov->pos + PCI_SRIOV_NUM_VF, nr_virtfn); 46 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_OFFSET, &iov->offset); 47 pci_read_config_word(dev, iov->pos + PCI_SRIOV_VF_STRIDE, &iov->stride); 48} 49 50/* 51 * The PF consumes one bus number. NumVFs, First VF Offset, and VF Stride 52 * determine how many additional bus numbers will be consumed by VFs. 53 * 54 * Iterate over all valid NumVFs, validate offset and stride, and calculate 55 * the maximum number of bus numbers that could ever be required. 56 */ 57static int compute_max_vf_buses(struct pci_dev *dev) 58{ 59 struct pci_sriov *iov = dev->sriov; 60 int nr_virtfn, busnr, rc = 0; 61 62 for (nr_virtfn = iov->total_VFs; nr_virtfn; nr_virtfn--) { 63 pci_iov_set_numvfs(dev, nr_virtfn); 64 if (!iov->offset || (nr_virtfn > 1 && !iov->stride)) { 65 rc = -EIO; 66 goto out; 67 } 68 69 busnr = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 70 if (busnr > iov->max_VF_buses) 71 iov->max_VF_buses = busnr; 72 } 73 74out: 75 pci_iov_set_numvfs(dev, 0); 76 return rc; 77} 78 79static struct pci_bus *virtfn_add_bus(struct pci_bus *bus, int busnr) 80{ 81 struct pci_bus *child; 82 83 if (bus->number == busnr) 84 return bus; 85 86 child = pci_find_bus(pci_domain_nr(bus), busnr); 87 if (child) 88 return child; 89 90 child = pci_add_new_bus(bus, NULL, busnr); 91 if (!child) 92 return NULL; 93 94 pci_bus_insert_busn_res(child, busnr, busnr); 95 96 return child; 97} 98 99static void virtfn_remove_bus(struct pci_bus *physbus, struct pci_bus *virtbus) 100{ 101 if (physbus != virtbus && list_empty(&virtbus->devices)) 102 pci_remove_bus(virtbus); 103} 104 105resource_size_t pci_iov_resource_size(struct pci_dev *dev, int resno) 106{ 107 if (!dev->is_physfn) 108 return 0; 109 110 return dev->sriov->barsz[resno - PCI_IOV_RESOURCES]; 111} 112 113static void pci_read_vf_config_common(struct pci_dev *virtfn) 114{ 115 struct pci_dev *physfn = virtfn->physfn; 116 117 /* 118 * Some config registers are the same across all associated VFs. 119 * Read them once from VF0 so we can skip reading them from the 120 * other VFs. 121 * 122 * PCIe r4.0, sec 9.3.4.1, technically doesn't require all VFs to 123 * have the same Revision ID and Subsystem ID, but we assume they 124 * do. 125 */ 126 pci_read_config_dword(virtfn, PCI_CLASS_REVISION, 127 &physfn->sriov->class); 128 pci_read_config_byte(virtfn, PCI_HEADER_TYPE, 129 &physfn->sriov->hdr_type); 130 pci_read_config_word(virtfn, PCI_SUBSYSTEM_VENDOR_ID, 131 &physfn->sriov->subsystem_vendor); 132 pci_read_config_word(virtfn, PCI_SUBSYSTEM_ID, 133 &physfn->sriov->subsystem_device); 134} 135 136int pci_iov_sysfs_link(struct pci_dev *dev, 137 struct pci_dev *virtfn, int id) 138{ 139 char buf[VIRTFN_ID_LEN]; 140 int rc; 141 142 sprintf(buf, "virtfn%u", id); 143 rc = sysfs_create_link(&dev->dev.kobj, &virtfn->dev.kobj, buf); 144 if (rc) 145 goto failed; 146 rc = sysfs_create_link(&virtfn->dev.kobj, &dev->dev.kobj, "physfn"); 147 if (rc) 148 goto failed1; 149 150 kobject_uevent(&virtfn->dev.kobj, KOBJ_CHANGE); 151 152 return 0; 153 154failed1: 155 sysfs_remove_link(&dev->dev.kobj, buf); 156failed: 157 return rc; 158} 159 160int pci_iov_add_virtfn(struct pci_dev *dev, int id) 161{ 162 int i; 163 int rc = -ENOMEM; 164 u64 size; 165 struct pci_dev *virtfn; 166 struct resource *res; 167 struct pci_sriov *iov = dev->sriov; 168 struct pci_bus *bus; 169 170 bus = virtfn_add_bus(dev->bus, pci_iov_virtfn_bus(dev, id)); 171 if (!bus) 172 goto failed; 173 174 virtfn = pci_alloc_dev(bus); 175 if (!virtfn) 176 goto failed0; 177 178 virtfn->devfn = pci_iov_virtfn_devfn(dev, id); 179 virtfn->vendor = dev->vendor; 180 virtfn->device = iov->vf_device; 181 virtfn->is_virtfn = 1; 182 virtfn->physfn = pci_dev_get(dev); 183 virtfn->no_command_memory = 1; 184 185 if (id == 0) 186 pci_read_vf_config_common(virtfn); 187 188 rc = pci_setup_device(virtfn); 189 if (rc) 190 goto failed1; 191 192 virtfn->dev.parent = dev->dev.parent; 193 virtfn->multifunction = 0; 194 195 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 196 res = &dev->resource[i + PCI_IOV_RESOURCES]; 197 if (!res->parent) 198 continue; 199 virtfn->resource[i].name = pci_name(virtfn); 200 virtfn->resource[i].flags = res->flags; 201 size = pci_iov_resource_size(dev, i + PCI_IOV_RESOURCES); 202 virtfn->resource[i].start = res->start + size * id; 203 virtfn->resource[i].end = virtfn->resource[i].start + size - 1; 204 rc = request_resource(res, &virtfn->resource[i]); 205 BUG_ON(rc); 206 } 207 208 pci_device_add(virtfn, virtfn->bus); 209 rc = pci_iov_sysfs_link(dev, virtfn, id); 210 if (rc) 211 goto failed1; 212 213 pci_bus_add_device(virtfn); 214 215 return 0; 216 217failed1: 218 pci_stop_and_remove_bus_device(virtfn); 219 pci_dev_put(dev); 220failed0: 221 virtfn_remove_bus(dev->bus, bus); 222failed: 223 224 return rc; 225} 226 227void pci_iov_remove_virtfn(struct pci_dev *dev, int id) 228{ 229 char buf[VIRTFN_ID_LEN]; 230 struct pci_dev *virtfn; 231 232 virtfn = pci_get_domain_bus_and_slot(pci_domain_nr(dev->bus), 233 pci_iov_virtfn_bus(dev, id), 234 pci_iov_virtfn_devfn(dev, id)); 235 if (!virtfn) 236 return; 237 238 sprintf(buf, "virtfn%u", id); 239 sysfs_remove_link(&dev->dev.kobj, buf); 240 /* 241 * pci_stop_dev() could have been called for this virtfn already, 242 * so the directory for the virtfn may have been removed before. 243 * Double check to avoid spurious sysfs warnings. 244 */ 245 if (virtfn->dev.kobj.sd) 246 sysfs_remove_link(&virtfn->dev.kobj, "physfn"); 247 248 pci_stop_and_remove_bus_device(virtfn); 249 virtfn_remove_bus(dev->bus, virtfn->bus); 250 251 /* balance pci_get_domain_bus_and_slot() */ 252 pci_dev_put(virtfn); 253 pci_dev_put(dev); 254} 255 256static ssize_t sriov_totalvfs_show(struct device *dev, 257 struct device_attribute *attr, 258 char *buf) 259{ 260 struct pci_dev *pdev = to_pci_dev(dev); 261 262 return sprintf(buf, "%u\n", pci_sriov_get_totalvfs(pdev)); 263} 264 265static ssize_t sriov_numvfs_show(struct device *dev, 266 struct device_attribute *attr, 267 char *buf) 268{ 269 struct pci_dev *pdev = to_pci_dev(dev); 270 u16 num_vfs; 271 272 /* Serialize vs sriov_numvfs_store() so readers see valid num_VFs */ 273 device_lock(&pdev->dev); 274 num_vfs = pdev->sriov->num_VFs; 275 device_unlock(&pdev->dev); 276 277 return sprintf(buf, "%u\n", num_vfs); 278} 279 280/* 281 * num_vfs > 0; number of VFs to enable 282 * num_vfs = 0; disable all VFs 283 * 284 * Note: SRIOV spec does not allow partial VF 285 * disable, so it's all or none. 286 */ 287static ssize_t sriov_numvfs_store(struct device *dev, 288 struct device_attribute *attr, 289 const char *buf, size_t count) 290{ 291 struct pci_dev *pdev = to_pci_dev(dev); 292 int ret; 293 u16 num_vfs; 294 295 ret = kstrtou16(buf, 0, &num_vfs); 296 if (ret < 0) 297 return ret; 298 299 if (num_vfs > pci_sriov_get_totalvfs(pdev)) 300 return -ERANGE; 301 302 device_lock(&pdev->dev); 303 304 if (num_vfs == pdev->sriov->num_VFs) 305 goto exit; 306 307 /* is PF driver loaded w/callback */ 308 if (!pdev->driver || !pdev->driver->sriov_configure) { 309 pci_info(pdev, "Driver does not support SRIOV configuration via sysfs\n"); 310 ret = -ENOENT; 311 goto exit; 312 } 313 314 if (num_vfs == 0) { 315 /* disable VFs */ 316 ret = pdev->driver->sriov_configure(pdev, 0); 317 goto exit; 318 } 319 320 /* enable VFs */ 321 if (pdev->sriov->num_VFs) { 322 pci_warn(pdev, "%d VFs already enabled. Disable before enabling %d VFs\n", 323 pdev->sriov->num_VFs, num_vfs); 324 ret = -EBUSY; 325 goto exit; 326 } 327 328 ret = pdev->driver->sriov_configure(pdev, num_vfs); 329 if (ret < 0) 330 goto exit; 331 332 if (ret != num_vfs) 333 pci_warn(pdev, "%d VFs requested; only %d enabled\n", 334 num_vfs, ret); 335 336exit: 337 device_unlock(&pdev->dev); 338 339 if (ret < 0) 340 return ret; 341 342 return count; 343} 344 345static ssize_t sriov_offset_show(struct device *dev, 346 struct device_attribute *attr, 347 char *buf) 348{ 349 struct pci_dev *pdev = to_pci_dev(dev); 350 351 return sprintf(buf, "%u\n", pdev->sriov->offset); 352} 353 354static ssize_t sriov_stride_show(struct device *dev, 355 struct device_attribute *attr, 356 char *buf) 357{ 358 struct pci_dev *pdev = to_pci_dev(dev); 359 360 return sprintf(buf, "%u\n", pdev->sriov->stride); 361} 362 363static ssize_t sriov_vf_device_show(struct device *dev, 364 struct device_attribute *attr, 365 char *buf) 366{ 367 struct pci_dev *pdev = to_pci_dev(dev); 368 369 return sprintf(buf, "%x\n", pdev->sriov->vf_device); 370} 371 372static ssize_t sriov_drivers_autoprobe_show(struct device *dev, 373 struct device_attribute *attr, 374 char *buf) 375{ 376 struct pci_dev *pdev = to_pci_dev(dev); 377 378 return sprintf(buf, "%u\n", pdev->sriov->drivers_autoprobe); 379} 380 381static ssize_t sriov_drivers_autoprobe_store(struct device *dev, 382 struct device_attribute *attr, 383 const char *buf, size_t count) 384{ 385 struct pci_dev *pdev = to_pci_dev(dev); 386 bool drivers_autoprobe; 387 388 if (kstrtobool(buf, &drivers_autoprobe) < 0) 389 return -EINVAL; 390 391 pdev->sriov->drivers_autoprobe = drivers_autoprobe; 392 393 return count; 394} 395 396static DEVICE_ATTR_RO(sriov_totalvfs); 397static DEVICE_ATTR_RW(sriov_numvfs); 398static DEVICE_ATTR_RO(sriov_offset); 399static DEVICE_ATTR_RO(sriov_stride); 400static DEVICE_ATTR_RO(sriov_vf_device); 401static DEVICE_ATTR_RW(sriov_drivers_autoprobe); 402 403static struct attribute *sriov_dev_attrs[] = { 404 &dev_attr_sriov_totalvfs.attr, 405 &dev_attr_sriov_numvfs.attr, 406 &dev_attr_sriov_offset.attr, 407 &dev_attr_sriov_stride.attr, 408 &dev_attr_sriov_vf_device.attr, 409 &dev_attr_sriov_drivers_autoprobe.attr, 410 NULL, 411}; 412 413static umode_t sriov_attrs_are_visible(struct kobject *kobj, 414 struct attribute *a, int n) 415{ 416 struct device *dev = kobj_to_dev(kobj); 417 418 if (!dev_is_pf(dev)) 419 return 0; 420 421 return a->mode; 422} 423 424const struct attribute_group sriov_dev_attr_group = { 425 .attrs = sriov_dev_attrs, 426 .is_visible = sriov_attrs_are_visible, 427}; 428 429int __weak pcibios_sriov_enable(struct pci_dev *pdev, u16 num_vfs) 430{ 431 return 0; 432} 433 434int __weak pcibios_sriov_disable(struct pci_dev *pdev) 435{ 436 return 0; 437} 438 439static int sriov_add_vfs(struct pci_dev *dev, u16 num_vfs) 440{ 441 unsigned int i; 442 int rc; 443 444 if (dev->no_vf_scan) 445 return 0; 446 447 for (i = 0; i < num_vfs; i++) { 448 rc = pci_iov_add_virtfn(dev, i); 449 if (rc) 450 goto failed; 451 } 452 return 0; 453failed: 454 while (i--) 455 pci_iov_remove_virtfn(dev, i); 456 457 return rc; 458} 459 460static int sriov_enable(struct pci_dev *dev, int nr_virtfn) 461{ 462 int rc; 463 int i; 464 int nres; 465 u16 initial; 466 struct resource *res; 467 struct pci_dev *pdev; 468 struct pci_sriov *iov = dev->sriov; 469 int bars = 0; 470 int bus; 471 472 if (!nr_virtfn) 473 return 0; 474 475 if (iov->num_VFs) 476 return -EINVAL; 477 478 pci_read_config_word(dev, iov->pos + PCI_SRIOV_INITIAL_VF, &initial); 479 if (initial > iov->total_VFs || 480 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (initial != iov->total_VFs))) 481 return -EIO; 482 483 if (nr_virtfn < 0 || nr_virtfn > iov->total_VFs || 484 (!(iov->cap & PCI_SRIOV_CAP_VFM) && (nr_virtfn > initial))) 485 return -EINVAL; 486 487 nres = 0; 488 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 489 bars |= (1 << (i + PCI_IOV_RESOURCES)); 490 res = &dev->resource[i + PCI_IOV_RESOURCES]; 491 if (res->parent) 492 nres++; 493 } 494 if (nres != iov->nres) { 495 pci_err(dev, "not enough MMIO resources for SR-IOV\n"); 496 return -ENOMEM; 497 } 498 499 bus = pci_iov_virtfn_bus(dev, nr_virtfn - 1); 500 if (bus > dev->bus->busn_res.end) { 501 pci_err(dev, "can't enable %d VFs (bus %02x out of range of %pR)\n", 502 nr_virtfn, bus, &dev->bus->busn_res); 503 return -ENOMEM; 504 } 505 506 if (pci_enable_resources(dev, bars)) { 507 pci_err(dev, "SR-IOV: IOV BARS not allocated\n"); 508 return -ENOMEM; 509 } 510 511 if (iov->link != dev->devfn) { 512 pdev = pci_get_slot(dev->bus, iov->link); 513 if (!pdev) 514 return -ENODEV; 515 516 if (!pdev->is_physfn) { 517 pci_dev_put(pdev); 518 return -ENOSYS; 519 } 520 521 rc = sysfs_create_link(&dev->dev.kobj, 522 &pdev->dev.kobj, "dep_link"); 523 pci_dev_put(pdev); 524 if (rc) 525 return rc; 526 } 527 528 iov->initial_VFs = initial; 529 if (nr_virtfn < initial) 530 initial = nr_virtfn; 531 532 rc = pcibios_sriov_enable(dev, initial); 533 if (rc) { 534 pci_err(dev, "failure %d from pcibios_sriov_enable()\n", rc); 535 goto err_pcibios; 536 } 537 538 pci_iov_set_numvfs(dev, nr_virtfn); 539 iov->ctrl |= PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE; 540 pci_cfg_access_lock(dev); 541 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 542 msleep(100); 543 pci_cfg_access_unlock(dev); 544 545 rc = sriov_add_vfs(dev, initial); 546 if (rc) 547 goto err_pcibios; 548 549 kobject_uevent(&dev->dev.kobj, KOBJ_CHANGE); 550 iov->num_VFs = nr_virtfn; 551 552 return 0; 553 554err_pcibios: 555 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 556 pci_cfg_access_lock(dev); 557 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 558 ssleep(1); 559 pci_cfg_access_unlock(dev); 560 561 pcibios_sriov_disable(dev); 562 563 if (iov->link != dev->devfn) 564 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 565 566 pci_iov_set_numvfs(dev, 0); 567 return rc; 568} 569 570static void sriov_del_vfs(struct pci_dev *dev) 571{ 572 struct pci_sriov *iov = dev->sriov; 573 int i; 574 575 for (i = 0; i < iov->num_VFs; i++) 576 pci_iov_remove_virtfn(dev, i); 577} 578 579static void sriov_disable(struct pci_dev *dev) 580{ 581 struct pci_sriov *iov = dev->sriov; 582 583 if (!iov->num_VFs) 584 return; 585 586 sriov_del_vfs(dev); 587 iov->ctrl &= ~(PCI_SRIOV_CTRL_VFE | PCI_SRIOV_CTRL_MSE); 588 pci_cfg_access_lock(dev); 589 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 590 ssleep(1); 591 pci_cfg_access_unlock(dev); 592 593 pcibios_sriov_disable(dev); 594 595 if (iov->link != dev->devfn) 596 sysfs_remove_link(&dev->dev.kobj, "dep_link"); 597 598 iov->num_VFs = 0; 599 pci_iov_set_numvfs(dev, 0); 600} 601 602static int sriov_init(struct pci_dev *dev, int pos) 603{ 604 int i, bar64; 605 int rc; 606 int nres; 607 u32 pgsz; 608 u16 ctrl, total; 609 struct pci_sriov *iov; 610 struct resource *res; 611 struct pci_dev *pdev; 612 613 pci_read_config_word(dev, pos + PCI_SRIOV_CTRL, &ctrl); 614 if (ctrl & PCI_SRIOV_CTRL_VFE) { 615 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, 0); 616 ssleep(1); 617 } 618 619 ctrl = 0; 620 list_for_each_entry(pdev, &dev->bus->devices, bus_list) 621 if (pdev->is_physfn) 622 goto found; 623 624 pdev = NULL; 625 if (pci_ari_enabled(dev->bus)) 626 ctrl |= PCI_SRIOV_CTRL_ARI; 627 628found: 629 pci_write_config_word(dev, pos + PCI_SRIOV_CTRL, ctrl); 630 631 pci_read_config_word(dev, pos + PCI_SRIOV_TOTAL_VF, &total); 632 if (!total) 633 return 0; 634 635 pci_read_config_dword(dev, pos + PCI_SRIOV_SUP_PGSIZE, &pgsz); 636 i = PAGE_SHIFT > 12 ? PAGE_SHIFT - 12 : 0; 637 pgsz &= ~((1 << i) - 1); 638 if (!pgsz) 639 return -EIO; 640 641 pgsz &= ~(pgsz - 1); 642 pci_write_config_dword(dev, pos + PCI_SRIOV_SYS_PGSIZE, pgsz); 643 644 iov = kzalloc(sizeof(*iov), GFP_KERNEL); 645 if (!iov) 646 return -ENOMEM; 647 648 nres = 0; 649 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 650 res = &dev->resource[i + PCI_IOV_RESOURCES]; 651 /* 652 * If it is already FIXED, don't change it, something 653 * (perhaps EA or header fixups) wants it this way. 654 */ 655 if (res->flags & IORESOURCE_PCI_FIXED) 656 bar64 = (res->flags & IORESOURCE_MEM_64) ? 1 : 0; 657 else 658 bar64 = __pci_read_base(dev, pci_bar_unknown, res, 659 pos + PCI_SRIOV_BAR + i * 4); 660 if (!res->flags) 661 continue; 662 if (resource_size(res) & (PAGE_SIZE - 1)) { 663 rc = -EIO; 664 goto failed; 665 } 666 iov->barsz[i] = resource_size(res); 667 res->end = res->start + resource_size(res) * total - 1; 668 pci_info(dev, "VF(n) BAR%d space: %pR (contains BAR%d for %d VFs)\n", 669 i, res, i, total); 670 i += bar64; 671 nres++; 672 } 673 674 iov->pos = pos; 675 iov->nres = nres; 676 iov->ctrl = ctrl; 677 iov->total_VFs = total; 678 iov->driver_max_VFs = total; 679 pci_read_config_word(dev, pos + PCI_SRIOV_VF_DID, &iov->vf_device); 680 iov->pgsz = pgsz; 681 iov->self = dev; 682 iov->drivers_autoprobe = true; 683 pci_read_config_dword(dev, pos + PCI_SRIOV_CAP, &iov->cap); 684 pci_read_config_byte(dev, pos + PCI_SRIOV_FUNC_LINK, &iov->link); 685 if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) 686 iov->link = PCI_DEVFN(PCI_SLOT(dev->devfn), iov->link); 687 688 if (pdev) 689 iov->dev = pci_dev_get(pdev); 690 else 691 iov->dev = dev; 692 693 dev->sriov = iov; 694 dev->is_physfn = 1; 695 rc = compute_max_vf_buses(dev); 696 if (rc) 697 goto fail_max_buses; 698 699 return 0; 700 701fail_max_buses: 702 dev->sriov = NULL; 703 dev->is_physfn = 0; 704failed: 705 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) { 706 res = &dev->resource[i + PCI_IOV_RESOURCES]; 707 res->flags = 0; 708 } 709 710 kfree(iov); 711 return rc; 712} 713 714static void sriov_release(struct pci_dev *dev) 715{ 716 BUG_ON(dev->sriov->num_VFs); 717 718 if (dev != dev->sriov->dev) 719 pci_dev_put(dev->sriov->dev); 720 721 kfree(dev->sriov); 722 dev->sriov = NULL; 723} 724 725static void sriov_restore_state(struct pci_dev *dev) 726{ 727 int i; 728 u16 ctrl; 729 struct pci_sriov *iov = dev->sriov; 730 731 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &ctrl); 732 if (ctrl & PCI_SRIOV_CTRL_VFE) 733 return; 734 735 /* 736 * Restore PCI_SRIOV_CTRL_ARI before pci_iov_set_numvfs() because 737 * it reads offset & stride, which depend on PCI_SRIOV_CTRL_ARI. 738 */ 739 ctrl &= ~PCI_SRIOV_CTRL_ARI; 740 ctrl |= iov->ctrl & PCI_SRIOV_CTRL_ARI; 741 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, ctrl); 742 743 for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) 744 pci_update_resource(dev, i + PCI_IOV_RESOURCES); 745 746 pci_write_config_dword(dev, iov->pos + PCI_SRIOV_SYS_PGSIZE, iov->pgsz); 747 pci_iov_set_numvfs(dev, iov->num_VFs); 748 pci_write_config_word(dev, iov->pos + PCI_SRIOV_CTRL, iov->ctrl); 749 if (iov->ctrl & PCI_SRIOV_CTRL_VFE) 750 msleep(100); 751} 752 753/** 754 * pci_iov_init - initialize the IOV capability 755 * @dev: the PCI device 756 * 757 * Returns 0 on success, or negative on failure. 758 */ 759int pci_iov_init(struct pci_dev *dev) 760{ 761 int pos; 762 763 if (!pci_is_pcie(dev)) 764 return -ENODEV; 765 766 pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_SRIOV); 767 if (pos) 768 return sriov_init(dev, pos); 769 770 return -ENODEV; 771} 772 773/** 774 * pci_iov_release - release resources used by the IOV capability 775 * @dev: the PCI device 776 */ 777void pci_iov_release(struct pci_dev *dev) 778{ 779 if (dev->is_physfn) 780 sriov_release(dev); 781} 782 783/** 784 * pci_iov_remove - clean up SR-IOV state after PF driver is detached 785 * @dev: the PCI device 786 */ 787void pci_iov_remove(struct pci_dev *dev) 788{ 789 struct pci_sriov *iov = dev->sriov; 790 791 if (!dev->is_physfn) 792 return; 793 794 iov->driver_max_VFs = iov->total_VFs; 795 if (iov->num_VFs) 796 pci_warn(dev, "driver left SR-IOV enabled after remove\n"); 797} 798 799/** 800 * pci_iov_update_resource - update a VF BAR 801 * @dev: the PCI device 802 * @resno: the resource number 803 * 804 * Update a VF BAR in the SR-IOV capability of a PF. 805 */ 806void pci_iov_update_resource(struct pci_dev *dev, int resno) 807{ 808 struct pci_sriov *iov = dev->is_physfn ? dev->sriov : NULL; 809 struct resource *res = dev->resource + resno; 810 int vf_bar = resno - PCI_IOV_RESOURCES; 811 struct pci_bus_region region; 812 u16 cmd; 813 u32 new; 814 int reg; 815 816 /* 817 * The generic pci_restore_bars() path calls this for all devices, 818 * including VFs and non-SR-IOV devices. If this is not a PF, we 819 * have nothing to do. 820 */ 821 if (!iov) 822 return; 823 824 pci_read_config_word(dev, iov->pos + PCI_SRIOV_CTRL, &cmd); 825 if ((cmd & PCI_SRIOV_CTRL_VFE) && (cmd & PCI_SRIOV_CTRL_MSE)) { 826 dev_WARN(&dev->dev, "can't update enabled VF BAR%d %pR\n", 827 vf_bar, res); 828 return; 829 } 830 831 /* 832 * Ignore unimplemented BARs, unused resource slots for 64-bit 833 * BARs, and non-movable resources, e.g., those described via 834 * Enhanced Allocation. 835 */ 836 if (!res->flags) 837 return; 838 839 if (res->flags & IORESOURCE_UNSET) 840 return; 841 842 if (res->flags & IORESOURCE_PCI_FIXED) 843 return; 844 845 pcibios_resource_to_bus(dev->bus, ®ion, res); 846 new = region.start; 847 new |= res->flags & ~PCI_BASE_ADDRESS_MEM_MASK; 848 849 reg = iov->pos + PCI_SRIOV_BAR + 4 * vf_bar; 850 pci_write_config_dword(dev, reg, new); 851 if (res->flags & IORESOURCE_MEM_64) { 852 new = region.start >> 16 >> 16; 853 pci_write_config_dword(dev, reg + 4, new); 854 } 855} 856 857resource_size_t __weak pcibios_iov_resource_alignment(struct pci_dev *dev, 858 int resno) 859{ 860 return pci_iov_resource_size(dev, resno); 861} 862 863/** 864 * pci_sriov_resource_alignment - get resource alignment for VF BAR 865 * @dev: the PCI device 866 * @resno: the resource number 867 * 868 * Returns the alignment of the VF BAR found in the SR-IOV capability. 869 * This is not the same as the resource size which is defined as 870 * the VF BAR size multiplied by the number of VFs. The alignment 871 * is just the VF BAR size. 872 */ 873resource_size_t pci_sriov_resource_alignment(struct pci_dev *dev, int resno) 874{ 875 return pcibios_iov_resource_alignment(dev, resno); 876} 877 878/** 879 * pci_restore_iov_state - restore the state of the IOV capability 880 * @dev: the PCI device 881 */ 882void pci_restore_iov_state(struct pci_dev *dev) 883{ 884 if (dev->is_physfn) 885 sriov_restore_state(dev); 886} 887 888/** 889 * pci_vf_drivers_autoprobe - set PF property drivers_autoprobe for VFs 890 * @dev: the PCI device 891 * @auto_probe: set VF drivers auto probe flag 892 */ 893void pci_vf_drivers_autoprobe(struct pci_dev *dev, bool auto_probe) 894{ 895 if (dev->is_physfn) 896 dev->sriov->drivers_autoprobe = auto_probe; 897} 898 899/** 900 * pci_iov_bus_range - find bus range used by Virtual Function 901 * @bus: the PCI bus 902 * 903 * Returns max number of buses (exclude current one) used by Virtual 904 * Functions. 905 */ 906int pci_iov_bus_range(struct pci_bus *bus) 907{ 908 int max = 0; 909 struct pci_dev *dev; 910 911 list_for_each_entry(dev, &bus->devices, bus_list) { 912 if (!dev->is_physfn) 913 continue; 914 if (dev->sriov->max_VF_buses > max) 915 max = dev->sriov->max_VF_buses; 916 } 917 918 return max ? max - bus->number : 0; 919} 920 921/** 922 * pci_enable_sriov - enable the SR-IOV capability 923 * @dev: the PCI device 924 * @nr_virtfn: number of virtual functions to enable 925 * 926 * Returns 0 on success, or negative on failure. 927 */ 928int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn) 929{ 930 might_sleep(); 931 932 if (!dev->is_physfn) 933 return -ENOSYS; 934 935 return sriov_enable(dev, nr_virtfn); 936} 937EXPORT_SYMBOL_GPL(pci_enable_sriov); 938 939/** 940 * pci_disable_sriov - disable the SR-IOV capability 941 * @dev: the PCI device 942 */ 943void pci_disable_sriov(struct pci_dev *dev) 944{ 945 might_sleep(); 946 947 if (!dev->is_physfn) 948 return; 949 950 sriov_disable(dev); 951} 952EXPORT_SYMBOL_GPL(pci_disable_sriov); 953 954/** 955 * pci_num_vf - return number of VFs associated with a PF device_release_driver 956 * @dev: the PCI device 957 * 958 * Returns number of VFs, or 0 if SR-IOV is not enabled. 959 */ 960int pci_num_vf(struct pci_dev *dev) 961{ 962 if (!dev->is_physfn) 963 return 0; 964 965 return dev->sriov->num_VFs; 966} 967EXPORT_SYMBOL_GPL(pci_num_vf); 968 969/** 970 * pci_vfs_assigned - returns number of VFs are assigned to a guest 971 * @dev: the PCI device 972 * 973 * Returns number of VFs belonging to this device that are assigned to a guest. 974 * If device is not a physical function returns 0. 975 */ 976int pci_vfs_assigned(struct pci_dev *dev) 977{ 978 struct pci_dev *vfdev; 979 unsigned int vfs_assigned = 0; 980 unsigned short dev_id; 981 982 /* only search if we are a PF */ 983 if (!dev->is_physfn) 984 return 0; 985 986 /* 987 * determine the device ID for the VFs, the vendor ID will be the 988 * same as the PF so there is no need to check for that one 989 */ 990 dev_id = dev->sriov->vf_device; 991 992 /* loop through all the VFs to see if we own any that are assigned */ 993 vfdev = pci_get_device(dev->vendor, dev_id, NULL); 994 while (vfdev) { 995 /* 996 * It is considered assigned if it is a virtual function with 997 * our dev as the physical function and the assigned bit is set 998 */ 999 if (vfdev->is_virtfn && (vfdev->physfn == dev) && 1000 pci_is_dev_assigned(vfdev)) 1001 vfs_assigned++; 1002 1003 vfdev = pci_get_device(dev->vendor, dev_id, vfdev); 1004 } 1005 1006 return vfs_assigned; 1007} 1008EXPORT_SYMBOL_GPL(pci_vfs_assigned); 1009 1010/** 1011 * pci_sriov_set_totalvfs -- reduce the TotalVFs available 1012 * @dev: the PCI PF device 1013 * @numvfs: number that should be used for TotalVFs supported 1014 * 1015 * Should be called from PF driver's probe routine with 1016 * device's mutex held. 1017 * 1018 * Returns 0 if PF is an SRIOV-capable device and 1019 * value of numvfs valid. If not a PF return -ENOSYS; 1020 * if numvfs is invalid return -EINVAL; 1021 * if VFs already enabled, return -EBUSY. 1022 */ 1023int pci_sriov_set_totalvfs(struct pci_dev *dev, u16 numvfs) 1024{ 1025 if (!dev->is_physfn) 1026 return -ENOSYS; 1027 1028 if (numvfs > dev->sriov->total_VFs) 1029 return -EINVAL; 1030 1031 /* Shouldn't change if VFs already enabled */ 1032 if (dev->sriov->ctrl & PCI_SRIOV_CTRL_VFE) 1033 return -EBUSY; 1034 1035 dev->sriov->driver_max_VFs = numvfs; 1036 return 0; 1037} 1038EXPORT_SYMBOL_GPL(pci_sriov_set_totalvfs); 1039 1040/** 1041 * pci_sriov_get_totalvfs -- get total VFs supported on this device 1042 * @dev: the PCI PF device 1043 * 1044 * For a PCIe device with SRIOV support, return the PCIe 1045 * SRIOV capability value of TotalVFs or the value of driver_max_VFs 1046 * if the driver reduced it. Otherwise 0. 1047 */ 1048int pci_sriov_get_totalvfs(struct pci_dev *dev) 1049{ 1050 if (!dev->is_physfn) 1051 return 0; 1052 1053 return dev->sriov->driver_max_VFs; 1054} 1055EXPORT_SYMBOL_GPL(pci_sriov_get_totalvfs); 1056 1057/** 1058 * pci_sriov_configure_simple - helper to configure SR-IOV 1059 * @dev: the PCI device 1060 * @nr_virtfn: number of virtual functions to enable, 0 to disable 1061 * 1062 * Enable or disable SR-IOV for devices that don't require any PF setup 1063 * before enabling SR-IOV. Return value is negative on error, or number of 1064 * VFs allocated on success. 1065 */ 1066int pci_sriov_configure_simple(struct pci_dev *dev, int nr_virtfn) 1067{ 1068 int rc; 1069 1070 might_sleep(); 1071 1072 if (!dev->is_physfn) 1073 return -ENODEV; 1074 1075 if (pci_vfs_assigned(dev)) { 1076 pci_warn(dev, "Cannot modify SR-IOV while VFs are assigned\n"); 1077 return -EPERM; 1078 } 1079 1080 if (nr_virtfn == 0) { 1081 sriov_disable(dev); 1082 return 0; 1083 } 1084 1085 rc = sriov_enable(dev, nr_virtfn); 1086 if (rc < 0) 1087 return rc; 1088 1089 return nr_virtfn; 1090} 1091EXPORT_SYMBOL_GPL(pci_sriov_configure_simple); 1092