Lines Matching refs:vdev
124 struct vfio_pci_device *vdev = opaque;
125 struct pci_dev *tmp = NULL, *pdev = vdev->pdev;
158 static void vfio_pci_probe_mmaps(struct vfio_pci_device *vdev)
167 res = &vdev->pdev->resource[bar];
184 vdev->bar_mmap_supported[bar] = true;
209 &vdev->dummy_resources_list);
210 vdev->bar_mmap_supported[bar] = true;
222 vdev->bar_mmap_supported[bar] = false;
226 static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev);
227 static void vfio_pci_disable(struct vfio_pci_device *vdev);
261 static void vfio_pci_probe_power_state(struct vfio_pci_device *vdev)
263 struct pci_dev *pdev = vdev->pdev;
271 vdev->needs_pm_restore = !(pmcsr & PCI_PM_CTRL_NO_SOFT_RESET);
276 * D3->D0 transition. Save state prior to D0/1/2->D3, stash it on the vdev,
281 int vfio_pci_set_power_state(struct vfio_pci_device *vdev, pci_power_t state)
283 struct pci_dev *pdev = vdev->pdev;
287 if (vdev->needs_pm_restore) {
302 vdev->pm_save = pci_store_saved_state(pdev);
304 pci_load_and_free_saved_state(pdev, &vdev->pm_save);
312 static int vfio_pci_enable(struct vfio_pci_device *vdev)
314 struct pci_dev *pdev = vdev->pdev;
319 vfio_pci_set_power_state(vdev, PCI_D0);
335 vdev->reset_works = !ret;
337 vdev->pci_saved_state = pci_store_saved_state(pdev);
338 if (!vdev->pci_saved_state)
344 vdev->nointx = true;
347 vdev->pci_2_3 = pci_intx_mask_supported(pdev);
351 if (vdev->pci_2_3 && (cmd & PCI_COMMAND_INTX_DISABLE)) {
356 ret = vfio_config_init(vdev);
358 kfree(vdev->pci_saved_state);
359 vdev->pci_saved_state = NULL;
372 vdev->msix_bar = table & PCI_MSIX_TABLE_BIR;
373 vdev->msix_offset = table & PCI_MSIX_TABLE_OFFSET;
374 vdev->msix_size = ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) * 16;
376 vdev->msix_bar = 0xFF;
379 vdev->has_vga = true;
385 ret = vfio_pci_igd_init(vdev);
394 ret = vfio_pci_nvdia_v100_nvlink2_init(vdev);
403 ret = vfio_pci_ibm_npu2_init(vdev);
410 vfio_pci_probe_mmaps(vdev);
415 vfio_pci_disable(vdev);
419 static void vfio_pci_disable(struct vfio_pci_device *vdev)
421 struct pci_dev *pdev = vdev->pdev;
429 vfio_pci_set_irqs_ioctl(vdev, VFIO_IRQ_SET_DATA_NONE |
431 vdev->irq_type, 0, 0, NULL);
435 &vdev->ioeventfds_list, next) {
440 vdev->ioeventfds_nr = 0;
442 vdev->virq_disabled = false;
444 for (i = 0; i < vdev->num_regions; i++)
445 vdev->region[i].ops->release(vdev, &vdev->region[i]);
447 vdev->num_regions = 0;
448 kfree(vdev->region);
449 vdev->region = NULL; /* don't krealloc a freed pointer */
451 vfio_config_free(vdev);
455 if (!vdev->barmap[bar])
457 pci_iounmap(pdev, vdev->barmap[bar]);
459 vdev->barmap[bar] = NULL;
463 &vdev->dummy_resources_list, res_next) {
469 vdev->needs_reset = true;
477 if (pci_load_and_free_saved_state(pdev, &vdev->pci_saved_state)) {
480 if (!vdev->reset_works)
499 if (vdev->reset_works && pci_cfg_access_trylock(pdev)) {
502 vdev->needs_reset = false;
512 vfio_pci_try_bus_reset(vdev);
515 vfio_pci_set_power_state(vdev, PCI_D3hot);
520 static struct vfio_pci_device *get_pf_vdev(struct vfio_pci_device *vdev,
523 struct pci_dev *physfn = pci_physfn(vdev->pdev);
525 if (!vdev->pdev->is_virtfn)
540 static void vfio_pci_vf_token_user_add(struct vfio_pci_device *vdev, int val)
543 struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev);
558 struct vfio_pci_device *vdev = device_data;
560 mutex_lock(&vdev->reflck->lock);
562 if (!(--vdev->refcnt)) {
563 vfio_pci_vf_token_user_add(vdev, -1);
564 vfio_spapr_pci_eeh_release(vdev->pdev);
565 vfio_pci_disable(vdev);
567 mutex_lock(&vdev->igate);
568 if (vdev->err_trigger) {
569 eventfd_ctx_put(vdev->err_trigger);
570 vdev->err_trigger = NULL;
572 if (vdev->req_trigger) {
573 eventfd_ctx_put(vdev->req_trigger);
574 vdev->req_trigger = NULL;
576 mutex_unlock(&vdev->igate);
579 mutex_unlock(&vdev->reflck->lock);
586 struct vfio_pci_device *vdev = device_data;
592 mutex_lock(&vdev->reflck->lock);
594 if (!vdev->refcnt) {
595 ret = vfio_pci_enable(vdev);
599 vfio_spapr_pci_eeh_open(vdev->pdev);
600 vfio_pci_vf_token_user_add(vdev, 1);
602 vdev->refcnt++;
604 mutex_unlock(&vdev->reflck->lock);
610 static int vfio_pci_get_irq_count(struct vfio_pci_device *vdev, int irq_type)
616 vdev->nointx || vdev->pdev->is_virtfn)
619 pci_read_config_byte(vdev->pdev, PCI_INTERRUPT_PIN, &pin);
626 pos = vdev->pdev->msi_cap;
628 pci_read_config_word(vdev->pdev,
636 pos = vdev->pdev->msix_cap;
638 pci_read_config_word(vdev->pdev,
644 if (pci_is_pcie(vdev->pdev))
757 static int msix_mmappable_cap(struct vfio_pci_device *vdev,
768 int vfio_pci_register_dev_region(struct vfio_pci_device *vdev,
775 region = krealloc(vdev->region,
776 (vdev->num_regions + 1) * sizeof(*region),
781 vdev->region = region;
782 vdev->region[vdev->num_regions].type = type;
783 vdev->region[vdev->num_regions].subtype = subtype;
784 vdev->region[vdev->num_regions].ops = ops;
785 vdev->region[vdev->num_regions].size = size;
786 vdev->region[vdev->num_regions].flags = flags;
787 vdev->region[vdev->num_regions].data = data;
789 vdev->num_regions++;
803 struct vfio_pci_device *vdev = device_data;
829 if (vdev->reset_works)
832 info.num_regions = VFIO_PCI_NUM_REGIONS + vdev->num_regions;
836 int ret = vfio_pci_info_zdev_add_caps(vdev, &caps);
839 pci_warn(vdev->pdev, "Failed to setup zPCI info capabilities\n");
866 struct pci_dev *pdev = vdev->pdev;
896 if (vdev->bar_mmap_supported[info.index]) {
898 if (info.index == vdev->msix_bar) {
899 ret = msix_mmappable_cap(vdev, &caps);
930 cmd = vfio_pci_memory_lock_and_enable(vdev);
938 vfio_pci_memory_unlock_and_restore(vdev, cmd);
943 if (!vdev->has_vga)
959 VFIO_PCI_NUM_REGIONS + vdev->num_regions)
963 vdev->num_regions);
968 info.size = vdev->region[i].size;
969 info.flags = vdev->region[i].flags;
971 cap_type.type = vdev->region[i].type;
972 cap_type.subtype = vdev->region[i].subtype;
979 if (vdev->region[i].ops->add_capability) {
980 ret = vdev->region[i].ops->add_capability(vdev,
981 &vdev->region[i], &caps);
1026 if (pci_is_pcie(vdev->pdev))
1035 info.count = vfio_pci_get_irq_count(vdev, info.index);
1057 max = vfio_pci_get_irq_count(vdev, hdr.index);
1071 mutex_lock(&vdev->igate);
1073 ret = vfio_pci_set_irqs_ioctl(vdev, hdr.flags, hdr.index,
1076 mutex_unlock(&vdev->igate);
1084 if (!vdev->reset_works)
1087 vfio_pci_zap_and_down_write_memory_lock(vdev);
1088 ret = pci_try_reset_function(vdev->pdev);
1089 up_write(&vdev->memory_lock);
1111 if (!pci_probe_reset_slot(vdev->pdev->slot))
1113 else if (pci_probe_reset_bus(vdev->pdev->bus))
1117 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1141 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1184 if (!pci_probe_reset_slot(vdev->pdev->slot))
1186 else if (pci_probe_reset_bus(vdev->pdev->bus))
1195 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1258 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1278 ret = vfio_pci_for_each_slot_or_bus(vdev->pdev,
1298 ret = pci_reset_bus(vdev->pdev);
1341 return vfio_pci_ioeventfd(vdev, ioeventfd.offset,
1370 if (!vdev->vf_token)
1394 mutex_lock(&vdev->vf_token->lock);
1395 uuid_copy(&vdev->vf_token->uuid, &uuid);
1396 mutex_unlock(&vdev->vf_token->lock);
1411 struct vfio_pci_device *vdev = device_data;
1413 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1418 return vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
1423 return vfio_pci_bar_rw(vdev, buf, count, ppos, false);
1426 return vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
1429 return vfio_pci_vga_rw(vdev, buf, count, ppos, iswrite);
1432 return vdev->region[index].ops->rw(vdev, buf,
1458 static int vfio_pci_zap_and_vma_lock(struct vfio_pci_device *vdev, bool try)
1489 if (!mutex_trylock(&vdev->vma_lock))
1492 mutex_lock(&vdev->vma_lock);
1494 while (!list_empty(&vdev->vma_list)) {
1495 mmap_vma = list_first_entry(&vdev->vma_list,
1508 mutex_unlock(&vdev->vma_lock);
1519 if (!mutex_trylock(&vdev->vma_lock)) {
1525 mutex_lock(&vdev->vma_lock);
1528 &vdev->vma_list, vma_next) {
1540 mutex_unlock(&vdev->vma_lock);
1546 void vfio_pci_zap_and_down_write_memory_lock(struct vfio_pci_device *vdev)
1548 vfio_pci_zap_and_vma_lock(vdev, false);
1549 down_write(&vdev->memory_lock);
1550 mutex_unlock(&vdev->vma_lock);
1553 u16 vfio_pci_memory_lock_and_enable(struct vfio_pci_device *vdev)
1557 down_write(&vdev->memory_lock);
1558 pci_read_config_word(vdev->pdev, PCI_COMMAND, &cmd);
1560 pci_write_config_word(vdev->pdev, PCI_COMMAND,
1566 void vfio_pci_memory_unlock_and_restore(struct vfio_pci_device *vdev, u16 cmd)
1568 pci_write_config_word(vdev->pdev, PCI_COMMAND, cmd);
1569 up_write(&vdev->memory_lock);
1573 static int __vfio_pci_add_vma(struct vfio_pci_device *vdev,
1583 list_add(&mmap_vma->vma_next, &vdev->vma_list);
1599 struct vfio_pci_device *vdev = vma->vm_private_data;
1602 mutex_lock(&vdev->vma_lock);
1603 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1610 mutex_unlock(&vdev->vma_lock);
1616 struct vfio_pci_device *vdev = vma->vm_private_data;
1620 mutex_lock(&vdev->vma_lock);
1621 down_read(&vdev->memory_lock);
1623 if (!__vfio_pci_memory_enabled(vdev)) {
1634 list_for_each_entry(mmap_vma, &vdev->vma_list, vma_next) {
1647 if (__vfio_pci_add_vma(vdev, vma)) {
1653 up_read(&vdev->memory_lock);
1654 mutex_unlock(&vdev->vma_lock);
1666 struct vfio_pci_device *vdev = device_data;
1667 struct pci_dev *pdev = vdev->pdev;
1674 if (index >= VFIO_PCI_NUM_REGIONS + vdev->num_regions)
1682 struct vfio_pci_region *region = vdev->region + regnum;
1686 return region->ops->mmap(vdev, region, vma);
1691 if (!vdev->bar_mmap_supported[index])
1707 if (!vdev->barmap[index]) {
1713 vdev->barmap[index] = pci_iomap(pdev, index, 0);
1714 if (!vdev->barmap[index]) {
1720 vma->vm_private_data = vdev;
1736 struct vfio_pci_device *vdev = device_data;
1737 struct pci_dev *pdev = vdev->pdev;
1739 mutex_lock(&vdev->igate);
1741 if (vdev->req_trigger) {
1746 eventfd_signal(vdev->req_trigger, 1);
1752 mutex_unlock(&vdev->igate);
1755 static int vfio_pci_validate_vf_token(struct vfio_pci_device *vdev,
1783 if (!vdev->pdev->is_virtfn && !vdev->vf_token && !vf_token)
1786 if (vdev->pdev->is_virtfn) {
1788 struct vfio_pci_device *pf_vdev = get_pf_vdev(vdev, &pf_dev);
1795 pci_info_ratelimited(vdev->pdev,
1802 pci_info_ratelimited(vdev->pdev,
1814 pci_info_ratelimited(vdev->pdev,
1818 } else if (vdev->vf_token) {
1819 mutex_lock(&vdev->vf_token->lock);
1820 if (vdev->vf_token->users) {
1822 mutex_unlock(&vdev->vf_token->lock);
1823 pci_info_ratelimited(vdev->pdev,
1828 if (!uuid_equal(uuid, &vdev->vf_token->uuid)) {
1829 mutex_unlock(&vdev->vf_token->lock);
1830 pci_info_ratelimited(vdev->pdev,
1835 uuid_copy(&vdev->vf_token->uuid, uuid);
1838 mutex_unlock(&vdev->vf_token->lock);
1840 pci_info_ratelimited(vdev->pdev,
1852 struct vfio_pci_device *vdev = device_data;
1857 if (strncmp(pci_name(vdev->pdev), buf, strlen(pci_name(vdev->pdev))))
1860 if (strlen(buf) > strlen(pci_name(vdev->pdev))) {
1861 buf += strlen(pci_name(vdev->pdev));
1892 ret = vfio_pci_validate_vf_token(vdev, vf_token, &uuid);
1911 static int vfio_pci_reflck_attach(struct vfio_pci_device *vdev);
1917 struct vfio_pci_device *vdev = container_of(nb,
1924 pdev->is_virtfn && physfn == vdev->pdev) {
1925 pci_info(vdev->pdev, "Captured SR-IOV VF %s driver_override\n",
1930 pdev->is_virtfn && physfn == vdev->pdev) {
1934 pci_warn(vdev->pdev,
1942 static int vfio_pci_vf_init(struct vfio_pci_device *vdev)
1944 struct pci_dev *pdev = vdev->pdev;
1950 vdev->vf_token = kzalloc(sizeof(*vdev->vf_token), GFP_KERNEL);
1951 if (!vdev->vf_token)
1954 mutex_init(&vdev->vf_token->lock);
1955 uuid_gen(&vdev->vf_token->uuid);
1957 vdev->nb.notifier_call = vfio_pci_bus_notifier;
1958 ret = bus_register_notifier(&pci_bus_type, &vdev->nb);
1960 kfree(vdev->vf_token);
1966 static void vfio_pci_vf_uninit(struct vfio_pci_device *vdev)
1968 if (!vdev->vf_token)
1971 bus_unregister_notifier(&pci_bus_type, &vdev->nb);
1972 WARN_ON(vdev->vf_token->users);
1973 mutex_destroy(&vdev->vf_token->lock);
1974 kfree(vdev->vf_token);
1977 static int vfio_pci_vga_init(struct vfio_pci_device *vdev)
1979 struct pci_dev *pdev = vdev->pdev;
1985 ret = vga_client_register(pdev, vdev, NULL, vfio_pci_set_vga_decode);
1988 vga_set_legacy_decoding(pdev, vfio_pci_set_vga_decode(vdev, false));
1992 static void vfio_pci_vga_uninit(struct vfio_pci_device *vdev)
1994 struct pci_dev *pdev = vdev->pdev;
2006 struct vfio_pci_device *vdev;
2033 vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
2034 if (!vdev) {
2039 vdev->pdev = pdev;
2040 vdev->irq_type = VFIO_PCI_NUM_IRQS;
2041 mutex_init(&vdev->igate);
2042 spin_lock_init(&vdev->irqlock);
2043 mutex_init(&vdev->ioeventfds_lock);
2044 INIT_LIST_HEAD(&vdev->dummy_resources_list);
2045 INIT_LIST_HEAD(&vdev->ioeventfds_list);
2046 mutex_init(&vdev->vma_lock);
2047 INIT_LIST_HEAD(&vdev->vma_list);
2048 init_rwsem(&vdev->memory_lock);
2050 ret = vfio_pci_reflck_attach(vdev);
2053 ret = vfio_pci_vf_init(vdev);
2056 ret = vfio_pci_vga_init(vdev);
2060 vfio_pci_probe_power_state(vdev);
2072 vfio_pci_set_power_state(vdev, PCI_D0);
2073 vfio_pci_set_power_state(vdev, PCI_D3hot);
2076 ret = vfio_add_group_dev(&pdev->dev, &vfio_pci_ops, vdev);
2083 vfio_pci_set_power_state(vdev, PCI_D0);
2085 vfio_pci_vf_uninit(vdev);
2087 vfio_pci_reflck_put(vdev->reflck);
2089 kfree(vdev->pm_save);
2090 kfree(vdev);
2098 struct vfio_pci_device *vdev;
2102 vdev = vfio_del_group_dev(&pdev->dev);
2103 if (!vdev)
2106 vfio_pci_vf_uninit(vdev);
2107 vfio_pci_reflck_put(vdev->reflck);
2108 vfio_pci_vga_uninit(vdev);
2113 vfio_pci_set_power_state(vdev, PCI_D0);
2115 mutex_destroy(&vdev->ioeventfds_lock);
2116 kfree(vdev->region);
2117 kfree(vdev->pm_save);
2118 kfree(vdev);
2124 struct vfio_pci_device *vdev;
2131 vdev = vfio_device_data(device);
2132 if (vdev == NULL) {
2137 mutex_lock(&vdev->igate);
2139 if (vdev->err_trigger)
2140 eventfd_signal(vdev->err_trigger, 1);
2142 mutex_unlock(&vdev->igate);
2151 struct vfio_pci_device *vdev;
2164 vdev = vfio_device_data(device);
2165 if (!vdev) {
2218 struct vfio_pci_device *vdev;
2229 vdev = vfio_device_data(device);
2231 if (vdev->reflck) {
2232 vfio_pci_reflck_get(vdev->reflck);
2233 *preflck = vdev->reflck;
2242 static int vfio_pci_reflck_attach(struct vfio_pci_device *vdev)
2244 bool slot = !pci_probe_reset_slot(vdev->pdev->slot);
2248 if (pci_is_root_bus(vdev->pdev->bus) ||
2249 vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_reflck_find,
2250 &vdev->reflck, slot) <= 0)
2251 vdev->reflck = vfio_pci_reflck_alloc();
2255 return PTR_ERR_OR_ZERO(vdev->reflck);
2277 struct vfio_pci_device *vdev;
2291 vdev = vfio_device_data(device);
2294 if (vdev->refcnt) {
2307 struct vfio_pci_device *vdev;
2321 vdev = vfio_device_data(device);
2327 if (!vfio_pci_zap_and_vma_lock(vdev, true)) {
2343 * to hold vdev->reflck->lock, protecting the bus/slot reset group from
2352 static void vfio_pci_try_bus_reset(struct vfio_pci_device *vdev)
2359 if (!pci_probe_reset_slot(vdev->pdev->slot))
2361 else if (pci_probe_reset_bus(vdev->pdev->bus))
2364 if (vfio_pci_for_each_slot_or_bus(vdev->pdev, vfio_pci_count_devs,
2373 if (vfio_pci_for_each_slot_or_bus(vdev->pdev,
2382 ret = pci_reset_bus(vdev->pdev);
2401 if (tmp != vdev && !disable_idle_d3)