1// SPDX-License-Identifier: GPL-2.0 2/* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. 3 * 4 * Kernel side components to support tools/testing/selftests/iommu 5 */ 6#include <linux/slab.h> 7#include <linux/iommu.h> 8#include <linux/xarray.h> 9#include <linux/file.h> 10#include <linux/anon_inodes.h> 11#include <linux/fault-inject.h> 12#include <linux/platform_device.h> 13#include <uapi/linux/iommufd.h> 14 15#include "../iommu-priv.h" 16#include "io_pagetable.h" 17#include "iommufd_private.h" 18#include "iommufd_test.h" 19 20static DECLARE_FAULT_ATTR(fail_iommufd); 21static struct dentry *dbgfs_root; 22static struct platform_device *selftest_iommu_dev; 23 24size_t iommufd_test_memory_limit = 65536; 25 26enum { 27 MOCK_IO_PAGE_SIZE = PAGE_SIZE / 2, 28 29 /* 30 * Like a real page table alignment requires the low bits of the address 31 * to be zero. xarray also requires the high bit to be zero, so we store 32 * the pfns shifted. The upper bits are used for metadata. 33 */ 34 MOCK_PFN_MASK = ULONG_MAX / MOCK_IO_PAGE_SIZE, 35 36 _MOCK_PFN_START = MOCK_PFN_MASK + 1, 37 MOCK_PFN_START_IOVA = _MOCK_PFN_START, 38 MOCK_PFN_LAST_IOVA = _MOCK_PFN_START, 39}; 40 41/* 42 * Syzkaller has trouble randomizing the correct iova to use since it is linked 43 * to the map ioctl's output, and it has no ide about that. So, simplify things. 44 * In syzkaller mode the 64 bit IOVA is converted into an nth area and offset 45 * value. This has a much smaller randomization space and syzkaller can hit it. 46 */ 47static unsigned long iommufd_test_syz_conv_iova(struct io_pagetable *iopt, 48 u64 *iova) 49{ 50 struct syz_layout { 51 __u32 nth_area; 52 __u32 offset; 53 }; 54 struct syz_layout *syz = (void *)iova; 55 unsigned int nth = syz->nth_area; 56 struct iopt_area *area; 57 58 down_read(&iopt->iova_rwsem); 59 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area; 60 area = iopt_area_iter_next(area, 0, ULONG_MAX)) { 61 if (nth == 0) { 62 up_read(&iopt->iova_rwsem); 63 return iopt_area_iova(area) + syz->offset; 64 } 65 nth--; 66 } 67 up_read(&iopt->iova_rwsem); 68 69 return 0; 70} 71 72void iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, 73 unsigned int ioas_id, u64 *iova, u32 *flags) 74{ 75 struct iommufd_ioas *ioas; 76 77 if (!(*flags & MOCK_FLAGS_ACCESS_SYZ)) 78 return; 79 *flags &= ~(u32)MOCK_FLAGS_ACCESS_SYZ; 80 81 ioas = iommufd_get_ioas(ucmd->ictx, ioas_id); 82 if (IS_ERR(ioas)) 83 return; 84 *iova = iommufd_test_syz_conv_iova(&ioas->iopt, iova); 85 iommufd_put_object(&ioas->obj); 86} 87 88struct mock_iommu_domain { 89 struct iommu_domain domain; 90 struct xarray pfns; 91}; 92 93enum selftest_obj_type { 94 TYPE_IDEV, 95}; 96 97struct mock_dev { 98 struct device dev; 99}; 100 101struct selftest_obj { 102 struct iommufd_object obj; 103 enum selftest_obj_type type; 104 105 union { 106 struct { 107 struct iommufd_device *idev; 108 struct iommufd_ctx *ictx; 109 struct mock_dev *mock_dev; 110 } idev; 111 }; 112}; 113 114static void mock_domain_blocking_free(struct iommu_domain *domain) 115{ 116} 117 118static int mock_domain_nop_attach(struct iommu_domain *domain, 119 struct device *dev) 120{ 121 return 0; 122} 123 124static const struct iommu_domain_ops mock_blocking_ops = { 125 .free = mock_domain_blocking_free, 126 .attach_dev = mock_domain_nop_attach, 127}; 128 129static struct iommu_domain mock_blocking_domain = { 130 .type = IOMMU_DOMAIN_BLOCKED, 131 .ops = &mock_blocking_ops, 132}; 133 134static void *mock_domain_hw_info(struct device *dev, u32 *length, u32 *type) 135{ 136 struct iommu_test_hw_info *info; 137 138 info = kzalloc(sizeof(*info), GFP_KERNEL); 139 if (!info) 140 return ERR_PTR(-ENOMEM); 141 142 info->test_reg = IOMMU_HW_INFO_SELFTEST_REGVAL; 143 *length = sizeof(*info); 144 *type = IOMMU_HW_INFO_TYPE_SELFTEST; 145 146 return info; 147} 148 149static struct iommu_domain *mock_domain_alloc(unsigned int iommu_domain_type) 150{ 151 struct mock_iommu_domain *mock; 152 153 if (iommu_domain_type == IOMMU_DOMAIN_BLOCKED) 154 return &mock_blocking_domain; 155 156 if (iommu_domain_type != IOMMU_DOMAIN_UNMANAGED) 157 return NULL; 158 159 mock = kzalloc(sizeof(*mock), GFP_KERNEL); 160 if (!mock) 161 return NULL; 162 mock->domain.geometry.aperture_start = MOCK_APERTURE_START; 163 mock->domain.geometry.aperture_end = MOCK_APERTURE_LAST; 164 mock->domain.pgsize_bitmap = MOCK_IO_PAGE_SIZE; 165 xa_init(&mock->pfns); 166 return &mock->domain; 167} 168 169static void mock_domain_free(struct iommu_domain *domain) 170{ 171 struct mock_iommu_domain *mock = 172 container_of(domain, struct mock_iommu_domain, domain); 173 174 WARN_ON(!xa_empty(&mock->pfns)); 175 kfree(mock); 176} 177 178static int mock_domain_map_pages(struct iommu_domain *domain, 179 unsigned long iova, phys_addr_t paddr, 180 size_t pgsize, size_t pgcount, int prot, 181 gfp_t gfp, size_t *mapped) 182{ 183 struct mock_iommu_domain *mock = 184 container_of(domain, struct mock_iommu_domain, domain); 185 unsigned long flags = MOCK_PFN_START_IOVA; 186 unsigned long start_iova = iova; 187 188 /* 189 * xarray does not reliably work with fault injection because it does a 190 * retry allocation, so put our own failure point. 191 */ 192 if (iommufd_should_fail()) 193 return -ENOENT; 194 195 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 196 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 197 for (; pgcount; pgcount--) { 198 size_t cur; 199 200 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 201 void *old; 202 203 if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize) 204 flags = MOCK_PFN_LAST_IOVA; 205 old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZE, 206 xa_mk_value((paddr / MOCK_IO_PAGE_SIZE) | 207 flags), 208 gfp); 209 if (xa_is_err(old)) { 210 for (; start_iova != iova; 211 start_iova += MOCK_IO_PAGE_SIZE) 212 xa_erase(&mock->pfns, 213 start_iova / 214 MOCK_IO_PAGE_SIZE); 215 return xa_err(old); 216 } 217 WARN_ON(old); 218 iova += MOCK_IO_PAGE_SIZE; 219 paddr += MOCK_IO_PAGE_SIZE; 220 *mapped += MOCK_IO_PAGE_SIZE; 221 flags = 0; 222 } 223 } 224 return 0; 225} 226 227static size_t mock_domain_unmap_pages(struct iommu_domain *domain, 228 unsigned long iova, size_t pgsize, 229 size_t pgcount, 230 struct iommu_iotlb_gather *iotlb_gather) 231{ 232 struct mock_iommu_domain *mock = 233 container_of(domain, struct mock_iommu_domain, domain); 234 bool first = true; 235 size_t ret = 0; 236 void *ent; 237 238 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 239 WARN_ON(pgsize % MOCK_IO_PAGE_SIZE); 240 241 for (; pgcount; pgcount--) { 242 size_t cur; 243 244 for (cur = 0; cur != pgsize; cur += MOCK_IO_PAGE_SIZE) { 245 ent = xa_erase(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 246 WARN_ON(!ent); 247 /* 248 * iommufd generates unmaps that must be a strict 249 * superset of the map's performend So every starting 250 * IOVA should have been an iova passed to map, and the 251 * 252 * First IOVA must be present and have been a first IOVA 253 * passed to map_pages 254 */ 255 if (first) { 256 WARN_ON(!(xa_to_value(ent) & 257 MOCK_PFN_START_IOVA)); 258 first = false; 259 } 260 if (pgcount == 1 && cur + MOCK_IO_PAGE_SIZE == pgsize) 261 WARN_ON(!(xa_to_value(ent) & 262 MOCK_PFN_LAST_IOVA)); 263 264 iova += MOCK_IO_PAGE_SIZE; 265 ret += MOCK_IO_PAGE_SIZE; 266 } 267 } 268 return ret; 269} 270 271static phys_addr_t mock_domain_iova_to_phys(struct iommu_domain *domain, 272 dma_addr_t iova) 273{ 274 struct mock_iommu_domain *mock = 275 container_of(domain, struct mock_iommu_domain, domain); 276 void *ent; 277 278 WARN_ON(iova % MOCK_IO_PAGE_SIZE); 279 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 280 WARN_ON(!ent); 281 return (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE; 282} 283 284static bool mock_domain_capable(struct device *dev, enum iommu_cap cap) 285{ 286 return cap == IOMMU_CAP_CACHE_COHERENCY; 287} 288 289static void mock_domain_set_plaform_dma_ops(struct device *dev) 290{ 291 /* 292 * mock doesn't setup default domains because we can't hook into the 293 * normal probe path 294 */ 295} 296 297static struct iommu_device mock_iommu_device = { 298}; 299 300static struct iommu_device *mock_probe_device(struct device *dev) 301{ 302 return &mock_iommu_device; 303} 304 305static const struct iommu_ops mock_ops = { 306 .owner = THIS_MODULE, 307 .pgsize_bitmap = MOCK_IO_PAGE_SIZE, 308 .hw_info = mock_domain_hw_info, 309 .domain_alloc = mock_domain_alloc, 310 .capable = mock_domain_capable, 311 .set_platform_dma_ops = mock_domain_set_plaform_dma_ops, 312 .device_group = generic_device_group, 313 .probe_device = mock_probe_device, 314 .default_domain_ops = 315 &(struct iommu_domain_ops){ 316 .free = mock_domain_free, 317 .attach_dev = mock_domain_nop_attach, 318 .map_pages = mock_domain_map_pages, 319 .unmap_pages = mock_domain_unmap_pages, 320 .iova_to_phys = mock_domain_iova_to_phys, 321 }, 322}; 323 324static inline struct iommufd_hw_pagetable * 325get_md_pagetable(struct iommufd_ucmd *ucmd, u32 mockpt_id, 326 struct mock_iommu_domain **mock) 327{ 328 struct iommufd_hw_pagetable *hwpt; 329 struct iommufd_object *obj; 330 331 obj = iommufd_get_object(ucmd->ictx, mockpt_id, 332 IOMMUFD_OBJ_HW_PAGETABLE); 333 if (IS_ERR(obj)) 334 return ERR_CAST(obj); 335 hwpt = container_of(obj, struct iommufd_hw_pagetable, obj); 336 if (hwpt->domain->ops != mock_ops.default_domain_ops) { 337 iommufd_put_object(&hwpt->obj); 338 return ERR_PTR(-EINVAL); 339 } 340 *mock = container_of(hwpt->domain, struct mock_iommu_domain, domain); 341 return hwpt; 342} 343 344struct mock_bus_type { 345 struct bus_type bus; 346 struct notifier_block nb; 347}; 348 349static struct mock_bus_type iommufd_mock_bus_type = { 350 .bus = { 351 .name = "iommufd_mock", 352 }, 353}; 354 355static atomic_t mock_dev_num; 356 357static void mock_dev_release(struct device *dev) 358{ 359 struct mock_dev *mdev = container_of(dev, struct mock_dev, dev); 360 361 atomic_dec(&mock_dev_num); 362 kfree(mdev); 363} 364 365static struct mock_dev *mock_dev_create(void) 366{ 367 struct mock_dev *mdev; 368 int rc; 369 370 mdev = kzalloc(sizeof(*mdev), GFP_KERNEL); 371 if (!mdev) 372 return ERR_PTR(-ENOMEM); 373 374 device_initialize(&mdev->dev); 375 mdev->dev.release = mock_dev_release; 376 mdev->dev.bus = &iommufd_mock_bus_type.bus; 377 378 rc = dev_set_name(&mdev->dev, "iommufd_mock%u", 379 atomic_inc_return(&mock_dev_num)); 380 if (rc) 381 goto err_put; 382 383 rc = device_add(&mdev->dev); 384 if (rc) 385 goto err_put; 386 return mdev; 387 388err_put: 389 put_device(&mdev->dev); 390 return ERR_PTR(rc); 391} 392 393static void mock_dev_destroy(struct mock_dev *mdev) 394{ 395 device_unregister(&mdev->dev); 396} 397 398bool iommufd_selftest_is_mock_dev(struct device *dev) 399{ 400 return dev->release == mock_dev_release; 401} 402 403/* Create an hw_pagetable with the mock domain so we can test the domain ops */ 404static int iommufd_test_mock_domain(struct iommufd_ucmd *ucmd, 405 struct iommu_test_cmd *cmd) 406{ 407 struct iommufd_device *idev; 408 struct selftest_obj *sobj; 409 u32 pt_id = cmd->id; 410 u32 idev_id; 411 int rc; 412 413 sobj = iommufd_object_alloc(ucmd->ictx, sobj, IOMMUFD_OBJ_SELFTEST); 414 if (IS_ERR(sobj)) 415 return PTR_ERR(sobj); 416 417 sobj->idev.ictx = ucmd->ictx; 418 sobj->type = TYPE_IDEV; 419 420 sobj->idev.mock_dev = mock_dev_create(); 421 if (IS_ERR(sobj->idev.mock_dev)) { 422 rc = PTR_ERR(sobj->idev.mock_dev); 423 goto out_sobj; 424 } 425 426 idev = iommufd_device_bind(ucmd->ictx, &sobj->idev.mock_dev->dev, 427 &idev_id); 428 if (IS_ERR(idev)) { 429 rc = PTR_ERR(idev); 430 goto out_mdev; 431 } 432 sobj->idev.idev = idev; 433 434 rc = iommufd_device_attach(idev, &pt_id); 435 if (rc) 436 goto out_unbind; 437 438 /* Userspace must destroy the device_id to destroy the object */ 439 cmd->mock_domain.out_hwpt_id = pt_id; 440 cmd->mock_domain.out_stdev_id = sobj->obj.id; 441 cmd->mock_domain.out_idev_id = idev_id; 442 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 443 if (rc) 444 goto out_detach; 445 iommufd_object_finalize(ucmd->ictx, &sobj->obj); 446 return 0; 447 448out_detach: 449 iommufd_device_detach(idev); 450out_unbind: 451 iommufd_device_unbind(idev); 452out_mdev: 453 mock_dev_destroy(sobj->idev.mock_dev); 454out_sobj: 455 iommufd_object_abort(ucmd->ictx, &sobj->obj); 456 return rc; 457} 458 459/* Replace the mock domain with a manually allocated hw_pagetable */ 460static int iommufd_test_mock_domain_replace(struct iommufd_ucmd *ucmd, 461 unsigned int device_id, u32 pt_id, 462 struct iommu_test_cmd *cmd) 463{ 464 struct iommufd_object *dev_obj; 465 struct selftest_obj *sobj; 466 int rc; 467 468 /* 469 * Prefer to use the OBJ_SELFTEST because the destroy_rwsem will ensure 470 * it doesn't race with detach, which is not allowed. 471 */ 472 dev_obj = 473 iommufd_get_object(ucmd->ictx, device_id, IOMMUFD_OBJ_SELFTEST); 474 if (IS_ERR(dev_obj)) 475 return PTR_ERR(dev_obj); 476 477 sobj = container_of(dev_obj, struct selftest_obj, obj); 478 if (sobj->type != TYPE_IDEV) { 479 rc = -EINVAL; 480 goto out_dev_obj; 481 } 482 483 rc = iommufd_device_replace(sobj->idev.idev, &pt_id); 484 if (rc) 485 goto out_dev_obj; 486 487 cmd->mock_domain_replace.pt_id = pt_id; 488 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 489 490out_dev_obj: 491 iommufd_put_object(dev_obj); 492 return rc; 493} 494 495/* Add an additional reserved IOVA to the IOAS */ 496static int iommufd_test_add_reserved(struct iommufd_ucmd *ucmd, 497 unsigned int mockpt_id, 498 unsigned long start, size_t length) 499{ 500 struct iommufd_ioas *ioas; 501 int rc; 502 503 ioas = iommufd_get_ioas(ucmd->ictx, mockpt_id); 504 if (IS_ERR(ioas)) 505 return PTR_ERR(ioas); 506 down_write(&ioas->iopt.iova_rwsem); 507 rc = iopt_reserve_iova(&ioas->iopt, start, start + length - 1, NULL); 508 up_write(&ioas->iopt.iova_rwsem); 509 iommufd_put_object(&ioas->obj); 510 return rc; 511} 512 513/* Check that every pfn under each iova matches the pfn under a user VA */ 514static int iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, 515 unsigned int mockpt_id, unsigned long iova, 516 size_t length, void __user *uptr) 517{ 518 struct iommufd_hw_pagetable *hwpt; 519 struct mock_iommu_domain *mock; 520 uintptr_t end; 521 int rc; 522 523 if (iova % MOCK_IO_PAGE_SIZE || length % MOCK_IO_PAGE_SIZE || 524 (uintptr_t)uptr % MOCK_IO_PAGE_SIZE || 525 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 526 return -EINVAL; 527 528 hwpt = get_md_pagetable(ucmd, mockpt_id, &mock); 529 if (IS_ERR(hwpt)) 530 return PTR_ERR(hwpt); 531 532 for (; length; length -= MOCK_IO_PAGE_SIZE) { 533 struct page *pages[1]; 534 unsigned long pfn; 535 long npages; 536 void *ent; 537 538 npages = get_user_pages_fast((uintptr_t)uptr & PAGE_MASK, 1, 0, 539 pages); 540 if (npages < 0) { 541 rc = npages; 542 goto out_put; 543 } 544 if (WARN_ON(npages != 1)) { 545 rc = -EFAULT; 546 goto out_put; 547 } 548 pfn = page_to_pfn(pages[0]); 549 put_page(pages[0]); 550 551 ent = xa_load(&mock->pfns, iova / MOCK_IO_PAGE_SIZE); 552 if (!ent || 553 (xa_to_value(ent) & MOCK_PFN_MASK) * MOCK_IO_PAGE_SIZE != 554 pfn * PAGE_SIZE + ((uintptr_t)uptr % PAGE_SIZE)) { 555 rc = -EINVAL; 556 goto out_put; 557 } 558 iova += MOCK_IO_PAGE_SIZE; 559 uptr += MOCK_IO_PAGE_SIZE; 560 } 561 rc = 0; 562 563out_put: 564 iommufd_put_object(&hwpt->obj); 565 return rc; 566} 567 568/* Check that the page ref count matches, to look for missing pin/unpins */ 569static int iommufd_test_md_check_refs(struct iommufd_ucmd *ucmd, 570 void __user *uptr, size_t length, 571 unsigned int refs) 572{ 573 uintptr_t end; 574 575 if (length % PAGE_SIZE || (uintptr_t)uptr % PAGE_SIZE || 576 check_add_overflow((uintptr_t)uptr, (uintptr_t)length, &end)) 577 return -EINVAL; 578 579 for (; length; length -= PAGE_SIZE) { 580 struct page *pages[1]; 581 long npages; 582 583 npages = get_user_pages_fast((uintptr_t)uptr, 1, 0, pages); 584 if (npages < 0) 585 return npages; 586 if (WARN_ON(npages != 1)) 587 return -EFAULT; 588 if (!PageCompound(pages[0])) { 589 unsigned int count; 590 591 count = page_ref_count(pages[0]); 592 if (count / GUP_PIN_COUNTING_BIAS != refs) { 593 put_page(pages[0]); 594 return -EIO; 595 } 596 } 597 put_page(pages[0]); 598 uptr += PAGE_SIZE; 599 } 600 return 0; 601} 602 603struct selftest_access { 604 struct iommufd_access *access; 605 struct file *file; 606 struct mutex lock; 607 struct list_head items; 608 unsigned int next_id; 609 bool destroying; 610}; 611 612struct selftest_access_item { 613 struct list_head items_elm; 614 unsigned long iova; 615 size_t length; 616 unsigned int id; 617}; 618 619static const struct file_operations iommfd_test_staccess_fops; 620 621static struct selftest_access *iommufd_access_get(int fd) 622{ 623 struct file *file; 624 625 file = fget(fd); 626 if (!file) 627 return ERR_PTR(-EBADFD); 628 629 if (file->f_op != &iommfd_test_staccess_fops) { 630 fput(file); 631 return ERR_PTR(-EBADFD); 632 } 633 return file->private_data; 634} 635 636static void iommufd_test_access_unmap(void *data, unsigned long iova, 637 unsigned long length) 638{ 639 unsigned long iova_last = iova + length - 1; 640 struct selftest_access *staccess = data; 641 struct selftest_access_item *item; 642 struct selftest_access_item *tmp; 643 644 mutex_lock(&staccess->lock); 645 list_for_each_entry_safe(item, tmp, &staccess->items, items_elm) { 646 if (iova > item->iova + item->length - 1 || 647 iova_last < item->iova) 648 continue; 649 list_del(&item->items_elm); 650 iommufd_access_unpin_pages(staccess->access, item->iova, 651 item->length); 652 kfree(item); 653 } 654 mutex_unlock(&staccess->lock); 655} 656 657static int iommufd_test_access_item_destroy(struct iommufd_ucmd *ucmd, 658 unsigned int access_id, 659 unsigned int item_id) 660{ 661 struct selftest_access_item *item; 662 struct selftest_access *staccess; 663 664 staccess = iommufd_access_get(access_id); 665 if (IS_ERR(staccess)) 666 return PTR_ERR(staccess); 667 668 mutex_lock(&staccess->lock); 669 list_for_each_entry(item, &staccess->items, items_elm) { 670 if (item->id == item_id) { 671 list_del(&item->items_elm); 672 iommufd_access_unpin_pages(staccess->access, item->iova, 673 item->length); 674 mutex_unlock(&staccess->lock); 675 kfree(item); 676 fput(staccess->file); 677 return 0; 678 } 679 } 680 mutex_unlock(&staccess->lock); 681 fput(staccess->file); 682 return -ENOENT; 683} 684 685static int iommufd_test_staccess_release(struct inode *inode, 686 struct file *filep) 687{ 688 struct selftest_access *staccess = filep->private_data; 689 690 if (staccess->access) { 691 iommufd_test_access_unmap(staccess, 0, ULONG_MAX); 692 iommufd_access_destroy(staccess->access); 693 } 694 mutex_destroy(&staccess->lock); 695 kfree(staccess); 696 return 0; 697} 698 699static const struct iommufd_access_ops selftest_access_ops_pin = { 700 .needs_pin_pages = 1, 701 .unmap = iommufd_test_access_unmap, 702}; 703 704static const struct iommufd_access_ops selftest_access_ops = { 705 .unmap = iommufd_test_access_unmap, 706}; 707 708static const struct file_operations iommfd_test_staccess_fops = { 709 .release = iommufd_test_staccess_release, 710}; 711 712static struct selftest_access *iommufd_test_alloc_access(void) 713{ 714 struct selftest_access *staccess; 715 struct file *filep; 716 717 staccess = kzalloc(sizeof(*staccess), GFP_KERNEL_ACCOUNT); 718 if (!staccess) 719 return ERR_PTR(-ENOMEM); 720 INIT_LIST_HEAD(&staccess->items); 721 mutex_init(&staccess->lock); 722 723 filep = anon_inode_getfile("[iommufd_test_staccess]", 724 &iommfd_test_staccess_fops, staccess, 725 O_RDWR); 726 if (IS_ERR(filep)) { 727 kfree(staccess); 728 return ERR_CAST(filep); 729 } 730 staccess->file = filep; 731 return staccess; 732} 733 734static int iommufd_test_create_access(struct iommufd_ucmd *ucmd, 735 unsigned int ioas_id, unsigned int flags) 736{ 737 struct iommu_test_cmd *cmd = ucmd->cmd; 738 struct selftest_access *staccess; 739 struct iommufd_access *access; 740 u32 id; 741 int fdno; 742 int rc; 743 744 if (flags & ~MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) 745 return -EOPNOTSUPP; 746 747 staccess = iommufd_test_alloc_access(); 748 if (IS_ERR(staccess)) 749 return PTR_ERR(staccess); 750 751 fdno = get_unused_fd_flags(O_CLOEXEC); 752 if (fdno < 0) { 753 rc = -ENOMEM; 754 goto out_free_staccess; 755 } 756 757 access = iommufd_access_create( 758 ucmd->ictx, 759 (flags & MOCK_FLAGS_ACCESS_CREATE_NEEDS_PIN_PAGES) ? 760 &selftest_access_ops_pin : 761 &selftest_access_ops, 762 staccess, &id); 763 if (IS_ERR(access)) { 764 rc = PTR_ERR(access); 765 goto out_put_fdno; 766 } 767 rc = iommufd_access_attach(access, ioas_id); 768 if (rc) 769 goto out_destroy; 770 cmd->create_access.out_access_fd = fdno; 771 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 772 if (rc) 773 goto out_destroy; 774 775 staccess->access = access; 776 fd_install(fdno, staccess->file); 777 return 0; 778 779out_destroy: 780 iommufd_access_destroy(access); 781out_put_fdno: 782 put_unused_fd(fdno); 783out_free_staccess: 784 fput(staccess->file); 785 return rc; 786} 787 788static int iommufd_test_access_replace_ioas(struct iommufd_ucmd *ucmd, 789 unsigned int access_id, 790 unsigned int ioas_id) 791{ 792 struct selftest_access *staccess; 793 int rc; 794 795 staccess = iommufd_access_get(access_id); 796 if (IS_ERR(staccess)) 797 return PTR_ERR(staccess); 798 799 rc = iommufd_access_replace(staccess->access, ioas_id); 800 fput(staccess->file); 801 return rc; 802} 803 804/* Check that the pages in a page array match the pages in the user VA */ 805static int iommufd_test_check_pages(void __user *uptr, struct page **pages, 806 size_t npages) 807{ 808 for (; npages; npages--) { 809 struct page *tmp_pages[1]; 810 long rc; 811 812 rc = get_user_pages_fast((uintptr_t)uptr, 1, 0, tmp_pages); 813 if (rc < 0) 814 return rc; 815 if (WARN_ON(rc != 1)) 816 return -EFAULT; 817 put_page(tmp_pages[0]); 818 if (tmp_pages[0] != *pages) 819 return -EBADE; 820 pages++; 821 uptr += PAGE_SIZE; 822 } 823 return 0; 824} 825 826static int iommufd_test_access_pages(struct iommufd_ucmd *ucmd, 827 unsigned int access_id, unsigned long iova, 828 size_t length, void __user *uptr, 829 u32 flags) 830{ 831 struct iommu_test_cmd *cmd = ucmd->cmd; 832 struct selftest_access_item *item; 833 struct selftest_access *staccess; 834 struct page **pages; 835 size_t npages; 836 int rc; 837 838 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 839 if (length > 16*1024*1024) 840 return -ENOMEM; 841 842 if (flags & ~(MOCK_FLAGS_ACCESS_WRITE | MOCK_FLAGS_ACCESS_SYZ)) 843 return -EOPNOTSUPP; 844 845 staccess = iommufd_access_get(access_id); 846 if (IS_ERR(staccess)) 847 return PTR_ERR(staccess); 848 849 if (staccess->access->ops != &selftest_access_ops_pin) { 850 rc = -EOPNOTSUPP; 851 goto out_put; 852 } 853 854 if (flags & MOCK_FLAGS_ACCESS_SYZ) 855 iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt, 856 &cmd->access_pages.iova); 857 858 npages = (ALIGN(iova + length, PAGE_SIZE) - 859 ALIGN_DOWN(iova, PAGE_SIZE)) / 860 PAGE_SIZE; 861 pages = kvcalloc(npages, sizeof(*pages), GFP_KERNEL_ACCOUNT); 862 if (!pages) { 863 rc = -ENOMEM; 864 goto out_put; 865 } 866 867 /* 868 * Drivers will need to think very carefully about this locking. The 869 * core code can do multiple unmaps instantaneously after 870 * iommufd_access_pin_pages() and *all* the unmaps must not return until 871 * the range is unpinned. This simple implementation puts a global lock 872 * around the pin, which may not suit drivers that want this to be a 873 * performance path. drivers that get this wrong will trigger WARN_ON 874 * races and cause EDEADLOCK failures to userspace. 875 */ 876 mutex_lock(&staccess->lock); 877 rc = iommufd_access_pin_pages(staccess->access, iova, length, pages, 878 flags & MOCK_FLAGS_ACCESS_WRITE); 879 if (rc) 880 goto out_unlock; 881 882 /* For syzkaller allow uptr to be NULL to skip this check */ 883 if (uptr) { 884 rc = iommufd_test_check_pages( 885 uptr - (iova - ALIGN_DOWN(iova, PAGE_SIZE)), pages, 886 npages); 887 if (rc) 888 goto out_unaccess; 889 } 890 891 item = kzalloc(sizeof(*item), GFP_KERNEL_ACCOUNT); 892 if (!item) { 893 rc = -ENOMEM; 894 goto out_unaccess; 895 } 896 897 item->iova = iova; 898 item->length = length; 899 item->id = staccess->next_id++; 900 list_add_tail(&item->items_elm, &staccess->items); 901 902 cmd->access_pages.out_access_pages_id = item->id; 903 rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd)); 904 if (rc) 905 goto out_free_item; 906 goto out_unlock; 907 908out_free_item: 909 list_del(&item->items_elm); 910 kfree(item); 911out_unaccess: 912 iommufd_access_unpin_pages(staccess->access, iova, length); 913out_unlock: 914 mutex_unlock(&staccess->lock); 915 kvfree(pages); 916out_put: 917 fput(staccess->file); 918 return rc; 919} 920 921static int iommufd_test_access_rw(struct iommufd_ucmd *ucmd, 922 unsigned int access_id, unsigned long iova, 923 size_t length, void __user *ubuf, 924 unsigned int flags) 925{ 926 struct iommu_test_cmd *cmd = ucmd->cmd; 927 struct selftest_access *staccess; 928 void *tmp; 929 int rc; 930 931 /* Prevent syzkaller from triggering a WARN_ON in kvzalloc() */ 932 if (length > 16*1024*1024) 933 return -ENOMEM; 934 935 if (flags & ~(MOCK_ACCESS_RW_WRITE | MOCK_ACCESS_RW_SLOW_PATH | 936 MOCK_FLAGS_ACCESS_SYZ)) 937 return -EOPNOTSUPP; 938 939 staccess = iommufd_access_get(access_id); 940 if (IS_ERR(staccess)) 941 return PTR_ERR(staccess); 942 943 tmp = kvzalloc(length, GFP_KERNEL_ACCOUNT); 944 if (!tmp) { 945 rc = -ENOMEM; 946 goto out_put; 947 } 948 949 if (flags & MOCK_ACCESS_RW_WRITE) { 950 if (copy_from_user(tmp, ubuf, length)) { 951 rc = -EFAULT; 952 goto out_free; 953 } 954 } 955 956 if (flags & MOCK_FLAGS_ACCESS_SYZ) 957 iova = iommufd_test_syz_conv_iova(&staccess->access->ioas->iopt, 958 &cmd->access_rw.iova); 959 960 rc = iommufd_access_rw(staccess->access, iova, tmp, length, flags); 961 if (rc) 962 goto out_free; 963 if (!(flags & MOCK_ACCESS_RW_WRITE)) { 964 if (copy_to_user(ubuf, tmp, length)) { 965 rc = -EFAULT; 966 goto out_free; 967 } 968 } 969 970out_free: 971 kvfree(tmp); 972out_put: 973 fput(staccess->file); 974 return rc; 975} 976static_assert((unsigned int)MOCK_ACCESS_RW_WRITE == IOMMUFD_ACCESS_RW_WRITE); 977static_assert((unsigned int)MOCK_ACCESS_RW_SLOW_PATH == 978 __IOMMUFD_ACCESS_RW_SLOW_PATH); 979 980void iommufd_selftest_destroy(struct iommufd_object *obj) 981{ 982 struct selftest_obj *sobj = container_of(obj, struct selftest_obj, obj); 983 984 switch (sobj->type) { 985 case TYPE_IDEV: 986 iommufd_device_detach(sobj->idev.idev); 987 iommufd_device_unbind(sobj->idev.idev); 988 mock_dev_destroy(sobj->idev.mock_dev); 989 break; 990 } 991} 992 993int iommufd_test(struct iommufd_ucmd *ucmd) 994{ 995 struct iommu_test_cmd *cmd = ucmd->cmd; 996 997 switch (cmd->op) { 998 case IOMMU_TEST_OP_ADD_RESERVED: 999 return iommufd_test_add_reserved(ucmd, cmd->id, 1000 cmd->add_reserved.start, 1001 cmd->add_reserved.length); 1002 case IOMMU_TEST_OP_MOCK_DOMAIN: 1003 return iommufd_test_mock_domain(ucmd, cmd); 1004 case IOMMU_TEST_OP_MOCK_DOMAIN_REPLACE: 1005 return iommufd_test_mock_domain_replace( 1006 ucmd, cmd->id, cmd->mock_domain_replace.pt_id, cmd); 1007 case IOMMU_TEST_OP_MD_CHECK_MAP: 1008 return iommufd_test_md_check_pa( 1009 ucmd, cmd->id, cmd->check_map.iova, 1010 cmd->check_map.length, 1011 u64_to_user_ptr(cmd->check_map.uptr)); 1012 case IOMMU_TEST_OP_MD_CHECK_REFS: 1013 return iommufd_test_md_check_refs( 1014 ucmd, u64_to_user_ptr(cmd->check_refs.uptr), 1015 cmd->check_refs.length, cmd->check_refs.refs); 1016 case IOMMU_TEST_OP_CREATE_ACCESS: 1017 return iommufd_test_create_access(ucmd, cmd->id, 1018 cmd->create_access.flags); 1019 case IOMMU_TEST_OP_ACCESS_REPLACE_IOAS: 1020 return iommufd_test_access_replace_ioas( 1021 ucmd, cmd->id, cmd->access_replace_ioas.ioas_id); 1022 case IOMMU_TEST_OP_ACCESS_PAGES: 1023 return iommufd_test_access_pages( 1024 ucmd, cmd->id, cmd->access_pages.iova, 1025 cmd->access_pages.length, 1026 u64_to_user_ptr(cmd->access_pages.uptr), 1027 cmd->access_pages.flags); 1028 case IOMMU_TEST_OP_ACCESS_RW: 1029 return iommufd_test_access_rw( 1030 ucmd, cmd->id, cmd->access_rw.iova, 1031 cmd->access_rw.length, 1032 u64_to_user_ptr(cmd->access_rw.uptr), 1033 cmd->access_rw.flags); 1034 case IOMMU_TEST_OP_DESTROY_ACCESS_PAGES: 1035 return iommufd_test_access_item_destroy( 1036 ucmd, cmd->id, cmd->destroy_access_pages.access_pages_id); 1037 case IOMMU_TEST_OP_SET_TEMP_MEMORY_LIMIT: 1038 /* Protect _batch_init(), can not be less than elmsz */ 1039 if (cmd->memory_limit.limit < 1040 sizeof(unsigned long) + sizeof(u32)) 1041 return -EINVAL; 1042 iommufd_test_memory_limit = cmd->memory_limit.limit; 1043 return 0; 1044 default: 1045 return -EOPNOTSUPP; 1046 } 1047} 1048 1049bool iommufd_should_fail(void) 1050{ 1051 return should_fail(&fail_iommufd, 1); 1052} 1053 1054int __init iommufd_test_init(void) 1055{ 1056 struct platform_device_info pdevinfo = { 1057 .name = "iommufd_selftest_iommu", 1058 }; 1059 int rc; 1060 1061 dbgfs_root = 1062 fault_create_debugfs_attr("fail_iommufd", NULL, &fail_iommufd); 1063 1064 selftest_iommu_dev = platform_device_register_full(&pdevinfo); 1065 if (IS_ERR(selftest_iommu_dev)) { 1066 rc = PTR_ERR(selftest_iommu_dev); 1067 goto err_dbgfs; 1068 } 1069 1070 rc = bus_register(&iommufd_mock_bus_type.bus); 1071 if (rc) 1072 goto err_platform; 1073 1074 rc = iommu_device_sysfs_add(&mock_iommu_device, 1075 &selftest_iommu_dev->dev, NULL, "%s", 1076 dev_name(&selftest_iommu_dev->dev)); 1077 if (rc) 1078 goto err_bus; 1079 1080 rc = iommu_device_register_bus(&mock_iommu_device, &mock_ops, 1081 &iommufd_mock_bus_type.bus, 1082 &iommufd_mock_bus_type.nb); 1083 if (rc) 1084 goto err_sysfs; 1085 return 0; 1086 1087err_sysfs: 1088 iommu_device_sysfs_remove(&mock_iommu_device); 1089err_bus: 1090 bus_unregister(&iommufd_mock_bus_type.bus); 1091err_platform: 1092 platform_device_unregister(selftest_iommu_dev); 1093err_dbgfs: 1094 debugfs_remove_recursive(dbgfs_root); 1095 return rc; 1096} 1097 1098void iommufd_test_exit(void) 1099{ 1100 iommu_device_sysfs_remove(&mock_iommu_device); 1101 iommu_device_unregister_bus(&mock_iommu_device, 1102 &iommufd_mock_bus_type.bus, 1103 &iommufd_mock_bus_type.nb); 1104 bus_unregister(&iommufd_mock_bus_type.bus); 1105 platform_device_unregister(selftest_iommu_dev); 1106 debugfs_remove_recursive(dbgfs_root); 1107} 1108