Lines Matching defs:iopt
31 struct io_pagetable *iopt,
35 lockdep_assert_held(&iopt->iova_rwsem);
39 iter->area = iopt_area_iter_first(iopt, iova, iova);
108 static int iopt_alloc_iova(struct io_pagetable *iopt, unsigned long *iova,
116 lockdep_assert_held(&iopt->iova_rwsem);
133 if (iova_alignment < iopt->iova_alignment)
136 interval_tree_for_each_span(&allowed_span, &iopt->allowed_itree,
138 if (RB_EMPTY_ROOT(&iopt->allowed_itree.rb_root)) {
149 &used_span, &iopt->reserved_itree, &iopt->area_itree,
163 static int iopt_check_iova(struct io_pagetable *iopt, unsigned long iova,
168 lockdep_assert_held(&iopt->iova_rwsem);
170 if ((iova & (iopt->iova_alignment - 1)))
177 if (iopt_reserved_iter_first(iopt, iova, last))
181 if (iopt_area_iter_first(iopt, iova, last))
189 static int iopt_insert_area(struct io_pagetable *iopt, struct iopt_area *area,
194 lockdep_assert_held_write(&iopt->iova_rwsem);
201 if (area->page_offset & (iopt->iova_alignment - 1))
219 area->iopt = iopt;
220 interval_tree_insert(&area->node, &iopt->area_itree);
236 static int iopt_alloc_area_pages(struct io_pagetable *iopt,
251 down_write(&iopt->iova_rwsem);
252 if ((length & (iopt->iova_alignment - 1)) || !length) {
262 iopt, dst_iova,
267 WARN_ON(iopt_check_iova(iopt, *dst_iova, length))) {
272 rc = iopt_check_iova(iopt, *dst_iova, length);
283 rc = iopt_insert_area(iopt, elm->area, elm->pages, iova,
291 up_write(&iopt->iova_rwsem);
299 if (area->iopt) {
300 down_write(&area->iopt->iova_rwsem);
301 interval_tree_remove(&area->node, &area->iopt->area_itree);
302 up_write(&area->iopt->iova_rwsem);
344 int iopt_map_pages(struct io_pagetable *iopt, struct list_head *pages_list,
351 rc = iopt_alloc_area_pages(iopt, pages_list, length, dst_iova,
356 down_read(&iopt->domains_rwsem);
361 down_write(&iopt->iova_rwsem);
372 up_write(&iopt->iova_rwsem);
374 up_read(&iopt->domains_rwsem);
380 * @ictx: iommufd_ctx the iopt is part of
381 * @iopt: io_pagetable to act on
397 int iopt_map_user_pages(struct iommufd_ctx *ictx, struct io_pagetable *iopt,
416 rc = iopt_map_pages(iopt, &pages_list, length, iova, iommu_prot, flags);
427 int iopt_get_pages(struct io_pagetable *iopt, unsigned long iova,
440 down_read(&iopt->iova_rwsem);
441 iopt_for_each_contig_area(&iter, area, iopt, iova, last_iova) {
460 up_read(&iopt->iova_rwsem);
463 up_read(&iopt->iova_rwsem);
468 static int iopt_unmap_iova_range(struct io_pagetable *iopt, unsigned long start,
482 down_read(&iopt->domains_rwsem);
483 down_write(&iopt->iova_rwsem);
484 while ((area = iopt_area_iter_first(iopt, start, last))) {
513 up_write(&iopt->iova_rwsem);
514 up_read(&iopt->domains_rwsem);
516 iommufd_access_notify_unmap(iopt, area_first, length);
526 up_write(&iopt->iova_rwsem);
534 down_write(&iopt->iova_rwsem);
540 up_write(&iopt->iova_rwsem);
541 up_read(&iopt->domains_rwsem);
549 * @iopt: io_pagetable to act on
557 int iopt_unmap_iova(struct io_pagetable *iopt, unsigned long iova,
568 return iopt_unmap_iova_range(iopt, iova, iova_last, unmapped);
571 int iopt_unmap_all(struct io_pagetable *iopt, unsigned long *unmapped)
575 rc = iopt_unmap_iova_range(iopt, 0, ULONG_MAX, unmapped);
583 int iopt_set_allow_iova(struct io_pagetable *iopt,
588 down_write(&iopt->iova_rwsem);
589 swap(*allowed_iova, iopt->allowed_itree);
591 for (allowed = iopt_allowed_iter_first(iopt, 0, ULONG_MAX); allowed;
593 if (iopt_reserved_iter_first(iopt, allowed->node.start,
595 swap(*allowed_iova, iopt->allowed_itree);
596 up_write(&iopt->iova_rwsem);
600 up_write(&iopt->iova_rwsem);
604 int iopt_reserve_iova(struct io_pagetable *iopt, unsigned long start,
609 lockdep_assert_held_write(&iopt->iova_rwsem);
611 if (iopt_area_iter_first(iopt, start, last) ||
612 iopt_allowed_iter_first(iopt, start, last))
621 interval_tree_insert(&reserved->node, &iopt->reserved_itree);
625 static void __iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner)
629 lockdep_assert_held_write(&iopt->iova_rwsem);
631 for (reserved = iopt_reserved_iter_first(iopt, 0, ULONG_MAX); reserved;
637 &iopt->reserved_itree);
643 void iopt_remove_reserved_iova(struct io_pagetable *iopt, void *owner)
645 down_write(&iopt->iova_rwsem);
646 __iopt_remove_reserved_iova(iopt, owner);
647 up_write(&iopt->iova_rwsem);
650 void iopt_init_table(struct io_pagetable *iopt)
652 init_rwsem(&iopt->iova_rwsem);
653 init_rwsem(&iopt->domains_rwsem);
654 iopt->area_itree = RB_ROOT_CACHED;
655 iopt->allowed_itree = RB_ROOT_CACHED;
656 iopt->reserved_itree = RB_ROOT_CACHED;
657 xa_init_flags(&iopt->domains, XA_FLAGS_ACCOUNT);
658 xa_init_flags(&iopt->access_list, XA_FLAGS_ALLOC);
661 * iopt's start as SW tables that can use the entire size_t IOVA space
665 iopt->iova_alignment = 1;
668 void iopt_destroy_table(struct io_pagetable *iopt)
673 iopt_remove_reserved_iova(iopt, NULL);
675 while ((node = interval_tree_iter_first(&iopt->allowed_itree, 0,
677 interval_tree_remove(node, &iopt->allowed_itree);
681 WARN_ON(!RB_EMPTY_ROOT(&iopt->reserved_itree.rb_root));
682 WARN_ON(!xa_empty(&iopt->domains));
683 WARN_ON(!xa_empty(&iopt->access_list));
684 WARN_ON(!RB_EMPTY_ROOT(&iopt->area_itree.rb_root));
689 * @iopt: io_pagetable to act on
692 * This is used when removing a domain from the iopt. Every area in the iopt
696 static void iopt_unfill_domain(struct io_pagetable *iopt,
701 lockdep_assert_held(&iopt->iova_rwsem);
702 lockdep_assert_held_write(&iopt->domains_rwsem);
708 if (iopt->next_domain_id != 0) {
711 xa_load(&iopt->domains, 0);
713 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
732 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
750 * @iopt: io_pagetable to act on
753 * Fill the domain with PFNs from every area in the iopt. On failure the domain
756 static int iopt_fill_domain(struct io_pagetable *iopt,
763 lockdep_assert_held(&iopt->iova_rwsem);
764 lockdep_assert_held_write(&iopt->domains_rwsem);
766 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
780 WARN_ON(iopt->next_domain_id != 0);
791 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
800 if (iopt->next_domain_id == 0) {
812 static int iopt_check_iova_alignment(struct io_pagetable *iopt,
818 lockdep_assert_held(&iopt->iova_rwsem);
819 lockdep_assert_held(&iopt->domains_rwsem);
821 for (area = iopt_area_iter_first(iopt, 0, ULONG_MAX); area;
832 xa_for_each(&iopt->access_list, index, access)
840 int iopt_table_add_domain(struct io_pagetable *iopt,
849 down_write(&iopt->domains_rwsem);
850 down_write(&iopt->iova_rwsem);
852 xa_for_each(&iopt->domains, index, iter_domain) {
869 iopt->iova_alignment);
874 if (new_iova_alignment != iopt->iova_alignment) {
875 rc = iopt_check_iova_alignment(iopt, new_iova_alignment);
882 rc = iopt_reserve_iova(iopt, 0, geometry->aperture_start - 1,
888 rc = iopt_reserve_iova(iopt, geometry->aperture_end + 1,
894 rc = xa_reserve(&iopt->domains, iopt->next_domain_id, GFP_KERNEL);
898 rc = iopt_fill_domain(iopt, domain);
902 iopt->iova_alignment = new_iova_alignment;
903 xa_store(&iopt->domains, iopt->next_domain_id, domain, GFP_KERNEL);
904 iopt->next_domain_id++;
905 up_write(&iopt->iova_rwsem);
906 up_write(&iopt->domains_rwsem);
909 xa_release(&iopt->domains, iopt->next_domain_id);
911 __iopt_remove_reserved_iova(iopt, domain);
913 up_write(&iopt->iova_rwsem);
914 up_write(&iopt->domains_rwsem);
918 static int iopt_calculate_iova_alignment(struct io_pagetable *iopt)
925 lockdep_assert_held_write(&iopt->iova_rwsem);
926 lockdep_assert_held(&iopt->domains_rwsem);
929 if (iopt->disable_large_pages)
934 xa_for_each(&iopt->domains, index, domain)
938 xa_for_each(&iopt->access_list, index, access)
943 if (new_iova_alignment > iopt->iova_alignment) {
946 rc = iopt_check_iova_alignment(iopt, new_iova_alignment);
950 iopt->iova_alignment = new_iova_alignment;
954 void iopt_table_remove_domain(struct io_pagetable *iopt,
960 down_write(&iopt->domains_rwsem);
961 down_write(&iopt->iova_rwsem);
963 xa_for_each(&iopt->domains, index, iter_domain)
966 if (WARN_ON(iter_domain != domain) || index >= iopt->next_domain_id)
973 iopt->next_domain_id--;
974 iter_domain = xa_erase(&iopt->domains, iopt->next_domain_id);
975 if (index != iopt->next_domain_id)
976 xa_store(&iopt->domains, index, iter_domain, GFP_KERNEL);
978 iopt_unfill_domain(iopt, domain);
979 __iopt_remove_reserved_iova(iopt, domain);
981 WARN_ON(iopt_calculate_iova_alignment(iopt));
983 up_write(&iopt->iova_rwsem);
984 up_write(&iopt->domains_rwsem);
998 unsigned long alignment = area->iopt->iova_alignment;
1002 struct io_pagetable *iopt = area->iopt;
1008 lockdep_assert_held_write(&iopt->iova_rwsem);
1044 if (area->storage_domain && !iopt->disable_large_pages) {
1049 interval_tree_remove(&area->node, &iopt->area_itree);
1050 rc = iopt_insert_area(iopt, lhs, area->pages, start_iova,
1057 rc = iopt_insert_area(iopt, rhs, area->pages, new_start,
1088 interval_tree_remove(&lhs->node, &iopt->area_itree);
1090 interval_tree_insert(&area->node, &iopt->area_itree);
1099 int iopt_cut_iova(struct io_pagetable *iopt, unsigned long *iovas,
1105 down_write(&iopt->iova_rwsem);
1109 area = iopt_area_iter_first(iopt, iovas[i], iovas[i]);
1116 up_write(&iopt->iova_rwsem);
1120 void iopt_enable_large_pages(struct io_pagetable *iopt)
1124 down_write(&iopt->domains_rwsem);
1125 down_write(&iopt->iova_rwsem);
1126 WRITE_ONCE(iopt->disable_large_pages, false);
1127 rc = iopt_calculate_iova_alignment(iopt);
1129 up_write(&iopt->iova_rwsem);
1130 up_write(&iopt->domains_rwsem);
1133 int iopt_disable_large_pages(struct io_pagetable *iopt)
1137 down_write(&iopt->domains_rwsem);
1138 down_write(&iopt->iova_rwsem);
1139 if (iopt->disable_large_pages)
1143 if (!xa_empty(&iopt->domains) &&
1144 !RB_EMPTY_ROOT(&iopt->area_itree.rb_root)) {
1149 WRITE_ONCE(iopt->disable_large_pages, true);
1150 rc = iopt_calculate_iova_alignment(iopt);
1152 WRITE_ONCE(iopt->disable_large_pages, false);
1154 up_write(&iopt->iova_rwsem);
1155 up_write(&iopt->domains_rwsem);
1159 int iopt_add_access(struct io_pagetable *iopt, struct iommufd_access *access)
1164 down_write(&iopt->domains_rwsem);
1165 down_write(&iopt->iova_rwsem);
1166 rc = xa_alloc(&iopt->access_list, &new_id, access, xa_limit_16b,
1172 rc = iopt_calculate_iova_alignment(iopt);
1174 xa_erase(&iopt->access_list, new_id);
1180 up_write(&iopt->iova_rwsem);
1181 up_write(&iopt->domains_rwsem);
1185 void iopt_remove_access(struct io_pagetable *iopt,
1189 down_write(&iopt->domains_rwsem);
1190 down_write(&iopt->iova_rwsem);
1191 WARN_ON(xa_erase(&iopt->access_list, iopt_access_list_id) != access);
1192 WARN_ON(iopt_calculate_iova_alignment(iopt));
1193 up_write(&iopt->iova_rwsem);
1194 up_write(&iopt->domains_rwsem);
1198 int iopt_table_enforce_dev_resv_regions(struct io_pagetable *iopt,
1211 down_write(&iopt->iova_rwsem);
1226 rc = iopt_reserve_iova(iopt, resv->start,
1242 __iopt_remove_reserved_iova(iopt, dev);
1245 up_write(&iopt->iova_rwsem);