1/* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2/************************************************************************** 3 * 4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28/* 29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30 */ 31 32#define pr_fmt(fmt) "[TTM] " fmt 33 34#include <drm/ttm/ttm_module.h> 35#include <drm/ttm/ttm_bo_driver.h> 36#include <drm/ttm/ttm_placement.h> 37#include <drm/drm_vma_manager.h> 38#include <linux/mm.h> 39#include <linux/pfn_t.h> 40#include <linux/rbtree.h> 41#include <linux/module.h> 42#include <linux/uaccess.h> 43#include <linux/mem_encrypt.h> 44 45static vm_fault_t ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo, 46 struct vm_fault *vmf) 47{ 48 vm_fault_t ret = 0; 49 int err = 0; 50 51 if (likely(!bo->moving)) 52 goto out_unlock; 53 54 /* 55 * Quick non-stalling check for idle. 56 */ 57 if (dma_fence_is_signaled(bo->moving)) 58 goto out_clear; 59 60 /* 61 * If possible, avoid waiting for GPU with mmap_lock 62 * held. We only do this if the fault allows retry and this 63 * is the first attempt. 64 */ 65 if (fault_flag_allow_retry_first(vmf->flags)) { 66 ret = VM_FAULT_RETRY; 67 if (vmf->flags & FAULT_FLAG_RETRY_NOWAIT) 68 goto out_unlock; 69 70 ttm_bo_get(bo); 71 mmap_read_unlock(vmf->vma->vm_mm); 72 (void) dma_fence_wait(bo->moving, true); 73 dma_resv_unlock(bo->base.resv); 74 ttm_bo_put(bo); 75 goto out_unlock; 76 } 77 78 /* 79 * Ordinary wait. 80 */ 81 err = dma_fence_wait(bo->moving, true); 82 if (unlikely(err != 0)) { 83 ret = (err != -ERESTARTSYS) ? VM_FAULT_SIGBUS : 84 VM_FAULT_NOPAGE; 85 goto out_unlock; 86 } 87 88out_clear: 89 dma_fence_put(bo->moving); 90 bo->moving = NULL; 91 92out_unlock: 93 return ret; 94} 95 96static unsigned long ttm_bo_io_mem_pfn(struct ttm_buffer_object *bo, 97 unsigned long page_offset) 98{ 99 struct ttm_bo_device *bdev = bo->bdev; 100 101 if (bdev->driver->io_mem_pfn) 102 return bdev->driver->io_mem_pfn(bo, page_offset); 103 104 return (bo->mem.bus.offset >> PAGE_SHIFT) + page_offset; 105} 106 107/** 108 * ttm_bo_vm_reserve - Reserve a buffer object in a retryable vm callback 109 * @bo: The buffer object 110 * @vmf: The fault structure handed to the callback 111 * 112 * vm callbacks like fault() and *_mkwrite() allow for the mm_sem to be dropped 113 * during long waits, and after the wait the callback will be restarted. This 114 * is to allow other threads using the same virtual memory space concurrent 115 * access to map(), unmap() completely unrelated buffer objects. TTM buffer 116 * object reservations sometimes wait for GPU and should therefore be 117 * considered long waits. This function reserves the buffer object interruptibly 118 * taking this into account. Starvation is avoided by the vm system not 119 * allowing too many repeated restarts. 120 * This function is intended to be used in customized fault() and _mkwrite() 121 * handlers. 122 * 123 * Return: 124 * 0 on success and the bo was reserved. 125 * VM_FAULT_RETRY if blocking wait. 126 * VM_FAULT_NOPAGE if blocking wait and retrying was not allowed. 127 */ 128vm_fault_t ttm_bo_vm_reserve(struct ttm_buffer_object *bo, 129 struct vm_fault *vmf) 130{ 131 /* 132 * Work around locking order reversal in fault / nopfn 133 * between mmap_lock and bo_reserve: Perform a trylock operation 134 * for reserve, and if it fails, retry the fault after waiting 135 * for the buffer to become unreserved. 136 */ 137 if (unlikely(!dma_resv_trylock(bo->base.resv))) { 138 /* 139 * If the fault allows retry and this is the first 140 * fault attempt, we try to release the mmap_lock 141 * before waiting 142 */ 143 if (fault_flag_allow_retry_first(vmf->flags)) { 144 if (!(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) { 145 ttm_bo_get(bo); 146 mmap_read_unlock(vmf->vma->vm_mm); 147 if (!dma_resv_lock_interruptible(bo->base.resv, 148 NULL)) 149 dma_resv_unlock(bo->base.resv); 150 ttm_bo_put(bo); 151 } 152 153 return VM_FAULT_RETRY; 154 } 155 156 if (dma_resv_lock_interruptible(bo->base.resv, NULL)) 157 return VM_FAULT_NOPAGE; 158 } 159 160 return 0; 161} 162EXPORT_SYMBOL(ttm_bo_vm_reserve); 163 164#ifdef CONFIG_TRANSPARENT_HUGEPAGE 165/** 166 * ttm_bo_vm_insert_huge - Insert a pfn for PUD or PMD faults 167 * @vmf: Fault data 168 * @bo: The buffer object 169 * @page_offset: Page offset from bo start 170 * @fault_page_size: The size of the fault in pages. 171 * @pgprot: The page protections. 172 * Does additional checking whether it's possible to insert a PUD or PMD 173 * pfn and performs the insertion. 174 * 175 * Return: VM_FAULT_NOPAGE on successful insertion, VM_FAULT_FALLBACK if 176 * a huge fault was not possible, or on insertion error. 177 */ 178static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf, 179 struct ttm_buffer_object *bo, 180 pgoff_t page_offset, 181 pgoff_t fault_page_size, 182 pgprot_t pgprot) 183{ 184 pgoff_t i; 185 vm_fault_t ret; 186 unsigned long pfn; 187 pfn_t pfnt; 188 struct ttm_tt *ttm = bo->ttm; 189 bool write = vmf->flags & FAULT_FLAG_WRITE; 190 191 /* Fault should not cross bo boundary. */ 192 page_offset &= ~(fault_page_size - 1); 193 if (page_offset + fault_page_size > bo->num_pages) 194 goto out_fallback; 195 196 if (bo->mem.bus.is_iomem) 197 pfn = ttm_bo_io_mem_pfn(bo, page_offset); 198 else 199 pfn = page_to_pfn(ttm->pages[page_offset]); 200 201 /* pfn must be fault_page_size aligned. */ 202 if ((pfn & (fault_page_size - 1)) != 0) 203 goto out_fallback; 204 205 /* Check that memory is contiguous. */ 206 if (!bo->mem.bus.is_iomem) { 207 for (i = 1; i < fault_page_size; ++i) { 208 if (page_to_pfn(ttm->pages[page_offset + i]) != pfn + i) 209 goto out_fallback; 210 } 211 } else if (bo->bdev->driver->io_mem_pfn) { 212 for (i = 1; i < fault_page_size; ++i) { 213 if (ttm_bo_io_mem_pfn(bo, page_offset + i) != pfn + i) 214 goto out_fallback; 215 } 216 } 217 218 pfnt = __pfn_to_pfn_t(pfn, PFN_DEV); 219 if (fault_page_size == (HPAGE_PMD_SIZE >> PAGE_SHIFT)) 220 ret = vmf_insert_pfn_pmd_prot(vmf, pfnt, pgprot, write); 221#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD 222 else if (fault_page_size == (HPAGE_PUD_SIZE >> PAGE_SHIFT)) 223 ret = vmf_insert_pfn_pud_prot(vmf, pfnt, pgprot, write); 224#endif 225 else 226 WARN_ON_ONCE(ret = VM_FAULT_FALLBACK); 227 228 if (ret != VM_FAULT_NOPAGE) 229 goto out_fallback; 230 231 return VM_FAULT_NOPAGE; 232out_fallback: 233 count_vm_event(THP_FAULT_FALLBACK); 234 return VM_FAULT_FALLBACK; 235} 236#else 237static vm_fault_t ttm_bo_vm_insert_huge(struct vm_fault *vmf, 238 struct ttm_buffer_object *bo, 239 pgoff_t page_offset, 240 pgoff_t fault_page_size, 241 pgprot_t pgprot) 242{ 243 return VM_FAULT_FALLBACK; 244} 245#endif 246 247/** 248 * ttm_bo_vm_fault_reserved - TTM fault helper 249 * @vmf: The struct vm_fault given as argument to the fault callback 250 * @prot: The page protection to be used for this memory area. 251 * @num_prefault: Maximum number of prefault pages. The caller may want to 252 * specify this based on madvice settings and the size of the GPU object 253 * backed by the memory. 254 * @fault_page_size: The size of the fault in pages. 255 * 256 * This function inserts one or more page table entries pointing to the 257 * memory backing the buffer object, and then returns a return code 258 * instructing the caller to retry the page access. 259 * 260 * Return: 261 * VM_FAULT_NOPAGE on success or pending signal 262 * VM_FAULT_SIGBUS on unspecified error 263 * VM_FAULT_OOM on out-of-memory 264 * VM_FAULT_RETRY if retryable wait 265 */ 266vm_fault_t ttm_bo_vm_fault_reserved(struct vm_fault *vmf, 267 pgprot_t prot, 268 pgoff_t num_prefault, 269 pgoff_t fault_page_size) 270{ 271 struct vm_area_struct *vma = vmf->vma; 272 struct ttm_buffer_object *bo = vma->vm_private_data; 273 struct ttm_bo_device *bdev = bo->bdev; 274 unsigned long page_offset; 275 unsigned long page_last; 276 unsigned long pfn; 277 struct ttm_tt *ttm = NULL; 278 struct page *page; 279 int err; 280 pgoff_t i; 281 vm_fault_t ret = VM_FAULT_NOPAGE; 282 unsigned long address = vmf->address; 283 284 /* 285 * Refuse to fault imported pages. This should be handled 286 * (if at all) by redirecting mmap to the exporter. 287 */ 288 if (bo->ttm && (bo->ttm->page_flags & TTM_PAGE_FLAG_SG)) 289 return VM_FAULT_SIGBUS; 290 291 if (bdev->driver->fault_reserve_notify) { 292 struct dma_fence *moving = dma_fence_get(bo->moving); 293 294 err = bdev->driver->fault_reserve_notify(bo); 295 switch (err) { 296 case 0: 297 break; 298 case -EBUSY: 299 case -ERESTARTSYS: 300 dma_fence_put(moving); 301 return VM_FAULT_NOPAGE; 302 default: 303 dma_fence_put(moving); 304 return VM_FAULT_SIGBUS; 305 } 306 307 if (bo->moving != moving) { 308 ttm_bo_move_to_lru_tail_unlocked(bo); 309 } 310 dma_fence_put(moving); 311 } 312 313 /* 314 * Wait for buffer data in transit, due to a pipelined 315 * move. 316 */ 317 ret = ttm_bo_vm_fault_idle(bo, vmf); 318 if (unlikely(ret != 0)) 319 return ret; 320 321 err = ttm_mem_io_reserve(bdev, &bo->mem); 322 if (unlikely(err != 0)) 323 return VM_FAULT_SIGBUS; 324 325 page_offset = ((address - vma->vm_start) >> PAGE_SHIFT) + 326 vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node); 327 page_last = vma_pages(vma) + vma->vm_pgoff - 328 drm_vma_node_start(&bo->base.vma_node); 329 330 if (unlikely(page_offset >= bo->num_pages)) 331 return VM_FAULT_SIGBUS; 332 333 prot = ttm_io_prot(bo->mem.placement, prot); 334 if (!bo->mem.bus.is_iomem) { 335 struct ttm_operation_ctx ctx = { 336 .interruptible = false, 337 .no_wait_gpu = false, 338 .flags = TTM_OPT_FLAG_FORCE_ALLOC 339 340 }; 341 342 ttm = bo->ttm; 343 if (ttm_tt_populate(bdev, bo->ttm, &ctx)) 344 return VM_FAULT_OOM; 345 } else { 346 /* Iomem should not be marked encrypted */ 347 prot = pgprot_decrypted(prot); 348 } 349 350 /* We don't prefault on huge faults. Yet. */ 351 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && fault_page_size != 1) 352 return ttm_bo_vm_insert_huge(vmf, bo, page_offset, 353 fault_page_size, prot); 354 355 /* 356 * Speculatively prefault a number of pages. Only error on 357 * first page. 358 */ 359 for (i = 0; i < num_prefault; ++i) { 360 if (bo->mem.bus.is_iomem) { 361 pfn = ttm_bo_io_mem_pfn(bo, page_offset); 362 } else { 363 page = ttm->pages[page_offset]; 364 if (unlikely(!page && i == 0)) { 365 return VM_FAULT_OOM; 366 } else if (unlikely(!page)) { 367 break; 368 } 369 page->index = drm_vma_node_start(&bo->base.vma_node) + 370 page_offset; 371 pfn = page_to_pfn(page); 372 } 373 374 /* 375 * Note that the value of @prot at this point may differ from 376 * the value of @vma->vm_page_prot in the caching- and 377 * encryption bits. This is because the exact location of the 378 * data may not be known at mmap() time and may also change 379 * at arbitrary times while the data is mmap'ed. 380 * See vmf_insert_mixed_prot() for a discussion. 381 */ 382 if (vma->vm_flags & VM_MIXEDMAP) 383 ret = vmf_insert_mixed_prot(vma, address, 384 __pfn_to_pfn_t(pfn, PFN_DEV), 385 prot); 386 else 387 ret = vmf_insert_pfn_prot(vma, address, pfn, prot); 388 389 /* Never error on prefaulted PTEs */ 390 if (unlikely((ret & VM_FAULT_ERROR))) { 391 if (i == 0) 392 return VM_FAULT_NOPAGE; 393 else 394 break; 395 } 396 397 address += PAGE_SIZE; 398 if (unlikely(++page_offset >= page_last)) 399 break; 400 } 401 return ret; 402} 403EXPORT_SYMBOL(ttm_bo_vm_fault_reserved); 404 405vm_fault_t ttm_bo_vm_fault(struct vm_fault *vmf) 406{ 407 struct vm_area_struct *vma = vmf->vma; 408 pgprot_t prot; 409 struct ttm_buffer_object *bo = vma->vm_private_data; 410 vm_fault_t ret; 411 412 ret = ttm_bo_vm_reserve(bo, vmf); 413 if (ret) 414 return ret; 415 416 prot = vma->vm_page_prot; 417 ret = ttm_bo_vm_fault_reserved(vmf, prot, TTM_BO_VM_NUM_PREFAULT, 1); 418 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) 419 return ret; 420 421 dma_resv_unlock(bo->base.resv); 422 423 return ret; 424} 425EXPORT_SYMBOL(ttm_bo_vm_fault); 426 427void ttm_bo_vm_open(struct vm_area_struct *vma) 428{ 429 struct ttm_buffer_object *bo = vma->vm_private_data; 430 431 WARN_ON(bo->bdev->dev_mapping != vma->vm_file->f_mapping); 432 433 ttm_bo_get(bo); 434} 435EXPORT_SYMBOL(ttm_bo_vm_open); 436 437void ttm_bo_vm_close(struct vm_area_struct *vma) 438{ 439 struct ttm_buffer_object *bo = vma->vm_private_data; 440 441 ttm_bo_put(bo); 442 vma->vm_private_data = NULL; 443} 444EXPORT_SYMBOL(ttm_bo_vm_close); 445 446static int ttm_bo_vm_access_kmap(struct ttm_buffer_object *bo, 447 unsigned long offset, 448 uint8_t *buf, int len, int write) 449{ 450 unsigned long page = offset >> PAGE_SHIFT; 451 unsigned long bytes_left = len; 452 int ret; 453 454 /* Copy a page at a time, that way no extra virtual address 455 * mapping is needed 456 */ 457 offset -= page << PAGE_SHIFT; 458 do { 459 unsigned long bytes = min(bytes_left, PAGE_SIZE - offset); 460 struct ttm_bo_kmap_obj map; 461 void *ptr; 462 bool is_iomem; 463 464 ret = ttm_bo_kmap(bo, page, 1, &map); 465 if (ret) 466 return ret; 467 468 ptr = (uint8_t *)ttm_kmap_obj_virtual(&map, &is_iomem) + offset; 469 WARN_ON_ONCE(is_iomem); 470 if (write) 471 memcpy(ptr, buf, bytes); 472 else 473 memcpy(buf, ptr, bytes); 474 ttm_bo_kunmap(&map); 475 476 page++; 477 buf += bytes; 478 bytes_left -= bytes; 479 offset = 0; 480 } while (bytes_left); 481 482 return len; 483} 484 485int ttm_bo_vm_access(struct vm_area_struct *vma, unsigned long addr, 486 void *buf, int len, int write) 487{ 488 struct ttm_buffer_object *bo = vma->vm_private_data; 489 unsigned long offset = (addr) - vma->vm_start + 490 ((vma->vm_pgoff - drm_vma_node_start(&bo->base.vma_node)) 491 << PAGE_SHIFT); 492 int ret; 493 494 if (len < 1 || (offset + len) >> PAGE_SHIFT > bo->num_pages) 495 return -EIO; 496 497 ret = ttm_bo_reserve(bo, true, false, NULL); 498 if (ret) 499 return ret; 500 501 switch (bo->mem.mem_type) { 502 case TTM_PL_SYSTEM: 503 fallthrough; 504 case TTM_PL_TT: 505 ret = ttm_bo_vm_access_kmap(bo, offset, buf, len, write); 506 break; 507 default: 508 if (bo->bdev->driver->access_memory) 509 ret = bo->bdev->driver->access_memory( 510 bo, offset, buf, len, write); 511 else 512 ret = -EIO; 513 } 514 515 ttm_bo_unreserve(bo); 516 517 return ret; 518} 519EXPORT_SYMBOL(ttm_bo_vm_access); 520 521static const struct vm_operations_struct ttm_bo_vm_ops = { 522 .fault = ttm_bo_vm_fault, 523 .open = ttm_bo_vm_open, 524 .close = ttm_bo_vm_close, 525 .access = ttm_bo_vm_access, 526}; 527 528static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, 529 unsigned long offset, 530 unsigned long pages) 531{ 532 struct drm_vma_offset_node *node; 533 struct ttm_buffer_object *bo = NULL; 534 535 drm_vma_offset_lock_lookup(bdev->vma_manager); 536 537 node = drm_vma_offset_lookup_locked(bdev->vma_manager, offset, pages); 538 if (likely(node)) { 539 bo = container_of(node, struct ttm_buffer_object, 540 base.vma_node); 541 bo = ttm_bo_get_unless_zero(bo); 542 } 543 544 drm_vma_offset_unlock_lookup(bdev->vma_manager); 545 546 if (!bo) 547 pr_err("Could not find buffer object to map\n"); 548 549 return bo; 550} 551 552static void ttm_bo_mmap_vma_setup(struct ttm_buffer_object *bo, struct vm_area_struct *vma) 553{ 554 vma->vm_ops = &ttm_bo_vm_ops; 555 556 /* 557 * Note: We're transferring the bo reference to 558 * vma->vm_private_data here. 559 */ 560 561 vma->vm_private_data = bo; 562 563 /* 564 * We'd like to use VM_PFNMAP on shared mappings, where 565 * (vma->vm_flags & VM_SHARED) != 0, for performance reasons, 566 * but for some reason VM_PFNMAP + x86 PAT + write-combine is very 567 * bad for performance. Until that has been sorted out, use 568 * VM_MIXEDMAP on all mappings. See freedesktop.org bug #75719 569 */ 570 vma->vm_flags |= VM_MIXEDMAP; 571 vma->vm_flags |= VM_IO | VM_DONTEXPAND | VM_DONTDUMP; 572} 573 574int ttm_bo_mmap(struct file *filp, struct vm_area_struct *vma, 575 struct ttm_bo_device *bdev) 576{ 577 struct ttm_bo_driver *driver; 578 struct ttm_buffer_object *bo; 579 int ret; 580 581 if (unlikely(vma->vm_pgoff < DRM_FILE_PAGE_OFFSET_START)) 582 return -EINVAL; 583 584 bo = ttm_bo_vm_lookup(bdev, vma->vm_pgoff, vma_pages(vma)); 585 if (unlikely(!bo)) 586 return -EINVAL; 587 588 driver = bo->bdev->driver; 589 if (unlikely(!driver->verify_access)) { 590 ret = -EPERM; 591 goto out_unref; 592 } 593 ret = driver->verify_access(bo, filp); 594 if (unlikely(ret != 0)) 595 goto out_unref; 596 597 ttm_bo_mmap_vma_setup(bo, vma); 598 return 0; 599out_unref: 600 ttm_bo_put(bo); 601 return ret; 602} 603EXPORT_SYMBOL(ttm_bo_mmap); 604 605int ttm_bo_mmap_obj(struct vm_area_struct *vma, struct ttm_buffer_object *bo) 606{ 607 ttm_bo_get(bo); 608 ttm_bo_mmap_vma_setup(bo, vma); 609 return 0; 610} 611EXPORT_SYMBOL(ttm_bo_mmap_obj); 612