1/* 2 * Copyright 2007 Dave Airlied 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 * OTHER DEALINGS IN THE SOFTWARE. 23 */ 24/* 25 * Authors: Dave Airlied <airlied@linux.ie> 26 * Ben Skeggs <darktama@iinet.net.au> 27 * Jeremy Kolb <jkolb@brandeis.edu> 28 */ 29 30#include <linux/dma-mapping.h> 31#include <drm/ttm/ttm_tt.h> 32 33#include "nouveau_drv.h" 34#include "nouveau_chan.h" 35#include "nouveau_fence.h" 36 37#include "nouveau_bo.h" 38#include "nouveau_ttm.h" 39#include "nouveau_gem.h" 40#include "nouveau_mem.h" 41#include "nouveau_vmm.h" 42 43#include <nvif/class.h> 44#include <nvif/if500b.h> 45#include <nvif/if900b.h> 46 47static int nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 48 struct ttm_resource *reg); 49static void nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm); 50 51/* 52 * NV10-NV40 tiling helpers 53 */ 54 55static void 56nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg, 57 u32 addr, u32 size, u32 pitch, u32 flags) 58{ 59 struct nouveau_drm *drm = nouveau_drm(dev); 60 int i = reg - drm->tile.reg; 61 struct nvkm_fb *fb = nvxx_fb(&drm->client.device); 62 struct nvkm_fb_tile *tile = &fb->tile.region[i]; 63 64 nouveau_fence_unref(®->fence); 65 66 if (tile->pitch) 67 nvkm_fb_tile_fini(fb, i, tile); 68 69 if (pitch) 70 nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile); 71 72 nvkm_fb_tile_prog(fb, i, tile); 73} 74 75static struct nouveau_drm_tile * 76nv10_bo_get_tile_region(struct drm_device *dev, int i) 77{ 78 struct nouveau_drm *drm = nouveau_drm(dev); 79 struct nouveau_drm_tile *tile = &drm->tile.reg[i]; 80 81 spin_lock(&drm->tile.lock); 82 83 if (!tile->used && 84 (!tile->fence || nouveau_fence_done(tile->fence))) 85 tile->used = true; 86 else 87 tile = NULL; 88 89 spin_unlock(&drm->tile.lock); 90 return tile; 91} 92 93static void 94nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile, 95 struct dma_fence *fence) 96{ 97 struct nouveau_drm *drm = nouveau_drm(dev); 98 99 if (tile) { 100 spin_lock(&drm->tile.lock); 101 tile->fence = (struct nouveau_fence *)dma_fence_get(fence); 102 tile->used = false; 103 spin_unlock(&drm->tile.lock); 104 } 105} 106 107static struct nouveau_drm_tile * 108nv10_bo_set_tiling(struct drm_device *dev, u32 addr, 109 u32 size, u32 pitch, u32 zeta) 110{ 111 struct nouveau_drm *drm = nouveau_drm(dev); 112 struct nvkm_fb *fb = nvxx_fb(&drm->client.device); 113 struct nouveau_drm_tile *tile, *found = NULL; 114 int i; 115 116 for (i = 0; i < fb->tile.regions; i++) { 117 tile = nv10_bo_get_tile_region(dev, i); 118 119 if (pitch && !found) { 120 found = tile; 121 continue; 122 123 } else if (tile && fb->tile.region[i].pitch) { 124 /* Kill an unused tile region. */ 125 nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0); 126 } 127 128 nv10_bo_put_tile_region(dev, tile, NULL); 129 } 130 131 if (found) 132 nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta); 133 return found; 134} 135 136static void 137nouveau_bo_del_ttm(struct ttm_buffer_object *bo) 138{ 139 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 140 struct drm_device *dev = drm->dev; 141 struct nouveau_bo *nvbo = nouveau_bo(bo); 142 143 WARN_ON(nvbo->bo.pin_count > 0); 144 nouveau_bo_del_io_reserve_lru(bo); 145 nv10_bo_put_tile_region(dev, nvbo->tile, NULL); 146 147 /* 148 * If nouveau_bo_new() allocated this buffer, the GEM object was never 149 * initialized, so don't attempt to release it. 150 */ 151 if (bo->base.dev) 152 drm_gem_object_release(&bo->base); 153 else 154 dma_resv_fini(&bo->base._resv); 155 156 kfree(nvbo); 157} 158 159static inline u64 160roundup_64(u64 x, u32 y) 161{ 162 x += y - 1; 163 do_div(x, y); 164 return x * y; 165} 166 167static void 168nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, u64 *size) 169{ 170 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 171 struct nvif_device *device = &drm->client.device; 172 173 if (device->info.family < NV_DEVICE_INFO_V0_TESLA) { 174 if (nvbo->mode) { 175 if (device->info.chipset >= 0x40) { 176 *align = 65536; 177 *size = roundup_64(*size, 64 * nvbo->mode); 178 179 } else if (device->info.chipset >= 0x30) { 180 *align = 32768; 181 *size = roundup_64(*size, 64 * nvbo->mode); 182 183 } else if (device->info.chipset >= 0x20) { 184 *align = 16384; 185 *size = roundup_64(*size, 64 * nvbo->mode); 186 187 } else if (device->info.chipset >= 0x10) { 188 *align = 16384; 189 *size = roundup_64(*size, 32 * nvbo->mode); 190 } 191 } 192 } else { 193 *size = roundup_64(*size, (1 << nvbo->page)); 194 *align = max((1 << nvbo->page), *align); 195 } 196 197 *size = roundup_64(*size, PAGE_SIZE); 198} 199 200struct nouveau_bo * 201nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain, 202 u32 tile_mode, u32 tile_flags, bool internal) 203{ 204 struct nouveau_drm *drm = cli->drm; 205 struct nouveau_bo *nvbo; 206 struct nvif_mmu *mmu = &cli->mmu; 207 struct nvif_vmm *vmm = &nouveau_cli_vmm(cli)->vmm; 208 int i, pi = -1; 209 210 if (!*size) { 211 NV_WARN(drm, "skipped size %016llx\n", *size); 212 return ERR_PTR(-EINVAL); 213 } 214 215 nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL); 216 if (!nvbo) 217 return ERR_PTR(-ENOMEM); 218 219 INIT_LIST_HEAD(&nvbo->head); 220 INIT_LIST_HEAD(&nvbo->entry); 221 INIT_LIST_HEAD(&nvbo->vma_list); 222 nvbo->bo.bdev = &drm->ttm.bdev; 223 224 /* This is confusing, and doesn't actually mean we want an uncached 225 * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated 226 * into in nouveau_gem_new(). 227 */ 228 if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) { 229 /* Determine if we can get a cache-coherent map, forcing 230 * uncached mapping if we can't. 231 */ 232 if (!nouveau_drm_use_coherent_gpu_mapping(drm)) 233 nvbo->force_coherent = true; 234 } 235 236 nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG); 237 if (!nouveau_cli_uvmm(cli) || internal) { 238 /* for BO noVM allocs, don't assign kinds */ 239 if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) { 240 nvbo->kind = (tile_flags & 0x0000ff00) >> 8; 241 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 242 kfree(nvbo); 243 return ERR_PTR(-EINVAL); 244 } 245 246 nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind; 247 } else if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 248 nvbo->kind = (tile_flags & 0x00007f00) >> 8; 249 nvbo->comp = (tile_flags & 0x00030000) >> 16; 250 if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) { 251 kfree(nvbo); 252 return ERR_PTR(-EINVAL); 253 } 254 } else { 255 nvbo->zeta = (tile_flags & 0x00000007); 256 } 257 nvbo->mode = tile_mode; 258 259 /* Determine the desirable target GPU page size for the buffer. */ 260 for (i = 0; i < vmm->page_nr; i++) { 261 /* Because we cannot currently allow VMM maps to fail 262 * during buffer migration, we need to determine page 263 * size for the buffer up-front, and pre-allocate its 264 * page tables. 265 * 266 * Skip page sizes that can't support needed domains. 267 */ 268 if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE && 269 (domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) 270 continue; 271 if ((domain & NOUVEAU_GEM_DOMAIN_GART) && 272 (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) 273 continue; 274 275 /* Select this page size if it's the first that supports 276 * the potential memory domains, or when it's compatible 277 * with the requested compression settings. 278 */ 279 if (pi < 0 || !nvbo->comp || vmm->page[i].comp) 280 pi = i; 281 282 /* Stop once the buffer is larger than the current page size. */ 283 if (*size >= 1ULL << vmm->page[i].shift) 284 break; 285 } 286 287 if (WARN_ON(pi < 0)) { 288 kfree(nvbo); 289 return ERR_PTR(-EINVAL); 290 } 291 292 /* Disable compression if suitable settings couldn't be found. */ 293 if (nvbo->comp && !vmm->page[pi].comp) { 294 if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100) 295 nvbo->kind = mmu->kind[nvbo->kind]; 296 nvbo->comp = 0; 297 } 298 nvbo->page = vmm->page[pi].shift; 299 } else { 300 /* reject other tile flags when in VM mode. */ 301 if (tile_mode) 302 return ERR_PTR(-EINVAL); 303 if (tile_flags & ~NOUVEAU_GEM_TILE_NONCONTIG) 304 return ERR_PTR(-EINVAL); 305 306 /* Determine the desirable target GPU page size for the buffer. */ 307 for (i = 0; i < vmm->page_nr; i++) { 308 /* Because we cannot currently allow VMM maps to fail 309 * during buffer migration, we need to determine page 310 * size for the buffer up-front, and pre-allocate its 311 * page tables. 312 * 313 * Skip page sizes that can't support needed domains. 314 */ 315 if ((domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram) 316 continue; 317 if ((domain & NOUVEAU_GEM_DOMAIN_GART) && 318 (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT)) 319 continue; 320 321 /* pick the last one as it will be smallest. */ 322 pi = i; 323 324 /* Stop once the buffer is larger than the current page size. */ 325 if (*size >= 1ULL << vmm->page[i].shift) 326 break; 327 } 328 if (WARN_ON(pi < 0)) { 329 kfree(nvbo); 330 return ERR_PTR(-EINVAL); 331 } 332 nvbo->page = vmm->page[pi].shift; 333 } 334 335 nouveau_bo_fixup_align(nvbo, align, size); 336 337 return nvbo; 338} 339 340int 341nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain, 342 struct sg_table *sg, struct dma_resv *robj) 343{ 344 int type = sg ? ttm_bo_type_sg : ttm_bo_type_device; 345 int ret; 346 struct ttm_operation_ctx ctx = { 347 .interruptible = false, 348 .no_wait_gpu = false, 349 .resv = robj, 350 }; 351 352 nouveau_bo_placement_set(nvbo, domain, 0); 353 INIT_LIST_HEAD(&nvbo->io_reserve_lru); 354 355 ret = ttm_bo_init_reserved(nvbo->bo.bdev, &nvbo->bo, type, 356 &nvbo->placement, align >> PAGE_SHIFT, &ctx, 357 sg, robj, nouveau_bo_del_ttm); 358 if (ret) { 359 /* ttm will call nouveau_bo_del_ttm if it fails.. */ 360 return ret; 361 } 362 363 if (!robj) 364 ttm_bo_unreserve(&nvbo->bo); 365 366 return 0; 367} 368 369int 370nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align, 371 uint32_t domain, uint32_t tile_mode, uint32_t tile_flags, 372 struct sg_table *sg, struct dma_resv *robj, 373 struct nouveau_bo **pnvbo) 374{ 375 struct nouveau_bo *nvbo; 376 int ret; 377 378 nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode, 379 tile_flags, true); 380 if (IS_ERR(nvbo)) 381 return PTR_ERR(nvbo); 382 383 nvbo->bo.base.size = size; 384 dma_resv_init(&nvbo->bo.base._resv); 385 drm_vma_node_reset(&nvbo->bo.base.vma_node); 386 387 /* This must be called before ttm_bo_init_reserved(). Subsequent 388 * bo_move() callbacks might already iterate the GEMs GPUVA list. 389 */ 390 drm_gem_gpuva_init(&nvbo->bo.base); 391 392 ret = nouveau_bo_init(nvbo, size, align, domain, sg, robj); 393 if (ret) 394 return ret; 395 396 *pnvbo = nvbo; 397 return 0; 398} 399 400static void 401set_placement_list(struct ttm_place *pl, unsigned *n, uint32_t domain) 402{ 403 *n = 0; 404 405 if (domain & NOUVEAU_GEM_DOMAIN_VRAM) { 406 pl[*n].mem_type = TTM_PL_VRAM; 407 pl[*n].flags = 0; 408 (*n)++; 409 } 410 if (domain & NOUVEAU_GEM_DOMAIN_GART) { 411 pl[*n].mem_type = TTM_PL_TT; 412 pl[*n].flags = 0; 413 (*n)++; 414 } 415 if (domain & NOUVEAU_GEM_DOMAIN_CPU) { 416 pl[*n].mem_type = TTM_PL_SYSTEM; 417 pl[(*n)++].flags = 0; 418 } 419} 420 421static void 422set_placement_range(struct nouveau_bo *nvbo, uint32_t domain) 423{ 424 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 425 u64 vram_size = drm->client.device.info.ram_size; 426 unsigned i, fpfn, lpfn; 427 428 if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS && 429 nvbo->mode && (domain & NOUVEAU_GEM_DOMAIN_VRAM) && 430 nvbo->bo.base.size < vram_size / 4) { 431 /* 432 * Make sure that the color and depth buffers are handled 433 * by independent memory controller units. Up to a 9x 434 * speed up when alpha-blending and depth-test are enabled 435 * at the same time. 436 */ 437 if (nvbo->zeta) { 438 fpfn = (vram_size / 2) >> PAGE_SHIFT; 439 lpfn = ~0; 440 } else { 441 fpfn = 0; 442 lpfn = (vram_size / 2) >> PAGE_SHIFT; 443 } 444 for (i = 0; i < nvbo->placement.num_placement; ++i) { 445 nvbo->placements[i].fpfn = fpfn; 446 nvbo->placements[i].lpfn = lpfn; 447 } 448 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { 449 nvbo->busy_placements[i].fpfn = fpfn; 450 nvbo->busy_placements[i].lpfn = lpfn; 451 } 452 } 453} 454 455void 456nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain, 457 uint32_t busy) 458{ 459 struct ttm_placement *pl = &nvbo->placement; 460 461 pl->placement = nvbo->placements; 462 set_placement_list(nvbo->placements, &pl->num_placement, domain); 463 464 pl->busy_placement = nvbo->busy_placements; 465 set_placement_list(nvbo->busy_placements, &pl->num_busy_placement, 466 domain | busy); 467 468 set_placement_range(nvbo, domain); 469} 470 471int 472nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig) 473{ 474 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 475 struct ttm_buffer_object *bo = &nvbo->bo; 476 bool force = false, evict = false; 477 int ret; 478 479 ret = ttm_bo_reserve(bo, false, false, NULL); 480 if (ret) 481 return ret; 482 483 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 484 domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) { 485 if (!nvbo->contig) { 486 nvbo->contig = true; 487 force = true; 488 evict = true; 489 } 490 } 491 492 if (nvbo->bo.pin_count) { 493 bool error = evict; 494 495 switch (bo->resource->mem_type) { 496 case TTM_PL_VRAM: 497 error |= !(domain & NOUVEAU_GEM_DOMAIN_VRAM); 498 break; 499 case TTM_PL_TT: 500 error |= !(domain & NOUVEAU_GEM_DOMAIN_GART); 501 break; 502 default: 503 break; 504 } 505 506 if (error) { 507 NV_ERROR(drm, "bo %p pinned elsewhere: " 508 "0x%08x vs 0x%08x\n", bo, 509 bo->resource->mem_type, domain); 510 ret = -EBUSY; 511 } 512 ttm_bo_pin(&nvbo->bo); 513 goto out; 514 } 515 516 if (evict) { 517 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 518 ret = nouveau_bo_validate(nvbo, false, false); 519 if (ret) 520 goto out; 521 } 522 523 nouveau_bo_placement_set(nvbo, domain, 0); 524 ret = nouveau_bo_validate(nvbo, false, false); 525 if (ret) 526 goto out; 527 528 ttm_bo_pin(&nvbo->bo); 529 530 switch (bo->resource->mem_type) { 531 case TTM_PL_VRAM: 532 drm->gem.vram_available -= bo->base.size; 533 break; 534 case TTM_PL_TT: 535 drm->gem.gart_available -= bo->base.size; 536 break; 537 default: 538 break; 539 } 540 541out: 542 if (force && ret) 543 nvbo->contig = false; 544 ttm_bo_unreserve(bo); 545 return ret; 546} 547 548int 549nouveau_bo_unpin(struct nouveau_bo *nvbo) 550{ 551 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 552 struct ttm_buffer_object *bo = &nvbo->bo; 553 int ret; 554 555 ret = ttm_bo_reserve(bo, false, false, NULL); 556 if (ret) 557 return ret; 558 559 ttm_bo_unpin(&nvbo->bo); 560 if (!nvbo->bo.pin_count) { 561 switch (bo->resource->mem_type) { 562 case TTM_PL_VRAM: 563 drm->gem.vram_available += bo->base.size; 564 break; 565 case TTM_PL_TT: 566 drm->gem.gart_available += bo->base.size; 567 break; 568 default: 569 break; 570 } 571 } 572 573 ttm_bo_unreserve(bo); 574 return 0; 575} 576 577int 578nouveau_bo_map(struct nouveau_bo *nvbo) 579{ 580 int ret; 581 582 ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL); 583 if (ret) 584 return ret; 585 586 ret = ttm_bo_kmap(&nvbo->bo, 0, PFN_UP(nvbo->bo.base.size), &nvbo->kmap); 587 588 ttm_bo_unreserve(&nvbo->bo); 589 return ret; 590} 591 592void 593nouveau_bo_unmap(struct nouveau_bo *nvbo) 594{ 595 if (!nvbo) 596 return; 597 598 ttm_bo_kunmap(&nvbo->kmap); 599} 600 601void 602nouveau_bo_sync_for_device(struct nouveau_bo *nvbo) 603{ 604 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 605 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 606 int i, j; 607 608 if (!ttm_dma || !ttm_dma->dma_address) 609 return; 610 if (!ttm_dma->pages) { 611 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 612 return; 613 } 614 615 /* Don't waste time looping if the object is coherent */ 616 if (nvbo->force_coherent) 617 return; 618 619 i = 0; 620 while (i < ttm_dma->num_pages) { 621 struct page *p = ttm_dma->pages[i]; 622 size_t num_pages = 1; 623 624 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 625 if (++p != ttm_dma->pages[j]) 626 break; 627 628 ++num_pages; 629 } 630 dma_sync_single_for_device(drm->dev->dev, 631 ttm_dma->dma_address[i], 632 num_pages * PAGE_SIZE, DMA_TO_DEVICE); 633 i += num_pages; 634 } 635} 636 637void 638nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo) 639{ 640 struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev); 641 struct ttm_tt *ttm_dma = (struct ttm_tt *)nvbo->bo.ttm; 642 int i, j; 643 644 if (!ttm_dma || !ttm_dma->dma_address) 645 return; 646 if (!ttm_dma->pages) { 647 NV_DEBUG(drm, "ttm_dma 0x%p: pages NULL\n", ttm_dma); 648 return; 649 } 650 651 /* Don't waste time looping if the object is coherent */ 652 if (nvbo->force_coherent) 653 return; 654 655 i = 0; 656 while (i < ttm_dma->num_pages) { 657 struct page *p = ttm_dma->pages[i]; 658 size_t num_pages = 1; 659 660 for (j = i + 1; j < ttm_dma->num_pages; ++j) { 661 if (++p != ttm_dma->pages[j]) 662 break; 663 664 ++num_pages; 665 } 666 667 dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i], 668 num_pages * PAGE_SIZE, DMA_FROM_DEVICE); 669 i += num_pages; 670 } 671} 672 673void nouveau_bo_add_io_reserve_lru(struct ttm_buffer_object *bo) 674{ 675 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 676 struct nouveau_bo *nvbo = nouveau_bo(bo); 677 678 mutex_lock(&drm->ttm.io_reserve_mutex); 679 list_move_tail(&nvbo->io_reserve_lru, &drm->ttm.io_reserve_lru); 680 mutex_unlock(&drm->ttm.io_reserve_mutex); 681} 682 683void nouveau_bo_del_io_reserve_lru(struct ttm_buffer_object *bo) 684{ 685 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 686 struct nouveau_bo *nvbo = nouveau_bo(bo); 687 688 mutex_lock(&drm->ttm.io_reserve_mutex); 689 list_del_init(&nvbo->io_reserve_lru); 690 mutex_unlock(&drm->ttm.io_reserve_mutex); 691} 692 693int 694nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible, 695 bool no_wait_gpu) 696{ 697 struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu }; 698 int ret; 699 700 ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx); 701 if (ret) 702 return ret; 703 704 nouveau_bo_sync_for_device(nvbo); 705 706 return 0; 707} 708 709void 710nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val) 711{ 712 bool is_iomem; 713 u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 714 715 mem += index; 716 717 if (is_iomem) 718 iowrite16_native(val, (void __force __iomem *)mem); 719 else 720 *mem = val; 721} 722 723u32 724nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index) 725{ 726 bool is_iomem; 727 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 728 729 mem += index; 730 731 if (is_iomem) 732 return ioread32_native((void __force __iomem *)mem); 733 else 734 return *mem; 735} 736 737void 738nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val) 739{ 740 bool is_iomem; 741 u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem); 742 743 mem += index; 744 745 if (is_iomem) 746 iowrite32_native(val, (void __force __iomem *)mem); 747 else 748 *mem = val; 749} 750 751static struct ttm_tt * 752nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags) 753{ 754#if IS_ENABLED(CONFIG_AGP) 755 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 756 757 if (drm->agp.bridge) { 758 return ttm_agp_tt_create(bo, drm->agp.bridge, page_flags); 759 } 760#endif 761 762 return nouveau_sgdma_create_ttm(bo, page_flags); 763} 764 765static int 766nouveau_ttm_tt_bind(struct ttm_device *bdev, struct ttm_tt *ttm, 767 struct ttm_resource *reg) 768{ 769#if IS_ENABLED(CONFIG_AGP) 770 struct nouveau_drm *drm = nouveau_bdev(bdev); 771#endif 772 if (!reg) 773 return -EINVAL; 774#if IS_ENABLED(CONFIG_AGP) 775 if (drm->agp.bridge) 776 return ttm_agp_bind(ttm, reg); 777#endif 778 return nouveau_sgdma_bind(bdev, ttm, reg); 779} 780 781static void 782nouveau_ttm_tt_unbind(struct ttm_device *bdev, struct ttm_tt *ttm) 783{ 784#if IS_ENABLED(CONFIG_AGP) 785 struct nouveau_drm *drm = nouveau_bdev(bdev); 786 787 if (drm->agp.bridge) { 788 ttm_agp_unbind(ttm); 789 return; 790 } 791#endif 792 nouveau_sgdma_unbind(bdev, ttm); 793} 794 795static void 796nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl) 797{ 798 struct nouveau_bo *nvbo = nouveau_bo(bo); 799 800 switch (bo->resource->mem_type) { 801 case TTM_PL_VRAM: 802 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 803 NOUVEAU_GEM_DOMAIN_CPU); 804 break; 805 default: 806 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_CPU, 0); 807 break; 808 } 809 810 *pl = nvbo->placement; 811} 812 813static int 814nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo, 815 struct ttm_resource *reg) 816{ 817 struct nouveau_mem *old_mem = nouveau_mem(bo->resource); 818 struct nouveau_mem *new_mem = nouveau_mem(reg); 819 struct nvif_vmm *vmm = &drm->client.vmm.vmm; 820 int ret; 821 822 ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0, 823 old_mem->mem.size, &old_mem->vma[0]); 824 if (ret) 825 return ret; 826 827 ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0, 828 new_mem->mem.size, &old_mem->vma[1]); 829 if (ret) 830 goto done; 831 832 ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]); 833 if (ret) 834 goto done; 835 836 ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]); 837done: 838 if (ret) { 839 nvif_vmm_put(vmm, &old_mem->vma[1]); 840 nvif_vmm_put(vmm, &old_mem->vma[0]); 841 } 842 return 0; 843} 844 845static int 846nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, 847 struct ttm_operation_ctx *ctx, 848 struct ttm_resource *new_reg) 849{ 850 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 851 struct nouveau_channel *chan = drm->ttm.chan; 852 struct nouveau_cli *cli = (void *)chan->user.client; 853 struct nouveau_fence *fence; 854 int ret; 855 856 /* create temporary vmas for the transfer and attach them to the 857 * old nvkm_mem node, these will get cleaned up after ttm has 858 * destroyed the ttm_resource 859 */ 860 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) { 861 ret = nouveau_bo_move_prep(drm, bo, new_reg); 862 if (ret) 863 return ret; 864 } 865 866 if (drm_drv_uses_atomic_modeset(drm->dev)) 867 mutex_lock(&cli->mutex); 868 else 869 mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING); 870 871 ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, ctx->interruptible); 872 if (ret) 873 goto out_unlock; 874 875 ret = drm->ttm.move(chan, bo, bo->resource, new_reg); 876 if (ret) 877 goto out_unlock; 878 879 ret = nouveau_fence_new(&fence, chan); 880 if (ret) 881 goto out_unlock; 882 883 /* TODO: figure out a better solution here 884 * 885 * wait on the fence here explicitly as going through 886 * ttm_bo_move_accel_cleanup somehow doesn't seem to do it. 887 * 888 * Without this the operation can timeout and we'll fallback to a 889 * software copy, which might take several minutes to finish. 890 */ 891 nouveau_fence_wait(fence, false, false); 892 ret = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, false, 893 new_reg); 894 nouveau_fence_unref(&fence); 895 896out_unlock: 897 mutex_unlock(&cli->mutex); 898 return ret; 899} 900 901void 902nouveau_bo_move_init(struct nouveau_drm *drm) 903{ 904 static const struct _method_table { 905 const char *name; 906 int engine; 907 s32 oclass; 908 int (*exec)(struct nouveau_channel *, 909 struct ttm_buffer_object *, 910 struct ttm_resource *, struct ttm_resource *); 911 int (*init)(struct nouveau_channel *, u32 handle); 912 } _methods[] = { 913 { "COPY", 4, 0xc7b5, nve0_bo_move_copy, nve0_bo_move_init }, 914 { "GRCE", 0, 0xc7b5, nve0_bo_move_copy, nvc0_bo_move_init }, 915 { "COPY", 4, 0xc6b5, nve0_bo_move_copy, nve0_bo_move_init }, 916 { "GRCE", 0, 0xc6b5, nve0_bo_move_copy, nvc0_bo_move_init }, 917 { "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init }, 918 { "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init }, 919 { "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init }, 920 { "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init }, 921 { "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init }, 922 { "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init }, 923 { "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init }, 924 { "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 925 { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init }, 926 { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 927 { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init }, 928 { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init }, 929 { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init }, 930 { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init }, 931 { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init }, 932 { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init }, 933 { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init }, 934 { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init }, 935 { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init }, 936 {}, 937 }; 938 const struct _method_table *mthd = _methods; 939 const char *name = "CPU"; 940 int ret; 941 942 do { 943 struct nouveau_channel *chan; 944 945 if (mthd->engine) 946 chan = drm->cechan; 947 else 948 chan = drm->channel; 949 if (chan == NULL) 950 continue; 951 952 ret = nvif_object_ctor(&chan->user, "ttmBoMove", 953 mthd->oclass | (mthd->engine << 16), 954 mthd->oclass, NULL, 0, 955 &drm->ttm.copy); 956 if (ret == 0) { 957 ret = mthd->init(chan, drm->ttm.copy.handle); 958 if (ret) { 959 nvif_object_dtor(&drm->ttm.copy); 960 continue; 961 } 962 963 drm->ttm.move = mthd->exec; 964 drm->ttm.chan = chan; 965 name = mthd->name; 966 break; 967 } 968 } while ((++mthd)->exec); 969 970 NV_INFO(drm, "MM: using %s for buffer copies\n", name); 971} 972 973static void nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, 974 struct ttm_resource *new_reg) 975{ 976 struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL; 977 struct nouveau_bo *nvbo = nouveau_bo(bo); 978 struct nouveau_vma *vma; 979 long ret; 980 981 /* ttm can now (stupidly) pass the driver bos it didn't create... */ 982 if (bo->destroy != nouveau_bo_del_ttm) 983 return; 984 985 nouveau_bo_del_io_reserve_lru(bo); 986 987 if (mem && new_reg->mem_type != TTM_PL_SYSTEM && 988 mem->mem.page == nvbo->page) { 989 list_for_each_entry(vma, &nvbo->vma_list, head) { 990 nouveau_vma_map(vma, mem); 991 } 992 nouveau_uvmm_bo_map_all(nvbo, mem); 993 } else { 994 list_for_each_entry(vma, &nvbo->vma_list, head) { 995 ret = dma_resv_wait_timeout(bo->base.resv, 996 DMA_RESV_USAGE_BOOKKEEP, 997 false, 15 * HZ); 998 WARN_ON(ret <= 0); 999 nouveau_vma_unmap(vma); 1000 } 1001 nouveau_uvmm_bo_unmap_all(nvbo); 1002 } 1003 1004 if (new_reg) 1005 nvbo->offset = (new_reg->start << PAGE_SHIFT); 1006 1007} 1008 1009static int 1010nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_resource *new_reg, 1011 struct nouveau_drm_tile **new_tile) 1012{ 1013 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1014 struct drm_device *dev = drm->dev; 1015 struct nouveau_bo *nvbo = nouveau_bo(bo); 1016 u64 offset = new_reg->start << PAGE_SHIFT; 1017 1018 *new_tile = NULL; 1019 if (new_reg->mem_type != TTM_PL_VRAM) 1020 return 0; 1021 1022 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) { 1023 *new_tile = nv10_bo_set_tiling(dev, offset, bo->base.size, 1024 nvbo->mode, nvbo->zeta); 1025 } 1026 1027 return 0; 1028} 1029 1030static void 1031nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo, 1032 struct nouveau_drm_tile *new_tile, 1033 struct nouveau_drm_tile **old_tile) 1034{ 1035 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1036 struct drm_device *dev = drm->dev; 1037 struct dma_fence *fence; 1038 int ret; 1039 1040 ret = dma_resv_get_singleton(bo->base.resv, DMA_RESV_USAGE_WRITE, 1041 &fence); 1042 if (ret) 1043 dma_resv_wait_timeout(bo->base.resv, DMA_RESV_USAGE_WRITE, 1044 false, MAX_SCHEDULE_TIMEOUT); 1045 1046 nv10_bo_put_tile_region(dev, *old_tile, fence); 1047 *old_tile = new_tile; 1048} 1049 1050static int 1051nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, 1052 struct ttm_operation_ctx *ctx, 1053 struct ttm_resource *new_reg, 1054 struct ttm_place *hop) 1055{ 1056 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1057 struct nouveau_bo *nvbo = nouveau_bo(bo); 1058 struct ttm_resource *old_reg = bo->resource; 1059 struct nouveau_drm_tile *new_tile = NULL; 1060 int ret = 0; 1061 1062 1063 if (new_reg->mem_type == TTM_PL_TT) { 1064 ret = nouveau_ttm_tt_bind(bo->bdev, bo->ttm, new_reg); 1065 if (ret) 1066 return ret; 1067 } 1068 1069 nouveau_bo_move_ntfy(bo, new_reg); 1070 ret = ttm_bo_wait_ctx(bo, ctx); 1071 if (ret) 1072 goto out_ntfy; 1073 1074 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1075 ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile); 1076 if (ret) 1077 goto out_ntfy; 1078 } 1079 1080 /* Fake bo copy. */ 1081 if (!old_reg || (old_reg->mem_type == TTM_PL_SYSTEM && 1082 !bo->ttm)) { 1083 ttm_bo_move_null(bo, new_reg); 1084 goto out; 1085 } 1086 1087 if (old_reg->mem_type == TTM_PL_SYSTEM && 1088 new_reg->mem_type == TTM_PL_TT) { 1089 ttm_bo_move_null(bo, new_reg); 1090 goto out; 1091 } 1092 1093 if (old_reg->mem_type == TTM_PL_TT && 1094 new_reg->mem_type == TTM_PL_SYSTEM) { 1095 nouveau_ttm_tt_unbind(bo->bdev, bo->ttm); 1096 ttm_resource_free(bo, &bo->resource); 1097 ttm_bo_assign_mem(bo, new_reg); 1098 goto out; 1099 } 1100 1101 /* Hardware assisted copy. */ 1102 if (drm->ttm.move) { 1103 if ((old_reg->mem_type == TTM_PL_SYSTEM && 1104 new_reg->mem_type == TTM_PL_VRAM) || 1105 (old_reg->mem_type == TTM_PL_VRAM && 1106 new_reg->mem_type == TTM_PL_SYSTEM)) { 1107 hop->fpfn = 0; 1108 hop->lpfn = 0; 1109 hop->mem_type = TTM_PL_TT; 1110 hop->flags = 0; 1111 return -EMULTIHOP; 1112 } 1113 ret = nouveau_bo_move_m2mf(bo, evict, ctx, 1114 new_reg); 1115 } else 1116 ret = -ENODEV; 1117 1118 if (ret) { 1119 /* Fallback to software copy. */ 1120 ret = ttm_bo_move_memcpy(bo, ctx, new_reg); 1121 } 1122 1123out: 1124 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) { 1125 if (ret) 1126 nouveau_bo_vm_cleanup(bo, NULL, &new_tile); 1127 else 1128 nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile); 1129 } 1130out_ntfy: 1131 if (ret) { 1132 nouveau_bo_move_ntfy(bo, bo->resource); 1133 } 1134 return ret; 1135} 1136 1137static void 1138nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm, 1139 struct ttm_resource *reg) 1140{ 1141 struct nouveau_mem *mem = nouveau_mem(reg); 1142 1143 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1144 switch (reg->mem_type) { 1145 case TTM_PL_TT: 1146 if (mem->kind) 1147 nvif_object_unmap_handle(&mem->mem.object); 1148 break; 1149 case TTM_PL_VRAM: 1150 nvif_object_unmap_handle(&mem->mem.object); 1151 break; 1152 default: 1153 break; 1154 } 1155 } 1156} 1157 1158static int 1159nouveau_ttm_io_mem_reserve(struct ttm_device *bdev, struct ttm_resource *reg) 1160{ 1161 struct nouveau_drm *drm = nouveau_bdev(bdev); 1162 struct nvkm_device *device = nvxx_device(&drm->client.device); 1163 struct nouveau_mem *mem = nouveau_mem(reg); 1164 struct nvif_mmu *mmu = &drm->client.mmu; 1165 int ret; 1166 1167 mutex_lock(&drm->ttm.io_reserve_mutex); 1168retry: 1169 switch (reg->mem_type) { 1170 case TTM_PL_SYSTEM: 1171 /* System memory */ 1172 ret = 0; 1173 goto out; 1174 case TTM_PL_TT: 1175#if IS_ENABLED(CONFIG_AGP) 1176 if (drm->agp.bridge) { 1177 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1178 drm->agp.base; 1179 reg->bus.is_iomem = !drm->agp.cma; 1180 reg->bus.caching = ttm_write_combined; 1181 } 1182#endif 1183 if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 || 1184 !mem->kind) { 1185 /* untiled */ 1186 ret = 0; 1187 break; 1188 } 1189 fallthrough; /* tiled memory */ 1190 case TTM_PL_VRAM: 1191 reg->bus.offset = (reg->start << PAGE_SHIFT) + 1192 device->func->resource_addr(device, 1); 1193 reg->bus.is_iomem = true; 1194 1195 /* Some BARs do not support being ioremapped WC */ 1196 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA && 1197 mmu->type[drm->ttm.type_vram].type & NVIF_MEM_UNCACHED) 1198 reg->bus.caching = ttm_uncached; 1199 else 1200 reg->bus.caching = ttm_write_combined; 1201 1202 if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) { 1203 union { 1204 struct nv50_mem_map_v0 nv50; 1205 struct gf100_mem_map_v0 gf100; 1206 } args; 1207 u64 handle, length; 1208 u32 argc = 0; 1209 1210 switch (mem->mem.object.oclass) { 1211 case NVIF_CLASS_MEM_NV50: 1212 args.nv50.version = 0; 1213 args.nv50.ro = 0; 1214 args.nv50.kind = mem->kind; 1215 args.nv50.comp = mem->comp; 1216 argc = sizeof(args.nv50); 1217 break; 1218 case NVIF_CLASS_MEM_GF100: 1219 args.gf100.version = 0; 1220 args.gf100.ro = 0; 1221 args.gf100.kind = mem->kind; 1222 argc = sizeof(args.gf100); 1223 break; 1224 default: 1225 WARN_ON(1); 1226 break; 1227 } 1228 1229 ret = nvif_object_map_handle(&mem->mem.object, 1230 &args, argc, 1231 &handle, &length); 1232 if (ret != 1) { 1233 if (WARN_ON(ret == 0)) 1234 ret = -EINVAL; 1235 goto out; 1236 } 1237 1238 reg->bus.offset = handle; 1239 } 1240 ret = 0; 1241 break; 1242 default: 1243 ret = -EINVAL; 1244 } 1245 1246out: 1247 if (ret == -ENOSPC) { 1248 struct nouveau_bo *nvbo; 1249 1250 nvbo = list_first_entry_or_null(&drm->ttm.io_reserve_lru, 1251 typeof(*nvbo), 1252 io_reserve_lru); 1253 if (nvbo) { 1254 list_del_init(&nvbo->io_reserve_lru); 1255 drm_vma_node_unmap(&nvbo->bo.base.vma_node, 1256 bdev->dev_mapping); 1257 nouveau_ttm_io_mem_free_locked(drm, nvbo->bo.resource); 1258 nvbo->bo.resource->bus.offset = 0; 1259 nvbo->bo.resource->bus.addr = NULL; 1260 goto retry; 1261 } 1262 1263 } 1264 mutex_unlock(&drm->ttm.io_reserve_mutex); 1265 return ret; 1266} 1267 1268static void 1269nouveau_ttm_io_mem_free(struct ttm_device *bdev, struct ttm_resource *reg) 1270{ 1271 struct nouveau_drm *drm = nouveau_bdev(bdev); 1272 1273 mutex_lock(&drm->ttm.io_reserve_mutex); 1274 nouveau_ttm_io_mem_free_locked(drm, reg); 1275 mutex_unlock(&drm->ttm.io_reserve_mutex); 1276} 1277 1278vm_fault_t nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo) 1279{ 1280 struct nouveau_drm *drm = nouveau_bdev(bo->bdev); 1281 struct nouveau_bo *nvbo = nouveau_bo(bo); 1282 struct nvkm_device *device = nvxx_device(&drm->client.device); 1283 u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT; 1284 int i, ret; 1285 1286 /* as long as the bo isn't in vram, and isn't tiled, we've got 1287 * nothing to do here. 1288 */ 1289 if (bo->resource->mem_type != TTM_PL_VRAM) { 1290 if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA || 1291 !nvbo->kind) 1292 return 0; 1293 1294 if (bo->resource->mem_type != TTM_PL_SYSTEM) 1295 return 0; 1296 1297 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0); 1298 1299 } else { 1300 /* make sure bo is in mappable vram */ 1301 if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA || 1302 bo->resource->start + PFN_UP(bo->resource->size) < mappable) 1303 return 0; 1304 1305 for (i = 0; i < nvbo->placement.num_placement; ++i) { 1306 nvbo->placements[i].fpfn = 0; 1307 nvbo->placements[i].lpfn = mappable; 1308 } 1309 1310 for (i = 0; i < nvbo->placement.num_busy_placement; ++i) { 1311 nvbo->busy_placements[i].fpfn = 0; 1312 nvbo->busy_placements[i].lpfn = mappable; 1313 } 1314 1315 nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, 0); 1316 } 1317 1318 ret = nouveau_bo_validate(nvbo, false, false); 1319 if (unlikely(ret == -EBUSY || ret == -ERESTARTSYS)) 1320 return VM_FAULT_NOPAGE; 1321 else if (unlikely(ret)) 1322 return VM_FAULT_SIGBUS; 1323 1324 ttm_bo_move_to_lru_tail_unlocked(bo); 1325 return 0; 1326} 1327 1328static int 1329nouveau_ttm_tt_populate(struct ttm_device *bdev, 1330 struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) 1331{ 1332 struct ttm_tt *ttm_dma = (void *)ttm; 1333 struct nouveau_drm *drm; 1334 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1335 1336 if (ttm_tt_is_populated(ttm)) 1337 return 0; 1338 1339 if (slave && ttm->sg) { 1340 drm_prime_sg_to_dma_addr_array(ttm->sg, ttm_dma->dma_address, 1341 ttm->num_pages); 1342 return 0; 1343 } 1344 1345 drm = nouveau_bdev(bdev); 1346 1347 return ttm_pool_alloc(&drm->ttm.bdev.pool, ttm, ctx); 1348} 1349 1350static void 1351nouveau_ttm_tt_unpopulate(struct ttm_device *bdev, 1352 struct ttm_tt *ttm) 1353{ 1354 struct nouveau_drm *drm; 1355 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 1356 1357 if (slave) 1358 return; 1359 1360 nouveau_ttm_tt_unbind(bdev, ttm); 1361 1362 drm = nouveau_bdev(bdev); 1363 1364 return ttm_pool_free(&drm->ttm.bdev.pool, ttm); 1365} 1366 1367static void 1368nouveau_ttm_tt_destroy(struct ttm_device *bdev, 1369 struct ttm_tt *ttm) 1370{ 1371#if IS_ENABLED(CONFIG_AGP) 1372 struct nouveau_drm *drm = nouveau_bdev(bdev); 1373 if (drm->agp.bridge) { 1374 ttm_agp_destroy(ttm); 1375 return; 1376 } 1377#endif 1378 nouveau_sgdma_destroy(bdev, ttm); 1379} 1380 1381void 1382nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive) 1383{ 1384 struct dma_resv *resv = nvbo->bo.base.resv; 1385 1386 if (!fence) 1387 return; 1388 1389 dma_resv_add_fence(resv, &fence->base, exclusive ? 1390 DMA_RESV_USAGE_WRITE : DMA_RESV_USAGE_READ); 1391} 1392 1393static void 1394nouveau_bo_delete_mem_notify(struct ttm_buffer_object *bo) 1395{ 1396 nouveau_bo_move_ntfy(bo, NULL); 1397} 1398 1399struct ttm_device_funcs nouveau_bo_driver = { 1400 .ttm_tt_create = &nouveau_ttm_tt_create, 1401 .ttm_tt_populate = &nouveau_ttm_tt_populate, 1402 .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate, 1403 .ttm_tt_destroy = &nouveau_ttm_tt_destroy, 1404 .eviction_valuable = ttm_bo_eviction_valuable, 1405 .evict_flags = nouveau_bo_evict_flags, 1406 .delete_mem_notify = nouveau_bo_delete_mem_notify, 1407 .move = nouveau_bo_move, 1408 .io_mem_reserve = &nouveau_ttm_io_mem_reserve, 1409 .io_mem_free = &nouveau_ttm_io_mem_free, 1410}; 1411