1// SPDX-License-Identifier: MIT 2/* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6#include <linux/stop_machine.h> 7 8#include <asm/set_memory.h> 9#include <asm/smp.h> 10 11#include <drm/i915_drm.h> 12 13#include "intel_gt.h" 14#include "i915_drv.h" 15#include "i915_scatterlist.h" 16#include "i915_vgpu.h" 17 18#include "intel_gtt.h" 19 20static int 21i915_get_ggtt_vma_pages(struct i915_vma *vma); 22 23static void i915_ggtt_color_adjust(const struct drm_mm_node *node, 24 unsigned long color, 25 u64 *start, 26 u64 *end) 27{ 28 if (i915_node_color_differs(node, color)) 29 *start += I915_GTT_PAGE_SIZE; 30 31 /* 32 * Also leave a space between the unallocated reserved node after the 33 * GTT and any objects within the GTT, i.e. we use the color adjustment 34 * to insert a guard page to prevent prefetches crossing over the 35 * GTT boundary. 36 */ 37 node = list_next_entry(node, node_list); 38 if (node->color != color) 39 *end -= I915_GTT_PAGE_SIZE; 40} 41 42static int ggtt_init_hw(struct i915_ggtt *ggtt) 43{ 44 struct drm_i915_private *i915 = ggtt->vm.i915; 45 46 i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); 47 48 ggtt->vm.is_ggtt = true; 49 50 /* Only VLV supports read-only GGTT mappings */ 51 ggtt->vm.has_read_only = IS_VALLEYVIEW(i915); 52 53 if (!HAS_LLC(i915) && !HAS_PPGTT(i915)) 54 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust; 55 56 if (ggtt->mappable_end) { 57 if (!io_mapping_init_wc(&ggtt->iomap, 58 ggtt->gmadr.start, 59 ggtt->mappable_end)) { 60 ggtt->vm.cleanup(&ggtt->vm); 61 return -EIO; 62 } 63 64 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, 65 ggtt->mappable_end); 66 } 67 68 intel_ggtt_init_fences(ggtt); 69 70 return 0; 71} 72 73/** 74 * i915_ggtt_init_hw - Initialize GGTT hardware 75 * @i915: i915 device 76 */ 77int i915_ggtt_init_hw(struct drm_i915_private *i915) 78{ 79 int ret; 80 81 /* 82 * Note that we use page colouring to enforce a guard page at the 83 * end of the address space. This is required as the CS may prefetch 84 * beyond the end of the batch buffer, across the page boundary, 85 * and beyond the end of the GTT if we do not provide a guard. 86 */ 87 ret = ggtt_init_hw(&i915->ggtt); 88 if (ret) 89 return ret; 90 91 return 0; 92} 93 94/* 95 * Certain Gen5 chipsets require require idling the GPU before 96 * unmapping anything from the GTT when VT-d is enabled. 97 */ 98static bool needs_idle_maps(struct drm_i915_private *i915) 99{ 100 /* 101 * Query intel_iommu to see if we need the workaround. Presumably that 102 * was loaded first. 103 */ 104 return IS_GEN(i915, 5) && IS_MOBILE(i915) && intel_vtd_active(); 105} 106 107void i915_ggtt_suspend(struct i915_ggtt *ggtt) 108{ 109 struct i915_vma *vma, *vn; 110 int open; 111 112 mutex_lock(&ggtt->vm.mutex); 113 114 /* Skip rewriting PTE on VMA unbind. */ 115 open = atomic_xchg(&ggtt->vm.open, 0); 116 117 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { 118 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 119 i915_vma_wait_for_bind(vma); 120 121 if (i915_vma_is_pinned(vma)) 122 continue; 123 124 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) { 125 __i915_vma_evict(vma); 126 drm_mm_remove_node(&vma->node); 127 } 128 } 129 130 ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total); 131 ggtt->invalidate(ggtt); 132 atomic_set(&ggtt->vm.open, open); 133 134 mutex_unlock(&ggtt->vm.mutex); 135 136 intel_gt_check_and_clear_faults(ggtt->vm.gt); 137} 138 139void gen6_ggtt_invalidate(struct i915_ggtt *ggtt) 140{ 141 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 142 143 spin_lock_irq(&uncore->lock); 144 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 145 intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6); 146 spin_unlock_irq(&uncore->lock); 147} 148 149static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt) 150{ 151 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 152 153 /* 154 * Note that as an uncached mmio write, this will flush the 155 * WCB of the writes into the GGTT before it triggers the invalidate. 156 */ 157 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 158} 159 160static void guc_ggtt_invalidate(struct i915_ggtt *ggtt) 161{ 162 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 163 struct drm_i915_private *i915 = ggtt->vm.i915; 164 165 gen8_ggtt_invalidate(ggtt); 166 167 if (INTEL_GEN(i915) >= 12) 168 intel_uncore_write_fw(uncore, GEN12_GUC_TLB_INV_CR, 169 GEN12_GUC_TLB_INV_CR_INVALIDATE); 170 else 171 intel_uncore_write_fw(uncore, GEN8_GTCR, GEN8_GTCR_INVALIDATE); 172} 173 174static void gmch_ggtt_invalidate(struct i915_ggtt *ggtt) 175{ 176 intel_gtt_chipset_flush(); 177} 178 179static u64 gen8_ggtt_pte_encode(dma_addr_t addr, 180 enum i915_cache_level level, 181 u32 flags) 182{ 183 return addr | _PAGE_PRESENT; 184} 185 186static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 187{ 188 writeq(pte, addr); 189} 190 191static void gen8_ggtt_insert_page(struct i915_address_space *vm, 192 dma_addr_t addr, 193 u64 offset, 194 enum i915_cache_level level, 195 u32 unused) 196{ 197 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 198 gen8_pte_t __iomem *pte = 199 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 200 201 gen8_set_pte(pte, gen8_ggtt_pte_encode(addr, level, 0)); 202 203 ggtt->invalidate(ggtt); 204} 205 206static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 207 struct i915_vma *vma, 208 enum i915_cache_level level, 209 u32 flags) 210{ 211 const gen8_pte_t pte_encode = gen8_ggtt_pte_encode(0, level, 0); 212 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 213 gen8_pte_t __iomem *gte; 214 gen8_pte_t __iomem *end; 215 struct sgt_iter iter; 216 dma_addr_t addr; 217 218 /* 219 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 220 * not to allow the user to override access to a read only page. 221 */ 222 223 gte = (gen8_pte_t __iomem *)ggtt->gsm; 224 gte += vma->node.start / I915_GTT_PAGE_SIZE; 225 end = gte + vma->node.size / I915_GTT_PAGE_SIZE; 226 227 for_each_sgt_daddr(addr, iter, vma->pages) 228 gen8_set_pte(gte++, pte_encode | addr); 229 GEM_BUG_ON(gte > end); 230 231 /* Fill the allocated but "unused" space beyond the end of the buffer */ 232 while (gte < end) 233 gen8_set_pte(gte++, vm->scratch[0]->encode); 234 235 /* 236 * We want to flush the TLBs only after we're certain all the PTE 237 * updates have finished. 238 */ 239 ggtt->invalidate(ggtt); 240} 241 242static void gen6_ggtt_insert_page(struct i915_address_space *vm, 243 dma_addr_t addr, 244 u64 offset, 245 enum i915_cache_level level, 246 u32 flags) 247{ 248 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 249 gen6_pte_t __iomem *pte = 250 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 251 252 iowrite32(vm->pte_encode(addr, level, flags), pte); 253 254 ggtt->invalidate(ggtt); 255} 256 257/* 258 * Binds an object into the global gtt with the specified cache level. 259 * The object will be accessible to the GPU via commands whose operands 260 * reference offsets within the global GTT as well as accessible by the GPU 261 * through the GMADR mapped BAR (i915->mm.gtt->gtt). 262 */ 263static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 264 struct i915_vma *vma, 265 enum i915_cache_level level, 266 u32 flags) 267{ 268 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 269 gen6_pte_t __iomem *gte; 270 gen6_pte_t __iomem *end; 271 struct sgt_iter iter; 272 dma_addr_t addr; 273 274 gte = (gen6_pte_t __iomem *)ggtt->gsm; 275 gte += vma->node.start / I915_GTT_PAGE_SIZE; 276 end = gte + vma->node.size / I915_GTT_PAGE_SIZE; 277 278 for_each_sgt_daddr(addr, iter, vma->pages) 279 iowrite32(vm->pte_encode(addr, level, flags), gte++); 280 GEM_BUG_ON(gte > end); 281 282 /* Fill the allocated but "unused" space beyond the end of the buffer */ 283 while (gte < end) 284 iowrite32(vm->scratch[0]->encode, gte++); 285 286 /* 287 * We want to flush the TLBs only after we're certain all the PTE 288 * updates have finished. 289 */ 290 ggtt->invalidate(ggtt); 291} 292 293static void nop_clear_range(struct i915_address_space *vm, 294 u64 start, u64 length) 295{ 296} 297 298static void gen8_ggtt_clear_range(struct i915_address_space *vm, 299 u64 start, u64 length) 300{ 301 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 302 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 303 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 304 const gen8_pte_t scratch_pte = vm->scratch[0]->encode; 305 gen8_pte_t __iomem *gtt_base = 306 (gen8_pte_t __iomem *)ggtt->gsm + first_entry; 307 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 308 int i; 309 310 if (WARN(num_entries > max_entries, 311 "First entry = %d; Num entries = %d (max=%d)\n", 312 first_entry, num_entries, max_entries)) 313 num_entries = max_entries; 314 315 for (i = 0; i < num_entries; i++) 316 gen8_set_pte(>t_base[i], scratch_pte); 317} 318 319static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) 320{ 321 /* 322 * Make sure the internal GAM fifo has been cleared of all GTT 323 * writes before exiting stop_machine(). This guarantees that 324 * any aperture accesses waiting to start in another process 325 * cannot back up behind the GTT writes causing a hang. 326 * The register can be any arbitrary GAM register. 327 */ 328 intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6); 329} 330 331struct insert_page { 332 struct i915_address_space *vm; 333 dma_addr_t addr; 334 u64 offset; 335 enum i915_cache_level level; 336}; 337 338static int bxt_vtd_ggtt_insert_page__cb(void *_arg) 339{ 340 struct insert_page *arg = _arg; 341 342 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, arg->level, 0); 343 bxt_vtd_ggtt_wa(arg->vm); 344 345 return 0; 346} 347 348static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, 349 dma_addr_t addr, 350 u64 offset, 351 enum i915_cache_level level, 352 u32 unused) 353{ 354 struct insert_page arg = { vm, addr, offset, level }; 355 356 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); 357} 358 359struct insert_entries { 360 struct i915_address_space *vm; 361 struct i915_vma *vma; 362 enum i915_cache_level level; 363 u32 flags; 364}; 365 366static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) 367{ 368 struct insert_entries *arg = _arg; 369 370 gen8_ggtt_insert_entries(arg->vm, arg->vma, arg->level, arg->flags); 371 bxt_vtd_ggtt_wa(arg->vm); 372 373 return 0; 374} 375 376static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, 377 struct i915_vma *vma, 378 enum i915_cache_level level, 379 u32 flags) 380{ 381 struct insert_entries arg = { vm, vma, level, flags }; 382 383 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); 384} 385 386static void gen6_ggtt_clear_range(struct i915_address_space *vm, 387 u64 start, u64 length) 388{ 389 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 390 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 391 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 392 gen6_pte_t scratch_pte, __iomem *gtt_base = 393 (gen6_pte_t __iomem *)ggtt->gsm + first_entry; 394 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 395 int i; 396 397 if (WARN(num_entries > max_entries, 398 "First entry = %d; Num entries = %d (max=%d)\n", 399 first_entry, num_entries, max_entries)) 400 num_entries = max_entries; 401 402 scratch_pte = vm->scratch[0]->encode; 403 for (i = 0; i < num_entries; i++) 404 iowrite32(scratch_pte, >t_base[i]); 405} 406 407static void i915_ggtt_insert_page(struct i915_address_space *vm, 408 dma_addr_t addr, 409 u64 offset, 410 enum i915_cache_level cache_level, 411 u32 unused) 412{ 413 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 414 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 415 416 intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags); 417} 418 419static void i915_ggtt_insert_entries(struct i915_address_space *vm, 420 struct i915_vma *vma, 421 enum i915_cache_level cache_level, 422 u32 unused) 423{ 424 unsigned int flags = (cache_level == I915_CACHE_NONE) ? 425 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY; 426 427 intel_gtt_insert_sg_entries(vma->pages, vma->node.start >> PAGE_SHIFT, 428 flags); 429} 430 431static void i915_ggtt_clear_range(struct i915_address_space *vm, 432 u64 start, u64 length) 433{ 434 intel_gtt_clear_range(start >> PAGE_SHIFT, length >> PAGE_SHIFT); 435} 436 437static void ggtt_bind_vma(struct i915_address_space *vm, 438 struct i915_vm_pt_stash *stash, 439 struct i915_vma *vma, 440 enum i915_cache_level cache_level, 441 u32 flags) 442{ 443 struct drm_i915_gem_object *obj = vma->obj; 444 u32 pte_flags; 445 446 if (i915_vma_is_bound(vma, ~flags & I915_VMA_BIND_MASK)) 447 return; 448 449 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 450 pte_flags = 0; 451 if (i915_gem_object_is_readonly(obj)) 452 pte_flags |= PTE_READ_ONLY; 453 454 vm->insert_entries(vm, vma, cache_level, pte_flags); 455 vma->page_sizes.gtt = I915_GTT_PAGE_SIZE; 456} 457 458static void ggtt_unbind_vma(struct i915_address_space *vm, struct i915_vma *vma) 459{ 460 vm->clear_range(vm, vma->node.start, vma->size); 461} 462 463static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) 464{ 465 u64 size; 466 int ret; 467 468 if (!intel_uc_uses_guc(&ggtt->vm.gt->uc)) 469 return 0; 470 471 GEM_BUG_ON(ggtt->vm.total <= GUC_GGTT_TOP); 472 size = ggtt->vm.total - GUC_GGTT_TOP; 473 474 ret = i915_gem_gtt_reserve(&ggtt->vm, &ggtt->uc_fw, size, 475 GUC_GGTT_TOP, I915_COLOR_UNEVICTABLE, 476 PIN_NOEVICT); 477 if (ret) 478 drm_dbg(&ggtt->vm.i915->drm, 479 "Failed to reserve top of GGTT for GuC\n"); 480 481 return ret; 482} 483 484static void ggtt_release_guc_top(struct i915_ggtt *ggtt) 485{ 486 if (drm_mm_node_allocated(&ggtt->uc_fw)) 487 drm_mm_remove_node(&ggtt->uc_fw); 488} 489 490static void cleanup_init_ggtt(struct i915_ggtt *ggtt) 491{ 492 ggtt_release_guc_top(ggtt); 493 if (drm_mm_node_allocated(&ggtt->error_capture)) 494 drm_mm_remove_node(&ggtt->error_capture); 495 mutex_destroy(&ggtt->error_mutex); 496} 497 498static int init_ggtt(struct i915_ggtt *ggtt) 499{ 500 /* 501 * Let GEM Manage all of the aperture. 502 * 503 * However, leave one page at the end still bound to the scratch page. 504 * There are a number of places where the hardware apparently prefetches 505 * past the end of the object, and we've seen multiple hangs with the 506 * GPU head pointer stuck in a batchbuffer bound at the last page of the 507 * aperture. One page should be enough to keep any prefetching inside 508 * of the aperture. 509 */ 510 unsigned long hole_start, hole_end; 511 struct drm_mm_node *entry; 512 int ret; 513 514 /* 515 * GuC requires all resources that we're sharing with it to be placed in 516 * non-WOPCM memory. If GuC is not present or not in use we still need a 517 * small bias as ring wraparound at offset 0 sometimes hangs. No idea 518 * why. 519 */ 520 ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE, 521 intel_wopcm_guc_size(&ggtt->vm.i915->wopcm)); 522 523 ret = intel_vgt_balloon(ggtt); 524 if (ret) 525 return ret; 526 527 mutex_init(&ggtt->error_mutex); 528 if (ggtt->mappable_end) { 529 /* 530 * Reserve a mappable slot for our lockless error capture. 531 * 532 * We strongly prefer taking address 0x0 in order to protect 533 * other critical buffers against accidental overwrites, 534 * as writing to address 0 is a very common mistake. 535 * 536 * Since 0 may already be in use by the system (e.g. the BIOS 537 * framebuffer), we let the reservation fail quietly and hope 538 * 0 remains reserved always. 539 * 540 * If we fail to reserve 0, and then fail to find any space 541 * for an error-capture, remain silent. We can afford not 542 * to reserve an error_capture node as we have fallback 543 * paths, and we trust that 0 will remain reserved. However, 544 * the only likely reason for failure to insert is a driver 545 * bug, which we expect to cause other failures... 546 */ 547 ggtt->error_capture.size = I915_GTT_PAGE_SIZE; 548 ggtt->error_capture.color = I915_COLOR_UNEVICTABLE; 549 if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture)) 550 drm_mm_insert_node_in_range(&ggtt->vm.mm, 551 &ggtt->error_capture, 552 ggtt->error_capture.size, 0, 553 ggtt->error_capture.color, 554 0, ggtt->mappable_end, 555 DRM_MM_INSERT_LOW); 556 } 557 if (drm_mm_node_allocated(&ggtt->error_capture)) 558 drm_dbg(&ggtt->vm.i915->drm, 559 "Reserved GGTT:[%llx, %llx] for use by error capture\n", 560 ggtt->error_capture.start, 561 ggtt->error_capture.start + ggtt->error_capture.size); 562 563 /* 564 * The upper portion of the GuC address space has a sizeable hole 565 * (several MB) that is inaccessible by GuC. Reserve this range within 566 * GGTT as it can comfortably hold GuC/HuC firmware images. 567 */ 568 ret = ggtt_reserve_guc_top(ggtt); 569 if (ret) 570 goto err; 571 572 /* Clear any non-preallocated blocks */ 573 drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) { 574 drm_dbg(&ggtt->vm.i915->drm, 575 "clearing unused GTT space: [%lx, %lx]\n", 576 hole_start, hole_end); 577 ggtt->vm.clear_range(&ggtt->vm, hole_start, 578 hole_end - hole_start); 579 } 580 581 /* And finally clear the reserved guard page */ 582 ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE); 583 584 return 0; 585 586err: 587 cleanup_init_ggtt(ggtt); 588 return ret; 589} 590 591static void aliasing_gtt_bind_vma(struct i915_address_space *vm, 592 struct i915_vm_pt_stash *stash, 593 struct i915_vma *vma, 594 enum i915_cache_level cache_level, 595 u32 flags) 596{ 597 u32 pte_flags; 598 599 /* Currently applicable only to VLV */ 600 pte_flags = 0; 601 if (i915_gem_object_is_readonly(vma->obj)) 602 pte_flags |= PTE_READ_ONLY; 603 604 if (flags & I915_VMA_LOCAL_BIND) 605 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, 606 stash, vma, cache_level, flags); 607 608 if (flags & I915_VMA_GLOBAL_BIND) 609 vm->insert_entries(vm, vma, cache_level, pte_flags); 610} 611 612static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, 613 struct i915_vma *vma) 614{ 615 if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 616 vm->clear_range(vm, vma->node.start, vma->size); 617 618 if (i915_vma_is_bound(vma, I915_VMA_LOCAL_BIND)) 619 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma); 620} 621 622static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) 623{ 624 struct i915_vm_pt_stash stash = {}; 625 struct i915_ppgtt *ppgtt; 626 int err; 627 628 ppgtt = i915_ppgtt_create(ggtt->vm.gt); 629 if (IS_ERR(ppgtt)) 630 return PTR_ERR(ppgtt); 631 632 if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) { 633 err = -ENODEV; 634 goto err_ppgtt; 635 } 636 637 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total); 638 if (err) 639 goto err_ppgtt; 640 641 err = i915_vm_pin_pt_stash(&ppgtt->vm, &stash); 642 if (err) 643 goto err_stash; 644 645 /* 646 * Note we only pre-allocate as far as the end of the global 647 * GTT. On 48b / 4-level page-tables, the difference is very, 648 * very significant! We have to preallocate as GVT/vgpu does 649 * not like the page directory disappearing. 650 */ 651 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total); 652 653 ggtt->alias = ppgtt; 654 ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags; 655 656 GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != ggtt_bind_vma); 657 ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma; 658 659 GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != ggtt_unbind_vma); 660 ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma; 661 662 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 663 return 0; 664 665err_stash: 666 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 667err_ppgtt: 668 i915_vm_put(&ppgtt->vm); 669 return err; 670} 671 672static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt) 673{ 674 struct i915_ppgtt *ppgtt; 675 676 ppgtt = fetch_and_zero(&ggtt->alias); 677 if (!ppgtt) 678 return; 679 680 i915_vm_put(&ppgtt->vm); 681 682 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 683 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 684} 685 686int i915_init_ggtt(struct drm_i915_private *i915) 687{ 688 int ret; 689 690 ret = init_ggtt(&i915->ggtt); 691 if (ret) 692 return ret; 693 694 if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) { 695 ret = init_aliasing_ppgtt(&i915->ggtt); 696 if (ret) 697 cleanup_init_ggtt(&i915->ggtt); 698 } 699 700 return 0; 701} 702 703static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) 704{ 705 struct i915_vma *vma, *vn; 706 707 atomic_set(&ggtt->vm.open, 0); 708 709 rcu_barrier(); /* flush the RCU'ed__i915_vm_release */ 710 flush_workqueue(ggtt->vm.i915->wq); 711 712 mutex_lock(&ggtt->vm.mutex); 713 714 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) 715 WARN_ON(__i915_vma_unbind(vma)); 716 717 if (drm_mm_node_allocated(&ggtt->error_capture)) 718 drm_mm_remove_node(&ggtt->error_capture); 719 mutex_destroy(&ggtt->error_mutex); 720 721 ggtt_release_guc_top(ggtt); 722 intel_vgt_deballoon(ggtt); 723 724 ggtt->vm.cleanup(&ggtt->vm); 725 726 mutex_unlock(&ggtt->vm.mutex); 727 i915_address_space_fini(&ggtt->vm); 728 729 arch_phys_wc_del(ggtt->mtrr); 730 731 if (ggtt->iomap.size) 732 io_mapping_fini(&ggtt->iomap); 733} 734 735/** 736 * i915_ggtt_driver_release - Clean up GGTT hardware initialization 737 * @i915: i915 device 738 */ 739void i915_ggtt_driver_release(struct drm_i915_private *i915) 740{ 741 struct i915_ggtt *ggtt = &i915->ggtt; 742 743 fini_aliasing_ppgtt(ggtt); 744 745 intel_ggtt_fini_fences(ggtt); 746 ggtt_cleanup_hw(ggtt); 747} 748 749static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 750{ 751 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 752 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 753 return snb_gmch_ctl << 20; 754} 755 756static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 757{ 758 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 759 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 760 if (bdw_gmch_ctl) 761 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 762 763#ifdef CONFIG_X86_32 764 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */ 765 if (bdw_gmch_ctl > 4) 766 bdw_gmch_ctl = 4; 767#endif 768 769 return bdw_gmch_ctl << 20; 770} 771 772static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 773{ 774 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 775 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 776 777 if (gmch_ctrl) 778 return 1 << (20 + gmch_ctrl); 779 780 return 0; 781} 782 783static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) 784{ 785 struct drm_i915_private *i915 = ggtt->vm.i915; 786 struct pci_dev *pdev = i915->drm.pdev; 787 phys_addr_t phys_addr; 788 int ret; 789 790 /* For Modern GENs the PTEs and register space are split in the BAR */ 791 phys_addr = pci_resource_start(pdev, 0) + pci_resource_len(pdev, 0) / 2; 792 793 /* 794 * On BXT+/CNL+ writes larger than 64 bit to the GTT pagetable range 795 * will be dropped. For WC mappings in general we have 64 byte burst 796 * writes when the WC buffer is flushed, so we can't use it, but have to 797 * resort to an uncached mapping. The WC issue is easily caught by the 798 * readback check when writing GTT PTE entries. 799 */ 800 if (IS_GEN9_LP(i915) || INTEL_GEN(i915) >= 10) 801 ggtt->gsm = ioremap(phys_addr, size); 802 else 803 ggtt->gsm = ioremap_wc(phys_addr, size); 804 if (!ggtt->gsm) { 805 drm_err(&i915->drm, "Failed to map the ggtt page table\n"); 806 return -ENOMEM; 807 } 808 809 ret = setup_scratch_page(&ggtt->vm); 810 if (ret) { 811 drm_err(&i915->drm, "Scratch setup failed\n"); 812 /* iounmap will also get called at remove, but meh */ 813 iounmap(ggtt->gsm); 814 return ret; 815 } 816 817 ggtt->vm.scratch[0]->encode = 818 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]), 819 I915_CACHE_NONE, 0); 820 821 return 0; 822} 823 824int ggtt_set_pages(struct i915_vma *vma) 825{ 826 int ret; 827 828 GEM_BUG_ON(vma->pages); 829 830 ret = i915_get_ggtt_vma_pages(vma); 831 if (ret) 832 return ret; 833 834 vma->page_sizes = vma->obj->mm.page_sizes; 835 836 return 0; 837} 838 839static void gen6_gmch_remove(struct i915_address_space *vm) 840{ 841 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 842 843 iounmap(ggtt->gsm); 844 free_scratch(vm); 845} 846 847static struct resource pci_resource(struct pci_dev *pdev, int bar) 848{ 849 return (struct resource)DEFINE_RES_MEM(pci_resource_start(pdev, bar), 850 pci_resource_len(pdev, bar)); 851} 852 853static int gen8_gmch_probe(struct i915_ggtt *ggtt) 854{ 855 struct drm_i915_private *i915 = ggtt->vm.i915; 856 struct pci_dev *pdev = i915->drm.pdev; 857 unsigned int size; 858 u16 snb_gmch_ctl; 859 860 /* TODO: We're not aware of mappable constraints on gen8 yet */ 861 if (!IS_DGFX(i915)) { 862 ggtt->gmadr = pci_resource(pdev, 2); 863 ggtt->mappable_end = resource_size(&ggtt->gmadr); 864 } 865 866 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 867 if (IS_CHERRYVIEW(i915)) 868 size = chv_get_total_gtt_size(snb_gmch_ctl); 869 else 870 size = gen8_get_total_gtt_size(snb_gmch_ctl); 871 872 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 873 874 ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 875 ggtt->vm.cleanup = gen6_gmch_remove; 876 ggtt->vm.insert_page = gen8_ggtt_insert_page; 877 ggtt->vm.clear_range = nop_clear_range; 878 if (intel_scanout_needs_vtd_wa(i915)) 879 ggtt->vm.clear_range = gen8_ggtt_clear_range; 880 881 ggtt->vm.insert_entries = gen8_ggtt_insert_entries; 882 883 /* Serialize GTT updates with aperture access on BXT if VT-d is on. */ 884 if (intel_ggtt_update_needs_vtd_wa(i915) || 885 IS_CHERRYVIEW(i915) /* fails with concurrent use/update */) { 886 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; 887 ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; 888 ggtt->vm.bind_async_flags = 889 I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 890 } 891 892 ggtt->invalidate = gen8_ggtt_invalidate; 893 894 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 895 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 896 ggtt->vm.vma_ops.set_pages = ggtt_set_pages; 897 ggtt->vm.vma_ops.clear_pages = clear_pages; 898 899 ggtt->vm.pte_encode = gen8_ggtt_pte_encode; 900 901 setup_private_pat(ggtt->vm.gt->uncore); 902 903 return ggtt_probe_common(ggtt, size); 904} 905 906static u64 snb_pte_encode(dma_addr_t addr, 907 enum i915_cache_level level, 908 u32 flags) 909{ 910 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 911 912 switch (level) { 913 case I915_CACHE_L3_LLC: 914 case I915_CACHE_LLC: 915 pte |= GEN6_PTE_CACHE_LLC; 916 break; 917 case I915_CACHE_NONE: 918 pte |= GEN6_PTE_UNCACHED; 919 break; 920 default: 921 MISSING_CASE(level); 922 } 923 924 return pte; 925} 926 927static u64 ivb_pte_encode(dma_addr_t addr, 928 enum i915_cache_level level, 929 u32 flags) 930{ 931 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 932 933 switch (level) { 934 case I915_CACHE_L3_LLC: 935 pte |= GEN7_PTE_CACHE_L3_LLC; 936 break; 937 case I915_CACHE_LLC: 938 pte |= GEN6_PTE_CACHE_LLC; 939 break; 940 case I915_CACHE_NONE: 941 pte |= GEN6_PTE_UNCACHED; 942 break; 943 default: 944 MISSING_CASE(level); 945 } 946 947 return pte; 948} 949 950static u64 byt_pte_encode(dma_addr_t addr, 951 enum i915_cache_level level, 952 u32 flags) 953{ 954 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 955 956 if (!(flags & PTE_READ_ONLY)) 957 pte |= BYT_PTE_WRITEABLE; 958 959 if (level != I915_CACHE_NONE) 960 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 961 962 return pte; 963} 964 965static u64 hsw_pte_encode(dma_addr_t addr, 966 enum i915_cache_level level, 967 u32 flags) 968{ 969 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 970 971 if (level != I915_CACHE_NONE) 972 pte |= HSW_WB_LLC_AGE3; 973 974 return pte; 975} 976 977static u64 iris_pte_encode(dma_addr_t addr, 978 enum i915_cache_level level, 979 u32 flags) 980{ 981 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 982 983 switch (level) { 984 case I915_CACHE_NONE: 985 break; 986 case I915_CACHE_WT: 987 pte |= HSW_WT_ELLC_LLC_AGE3; 988 break; 989 default: 990 pte |= HSW_WB_ELLC_LLC_AGE3; 991 break; 992 } 993 994 return pte; 995} 996 997static int gen6_gmch_probe(struct i915_ggtt *ggtt) 998{ 999 struct drm_i915_private *i915 = ggtt->vm.i915; 1000 struct pci_dev *pdev = i915->drm.pdev; 1001 unsigned int size; 1002 u16 snb_gmch_ctl; 1003 1004 ggtt->gmadr = pci_resource(pdev, 2); 1005 ggtt->mappable_end = resource_size(&ggtt->gmadr); 1006 1007 /* 1008 * 64/512MB is the current min/max we actually know of, but this is 1009 * just a coarse sanity check. 1010 */ 1011 if (ggtt->mappable_end < (64<<20) || ggtt->mappable_end > (512<<20)) { 1012 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n", 1013 &ggtt->mappable_end); 1014 return -ENXIO; 1015 } 1016 1017 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 1018 1019 size = gen6_get_total_gtt_size(snb_gmch_ctl); 1020 ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE; 1021 1022 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1023 1024 ggtt->vm.clear_range = nop_clear_range; 1025 if (!HAS_FULL_PPGTT(i915) || intel_scanout_needs_vtd_wa(i915)) 1026 ggtt->vm.clear_range = gen6_ggtt_clear_range; 1027 ggtt->vm.insert_page = gen6_ggtt_insert_page; 1028 ggtt->vm.insert_entries = gen6_ggtt_insert_entries; 1029 ggtt->vm.cleanup = gen6_gmch_remove; 1030 1031 ggtt->invalidate = gen6_ggtt_invalidate; 1032 1033 if (HAS_EDRAM(i915)) 1034 ggtt->vm.pte_encode = iris_pte_encode; 1035 else if (IS_HASWELL(i915)) 1036 ggtt->vm.pte_encode = hsw_pte_encode; 1037 else if (IS_VALLEYVIEW(i915)) 1038 ggtt->vm.pte_encode = byt_pte_encode; 1039 else if (INTEL_GEN(i915) >= 7) 1040 ggtt->vm.pte_encode = ivb_pte_encode; 1041 else 1042 ggtt->vm.pte_encode = snb_pte_encode; 1043 1044 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 1045 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 1046 ggtt->vm.vma_ops.set_pages = ggtt_set_pages; 1047 ggtt->vm.vma_ops.clear_pages = clear_pages; 1048 1049 return ggtt_probe_common(ggtt, size); 1050} 1051 1052static void i915_gmch_remove(struct i915_address_space *vm) 1053{ 1054 intel_gmch_remove(); 1055} 1056 1057static int i915_gmch_probe(struct i915_ggtt *ggtt) 1058{ 1059 struct drm_i915_private *i915 = ggtt->vm.i915; 1060 phys_addr_t gmadr_base; 1061 int ret; 1062 1063 ret = intel_gmch_probe(i915->bridge_dev, i915->drm.pdev, NULL); 1064 if (!ret) { 1065 drm_err(&i915->drm, "failed to set up gmch\n"); 1066 return -EIO; 1067 } 1068 1069 intel_gtt_get(&ggtt->vm.total, &gmadr_base, &ggtt->mappable_end); 1070 1071 ggtt->gmadr = 1072 (struct resource)DEFINE_RES_MEM(gmadr_base, ggtt->mappable_end); 1073 1074 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1075 1076 ggtt->do_idle_maps = needs_idle_maps(i915); 1077 ggtt->vm.insert_page = i915_ggtt_insert_page; 1078 ggtt->vm.insert_entries = i915_ggtt_insert_entries; 1079 ggtt->vm.clear_range = i915_ggtt_clear_range; 1080 ggtt->vm.cleanup = i915_gmch_remove; 1081 1082 ggtt->invalidate = gmch_ggtt_invalidate; 1083 1084 ggtt->vm.vma_ops.bind_vma = ggtt_bind_vma; 1085 ggtt->vm.vma_ops.unbind_vma = ggtt_unbind_vma; 1086 ggtt->vm.vma_ops.set_pages = ggtt_set_pages; 1087 ggtt->vm.vma_ops.clear_pages = clear_pages; 1088 1089 if (unlikely(ggtt->do_idle_maps)) 1090 drm_notice(&i915->drm, 1091 "Applying Ironlake quirks for intel_iommu\n"); 1092 1093 return 0; 1094} 1095 1096static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) 1097{ 1098 struct drm_i915_private *i915 = gt->i915; 1099 int ret; 1100 1101 ggtt->vm.gt = gt; 1102 ggtt->vm.i915 = i915; 1103 ggtt->vm.dma = &i915->drm.pdev->dev; 1104 1105 if (INTEL_GEN(i915) <= 5) 1106 ret = i915_gmch_probe(ggtt); 1107 else if (INTEL_GEN(i915) < 8) 1108 ret = gen6_gmch_probe(ggtt); 1109 else 1110 ret = gen8_gmch_probe(ggtt); 1111 if (ret) 1112 return ret; 1113 1114 if ((ggtt->vm.total - 1) >> 32) { 1115 drm_err(&i915->drm, 1116 "We never expected a Global GTT with more than 32bits" 1117 " of address space! Found %lldM!\n", 1118 ggtt->vm.total >> 20); 1119 ggtt->vm.total = 1ULL << 32; 1120 ggtt->mappable_end = 1121 min_t(u64, ggtt->mappable_end, ggtt->vm.total); 1122 } 1123 1124 if (ggtt->mappable_end > ggtt->vm.total) { 1125 drm_err(&i915->drm, 1126 "mappable aperture extends past end of GGTT," 1127 " aperture=%pa, total=%llx\n", 1128 &ggtt->mappable_end, ggtt->vm.total); 1129 ggtt->mappable_end = ggtt->vm.total; 1130 } 1131 1132 /* GMADR is the PCI mmio aperture into the global GTT. */ 1133 drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20); 1134 drm_dbg(&i915->drm, "GMADR size = %lluM\n", 1135 (u64)ggtt->mappable_end >> 20); 1136 drm_dbg(&i915->drm, "DSM size = %lluM\n", 1137 (u64)resource_size(&intel_graphics_stolen_res) >> 20); 1138 1139 return 0; 1140} 1141 1142/** 1143 * i915_ggtt_probe_hw - Probe GGTT hardware location 1144 * @i915: i915 device 1145 */ 1146int i915_ggtt_probe_hw(struct drm_i915_private *i915) 1147{ 1148 int ret; 1149 1150 ret = ggtt_probe_hw(&i915->ggtt, &i915->gt); 1151 if (ret) 1152 return ret; 1153 1154 if (intel_vtd_active()) 1155 drm_info(&i915->drm, "VT-d active for gfx access\n"); 1156 1157 return 0; 1158} 1159 1160int i915_ggtt_enable_hw(struct drm_i915_private *i915) 1161{ 1162 if (INTEL_GEN(i915) < 6 && !intel_enable_gtt()) 1163 return -EIO; 1164 1165 return 0; 1166} 1167 1168void i915_ggtt_enable_guc(struct i915_ggtt *ggtt) 1169{ 1170 GEM_BUG_ON(ggtt->invalidate != gen8_ggtt_invalidate); 1171 1172 ggtt->invalidate = guc_ggtt_invalidate; 1173 1174 ggtt->invalidate(ggtt); 1175} 1176 1177void i915_ggtt_disable_guc(struct i915_ggtt *ggtt) 1178{ 1179 /* XXX Temporary pardon for error unload */ 1180 if (ggtt->invalidate == gen8_ggtt_invalidate) 1181 return; 1182 1183 /* We should only be called after i915_ggtt_enable_guc() */ 1184 GEM_BUG_ON(ggtt->invalidate != guc_ggtt_invalidate); 1185 1186 ggtt->invalidate = gen8_ggtt_invalidate; 1187 1188 ggtt->invalidate(ggtt); 1189} 1190 1191void i915_ggtt_resume(struct i915_ggtt *ggtt) 1192{ 1193 struct i915_vma *vma; 1194 bool flush = false; 1195 int open; 1196 1197 intel_gt_check_and_clear_faults(ggtt->vm.gt); 1198 1199 /* First fill our portion of the GTT with scratch pages */ 1200 ggtt->vm.clear_range(&ggtt->vm, 0, ggtt->vm.total); 1201 1202 /* Skip rewriting PTE on VMA unbind. */ 1203 open = atomic_xchg(&ggtt->vm.open, 0); 1204 1205 /* clflush objects bound into the GGTT and rebind them. */ 1206 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link) { 1207 struct drm_i915_gem_object *obj = vma->obj; 1208 unsigned int was_bound = 1209 atomic_read(&vma->flags) & I915_VMA_BIND_MASK; 1210 1211 GEM_BUG_ON(!was_bound); 1212 vma->ops->bind_vma(&ggtt->vm, NULL, vma, 1213 obj ? obj->cache_level : 0, 1214 was_bound); 1215 if (obj) { /* only used during resume => exclusive access */ 1216 flush |= fetch_and_zero(&obj->write_domain); 1217 obj->read_domains |= I915_GEM_DOMAIN_GTT; 1218 } 1219 } 1220 1221 atomic_set(&ggtt->vm.open, open); 1222 ggtt->invalidate(ggtt); 1223 1224 if (flush) 1225 wbinvd_on_all_cpus(); 1226 1227 if (INTEL_GEN(ggtt->vm.i915) >= 8) 1228 setup_private_pat(ggtt->vm.gt->uncore); 1229 1230 intel_ggtt_restore_fences(ggtt); 1231} 1232 1233static struct scatterlist * 1234rotate_pages(struct drm_i915_gem_object *obj, unsigned int offset, 1235 unsigned int width, unsigned int height, 1236 unsigned int stride, 1237 struct sg_table *st, struct scatterlist *sg) 1238{ 1239 unsigned int column, row; 1240 unsigned int src_idx; 1241 1242 for (column = 0; column < width; column++) { 1243 src_idx = stride * (height - 1) + column + offset; 1244 for (row = 0; row < height; row++) { 1245 st->nents++; 1246 /* 1247 * We don't need the pages, but need to initialize 1248 * the entries so the sg list can be happily traversed. 1249 * The only thing we need are DMA addresses. 1250 */ 1251 sg_set_page(sg, NULL, I915_GTT_PAGE_SIZE, 0); 1252 sg_dma_address(sg) = 1253 i915_gem_object_get_dma_address(obj, src_idx); 1254 sg_dma_len(sg) = I915_GTT_PAGE_SIZE; 1255 sg = sg_next(sg); 1256 src_idx -= stride; 1257 } 1258 } 1259 1260 return sg; 1261} 1262 1263static noinline struct sg_table * 1264intel_rotate_pages(struct intel_rotation_info *rot_info, 1265 struct drm_i915_gem_object *obj) 1266{ 1267 unsigned int size = intel_rotation_info_size(rot_info); 1268 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1269 struct sg_table *st; 1270 struct scatterlist *sg; 1271 int ret = -ENOMEM; 1272 int i; 1273 1274 /* Allocate target SG list. */ 1275 st = kmalloc(sizeof(*st), GFP_KERNEL); 1276 if (!st) 1277 goto err_st_alloc; 1278 1279 ret = sg_alloc_table(st, size, GFP_KERNEL); 1280 if (ret) 1281 goto err_sg_alloc; 1282 1283 st->nents = 0; 1284 sg = st->sgl; 1285 1286 for (i = 0 ; i < ARRAY_SIZE(rot_info->plane); i++) { 1287 sg = rotate_pages(obj, rot_info->plane[i].offset, 1288 rot_info->plane[i].width, rot_info->plane[i].height, 1289 rot_info->plane[i].stride, st, sg); 1290 } 1291 1292 return st; 1293 1294err_sg_alloc: 1295 kfree(st); 1296err_st_alloc: 1297 1298 drm_dbg(&i915->drm, "Failed to create rotated mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1299 obj->base.size, rot_info->plane[0].width, 1300 rot_info->plane[0].height, size); 1301 1302 return ERR_PTR(ret); 1303} 1304 1305static struct scatterlist * 1306remap_pages(struct drm_i915_gem_object *obj, unsigned int offset, 1307 unsigned int width, unsigned int height, 1308 unsigned int stride, 1309 struct sg_table *st, struct scatterlist *sg) 1310{ 1311 unsigned int row; 1312 1313 for (row = 0; row < height; row++) { 1314 unsigned int left = width * I915_GTT_PAGE_SIZE; 1315 1316 while (left) { 1317 dma_addr_t addr; 1318 unsigned int length; 1319 1320 /* 1321 * We don't need the pages, but need to initialize 1322 * the entries so the sg list can be happily traversed. 1323 * The only thing we need are DMA addresses. 1324 */ 1325 1326 addr = i915_gem_object_get_dma_address_len(obj, offset, &length); 1327 1328 length = min(left, length); 1329 1330 st->nents++; 1331 1332 sg_set_page(sg, NULL, length, 0); 1333 sg_dma_address(sg) = addr; 1334 sg_dma_len(sg) = length; 1335 sg = sg_next(sg); 1336 1337 offset += length / I915_GTT_PAGE_SIZE; 1338 left -= length; 1339 } 1340 1341 offset += stride - width; 1342 } 1343 1344 return sg; 1345} 1346 1347static noinline struct sg_table * 1348intel_remap_pages(struct intel_remapped_info *rem_info, 1349 struct drm_i915_gem_object *obj) 1350{ 1351 unsigned int size = intel_remapped_info_size(rem_info); 1352 struct drm_i915_private *i915 = to_i915(obj->base.dev); 1353 struct sg_table *st; 1354 struct scatterlist *sg; 1355 int ret = -ENOMEM; 1356 int i; 1357 1358 /* Allocate target SG list. */ 1359 st = kmalloc(sizeof(*st), GFP_KERNEL); 1360 if (!st) 1361 goto err_st_alloc; 1362 1363 ret = sg_alloc_table(st, size, GFP_KERNEL); 1364 if (ret) 1365 goto err_sg_alloc; 1366 1367 st->nents = 0; 1368 sg = st->sgl; 1369 1370 for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) { 1371 sg = remap_pages(obj, rem_info->plane[i].offset, 1372 rem_info->plane[i].width, rem_info->plane[i].height, 1373 rem_info->plane[i].stride, st, sg); 1374 } 1375 1376 i915_sg_trim(st); 1377 1378 return st; 1379 1380err_sg_alloc: 1381 kfree(st); 1382err_st_alloc: 1383 1384 drm_dbg(&i915->drm, "Failed to create remapped mapping for object size %zu! (%ux%u tiles, %u pages)\n", 1385 obj->base.size, rem_info->plane[0].width, 1386 rem_info->plane[0].height, size); 1387 1388 return ERR_PTR(ret); 1389} 1390 1391static noinline struct sg_table * 1392intel_partial_pages(const struct i915_ggtt_view *view, 1393 struct drm_i915_gem_object *obj) 1394{ 1395 struct sg_table *st; 1396 struct scatterlist *sg, *iter; 1397 unsigned int count = view->partial.size; 1398 unsigned int offset; 1399 int ret = -ENOMEM; 1400 1401 st = kmalloc(sizeof(*st), GFP_KERNEL); 1402 if (!st) 1403 goto err_st_alloc; 1404 1405 ret = sg_alloc_table(st, count, GFP_KERNEL); 1406 if (ret) 1407 goto err_sg_alloc; 1408 1409 iter = i915_gem_object_get_sg(obj, view->partial.offset, &offset); 1410 GEM_BUG_ON(!iter); 1411 1412 sg = st->sgl; 1413 st->nents = 0; 1414 do { 1415 unsigned int len; 1416 1417 len = min(iter->length - (offset << PAGE_SHIFT), 1418 count << PAGE_SHIFT); 1419 sg_set_page(sg, NULL, len, 0); 1420 sg_dma_address(sg) = 1421 sg_dma_address(iter) + (offset << PAGE_SHIFT); 1422 sg_dma_len(sg) = len; 1423 1424 st->nents++; 1425 count -= len >> PAGE_SHIFT; 1426 if (count == 0) { 1427 sg_mark_end(sg); 1428 i915_sg_trim(st); /* Drop any unused tail entries. */ 1429 1430 return st; 1431 } 1432 1433 sg = __sg_next(sg); 1434 iter = __sg_next(iter); 1435 offset = 0; 1436 } while (1); 1437 1438err_sg_alloc: 1439 kfree(st); 1440err_st_alloc: 1441 return ERR_PTR(ret); 1442} 1443 1444static int 1445i915_get_ggtt_vma_pages(struct i915_vma *vma) 1446{ 1447 int ret; 1448 1449 /* 1450 * The vma->pages are only valid within the lifespan of the borrowed 1451 * obj->mm.pages. When the obj->mm.pages sg_table is regenerated, so 1452 * must be the vma->pages. A simple rule is that vma->pages must only 1453 * be accessed when the obj->mm.pages are pinned. 1454 */ 1455 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(vma->obj)); 1456 1457 switch (vma->ggtt_view.type) { 1458 default: 1459 GEM_BUG_ON(vma->ggtt_view.type); 1460 fallthrough; 1461 case I915_GGTT_VIEW_NORMAL: 1462 vma->pages = vma->obj->mm.pages; 1463 return 0; 1464 1465 case I915_GGTT_VIEW_ROTATED: 1466 vma->pages = 1467 intel_rotate_pages(&vma->ggtt_view.rotated, vma->obj); 1468 break; 1469 1470 case I915_GGTT_VIEW_REMAPPED: 1471 vma->pages = 1472 intel_remap_pages(&vma->ggtt_view.remapped, vma->obj); 1473 break; 1474 1475 case I915_GGTT_VIEW_PARTIAL: 1476 vma->pages = intel_partial_pages(&vma->ggtt_view, vma->obj); 1477 break; 1478 } 1479 1480 ret = 0; 1481 if (IS_ERR(vma->pages)) { 1482 ret = PTR_ERR(vma->pages); 1483 vma->pages = NULL; 1484 drm_err(&vma->vm->i915->drm, 1485 "Failed to get pages for VMA view type %u (%d)!\n", 1486 vma->ggtt_view.type, ret); 1487 } 1488 return ret; 1489} 1490