1// SPDX-License-Identifier: MIT 2/* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6#include <asm/set_memory.h> 7#include <asm/smp.h> 8#include <linux/types.h> 9#include <linux/stop_machine.h> 10 11#include <drm/drm_managed.h> 12#include <drm/i915_drm.h> 13#include <drm/intel-gtt.h> 14 15#include "display/intel_display.h" 16#include "gem/i915_gem_lmem.h" 17 18#include "intel_ggtt_gmch.h" 19#include "intel_gt.h" 20#include "intel_gt_regs.h" 21#include "intel_pci_config.h" 22#include "i915_drv.h" 23#include "i915_pci.h" 24#include "i915_scatterlist.h" 25#include "i915_utils.h" 26#include "i915_vgpu.h" 27 28#include "intel_gtt.h" 29#include "gen8_ppgtt.h" 30 31static void i915_ggtt_color_adjust(const struct drm_mm_node *node, 32 unsigned long color, 33 u64 *start, 34 u64 *end) 35{ 36 if (i915_node_color_differs(node, color)) 37 *start += I915_GTT_PAGE_SIZE; 38 39 /* 40 * Also leave a space between the unallocated reserved node after the 41 * GTT and any objects within the GTT, i.e. we use the color adjustment 42 * to insert a guard page to prevent prefetches crossing over the 43 * GTT boundary. 44 */ 45 node = list_next_entry(node, node_list); 46 if (node->color != color) 47 *end -= I915_GTT_PAGE_SIZE; 48} 49 50static int ggtt_init_hw(struct i915_ggtt *ggtt) 51{ 52 struct drm_i915_private *i915 = ggtt->vm.i915; 53 54 i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); 55 56 ggtt->vm.is_ggtt = true; 57 58 /* Only VLV supports read-only GGTT mappings */ 59 ggtt->vm.has_read_only = IS_VALLEYVIEW(i915); 60 61 if (!HAS_LLC(i915) && !HAS_PPGTT(i915)) 62 ggtt->vm.mm.color_adjust = i915_ggtt_color_adjust; 63 64 if (ggtt->mappable_end) { 65 if (!io_mapping_init_wc(&ggtt->iomap, 66 ggtt->gmadr.start, 67 ggtt->mappable_end)) { 68 ggtt->vm.cleanup(&ggtt->vm); 69 return -EIO; 70 } 71 72 ggtt->mtrr = arch_phys_wc_add(ggtt->gmadr.start, 73 ggtt->mappable_end); 74 } 75 76 intel_ggtt_init_fences(ggtt); 77 78 return 0; 79} 80 81/** 82 * i915_ggtt_init_hw - Initialize GGTT hardware 83 * @i915: i915 device 84 */ 85int i915_ggtt_init_hw(struct drm_i915_private *i915) 86{ 87 int ret; 88 89 /* 90 * Note that we use page colouring to enforce a guard page at the 91 * end of the address space. This is required as the CS may prefetch 92 * beyond the end of the batch buffer, across the page boundary, 93 * and beyond the end of the GTT if we do not provide a guard. 94 */ 95 ret = ggtt_init_hw(to_gt(i915)->ggtt); 96 if (ret) 97 return ret; 98 99 return 0; 100} 101 102/** 103 * i915_ggtt_suspend_vm - Suspend the memory mappings for a GGTT or DPT VM 104 * @vm: The VM to suspend the mappings for 105 * 106 * Suspend the memory mappings for all objects mapped to HW via the GGTT or a 107 * DPT page table. 108 */ 109void i915_ggtt_suspend_vm(struct i915_address_space *vm) 110{ 111 struct i915_vma *vma, *vn; 112 int save_skip_rewrite; 113 114 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt); 115 116retry: 117 i915_gem_drain_freed_objects(vm->i915); 118 119 mutex_lock(&vm->mutex); 120 121 /* 122 * Skip rewriting PTE on VMA unbind. 123 * FIXME: Use an argument to i915_vma_unbind() instead? 124 */ 125 save_skip_rewrite = vm->skip_pte_rewrite; 126 vm->skip_pte_rewrite = true; 127 128 list_for_each_entry_safe(vma, vn, &vm->bound_list, vm_link) { 129 struct drm_i915_gem_object *obj = vma->obj; 130 131 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node)); 132 133 if (i915_vma_is_pinned(vma) || !i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) 134 continue; 135 136 /* unlikely to race when GPU is idle, so no worry about slowpath.. */ 137 if (WARN_ON(!i915_gem_object_trylock(obj, NULL))) { 138 /* 139 * No dead objects should appear here, GPU should be 140 * completely idle, and userspace suspended 141 */ 142 i915_gem_object_get(obj); 143 144 mutex_unlock(&vm->mutex); 145 146 i915_gem_object_lock(obj, NULL); 147 GEM_WARN_ON(i915_vma_unbind(vma)); 148 i915_gem_object_unlock(obj); 149 i915_gem_object_put(obj); 150 151 vm->skip_pte_rewrite = save_skip_rewrite; 152 goto retry; 153 } 154 155 if (!i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND)) { 156 i915_vma_wait_for_bind(vma); 157 158 __i915_vma_evict(vma, false); 159 drm_mm_remove_node(&vma->node); 160 } 161 162 i915_gem_object_unlock(obj); 163 } 164 165 vm->clear_range(vm, 0, vm->total); 166 167 vm->skip_pte_rewrite = save_skip_rewrite; 168 169 mutex_unlock(&vm->mutex); 170} 171 172void i915_ggtt_suspend(struct i915_ggtt *ggtt) 173{ 174 struct intel_gt *gt; 175 176 i915_ggtt_suspend_vm(&ggtt->vm); 177 ggtt->invalidate(ggtt); 178 179 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 180 intel_gt_check_and_clear_faults(gt); 181} 182 183void gen6_ggtt_invalidate(struct i915_ggtt *ggtt) 184{ 185 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 186 187 spin_lock_irq(&uncore->lock); 188 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN); 189 intel_uncore_read_fw(uncore, GFX_FLSH_CNTL_GEN6); 190 spin_unlock_irq(&uncore->lock); 191} 192 193static bool needs_wc_ggtt_mapping(struct drm_i915_private *i915) 194{ 195 /* 196 * On BXT+/ICL+ writes larger than 64 bit to the GTT pagetable range 197 * will be dropped. For WC mappings in general we have 64 byte burst 198 * writes when the WC buffer is flushed, so we can't use it, but have to 199 * resort to an uncached mapping. The WC issue is easily caught by the 200 * readback check when writing GTT PTE entries. 201 */ 202 if (!IS_GEN9_LP(i915) && GRAPHICS_VER(i915) < 11) 203 return true; 204 205 return false; 206} 207 208static void gen8_ggtt_invalidate(struct i915_ggtt *ggtt) 209{ 210 struct intel_uncore *uncore = ggtt->vm.gt->uncore; 211 212 /* 213 * Note that as an uncached mmio write, this will flush the 214 * WCB of the writes into the GGTT before it triggers the invalidate. 215 * 216 * Only perform this when GGTT is mapped as WC, see ggtt_probe_common(). 217 */ 218 if (needs_wc_ggtt_mapping(ggtt->vm.i915)) 219 intel_uncore_write_fw(uncore, GFX_FLSH_CNTL_GEN6, 220 GFX_FLSH_CNTL_EN); 221} 222 223static void guc_ggtt_invalidate(struct i915_ggtt *ggtt) 224{ 225 struct drm_i915_private *i915 = ggtt->vm.i915; 226 227 gen8_ggtt_invalidate(ggtt); 228 229 if (GRAPHICS_VER(i915) >= 12) { 230 struct intel_gt *gt; 231 232 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 233 intel_uncore_write_fw(gt->uncore, 234 GEN12_GUC_TLB_INV_CR, 235 GEN12_GUC_TLB_INV_CR_INVALIDATE); 236 } else { 237 intel_uncore_write_fw(ggtt->vm.gt->uncore, 238 GEN8_GTCR, GEN8_GTCR_INVALIDATE); 239 } 240} 241 242static u64 mtl_ggtt_pte_encode(dma_addr_t addr, 243 unsigned int pat_index, 244 u32 flags) 245{ 246 gen8_pte_t pte = addr | GEN8_PAGE_PRESENT; 247 248 WARN_ON_ONCE(addr & ~GEN12_GGTT_PTE_ADDR_MASK); 249 250 if (flags & PTE_LM) 251 pte |= GEN12_GGTT_PTE_LM; 252 253 if (pat_index & BIT(0)) 254 pte |= MTL_GGTT_PTE_PAT0; 255 256 if (pat_index & BIT(1)) 257 pte |= MTL_GGTT_PTE_PAT1; 258 259 return pte; 260} 261 262u64 gen8_ggtt_pte_encode(dma_addr_t addr, 263 unsigned int pat_index, 264 u32 flags) 265{ 266 gen8_pte_t pte = addr | GEN8_PAGE_PRESENT; 267 268 if (flags & PTE_LM) 269 pte |= GEN12_GGTT_PTE_LM; 270 271 return pte; 272} 273 274static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 275{ 276 writeq(pte, addr); 277} 278 279static void gen8_ggtt_insert_page(struct i915_address_space *vm, 280 dma_addr_t addr, 281 u64 offset, 282 unsigned int pat_index, 283 u32 flags) 284{ 285 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 286 gen8_pte_t __iomem *pte = 287 (gen8_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 288 289 gen8_set_pte(pte, ggtt->vm.pte_encode(addr, pat_index, flags)); 290 291 ggtt->invalidate(ggtt); 292} 293 294static void gen8_ggtt_insert_entries(struct i915_address_space *vm, 295 struct i915_vma_resource *vma_res, 296 unsigned int pat_index, 297 u32 flags) 298{ 299 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 300 const gen8_pte_t pte_encode = ggtt->vm.pte_encode(0, pat_index, flags); 301 gen8_pte_t __iomem *gte; 302 gen8_pte_t __iomem *end; 303 struct sgt_iter iter; 304 dma_addr_t addr; 305 306 /* 307 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 308 * not to allow the user to override access to a read only page. 309 */ 310 311 gte = (gen8_pte_t __iomem *)ggtt->gsm; 312 gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE; 313 end = gte + vma_res->guard / I915_GTT_PAGE_SIZE; 314 while (gte < end) 315 gen8_set_pte(gte++, vm->scratch[0]->encode); 316 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; 317 318 for_each_sgt_daddr(addr, iter, vma_res->bi.pages) 319 gen8_set_pte(gte++, pte_encode | addr); 320 GEM_BUG_ON(gte > end); 321 322 /* Fill the allocated but "unused" space beyond the end of the buffer */ 323 while (gte < end) 324 gen8_set_pte(gte++, vm->scratch[0]->encode); 325 326 /* 327 * We want to flush the TLBs only after we're certain all the PTE 328 * updates have finished. 329 */ 330 ggtt->invalidate(ggtt); 331} 332 333static void gen8_ggtt_clear_range(struct i915_address_space *vm, 334 u64 start, u64 length) 335{ 336 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 337 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 338 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 339 const gen8_pte_t scratch_pte = vm->scratch[0]->encode; 340 gen8_pte_t __iomem *gtt_base = 341 (gen8_pte_t __iomem *)ggtt->gsm + first_entry; 342 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 343 int i; 344 345 if (WARN(num_entries > max_entries, 346 "First entry = %d; Num entries = %d (max=%d)\n", 347 first_entry, num_entries, max_entries)) 348 num_entries = max_entries; 349 350 for (i = 0; i < num_entries; i++) 351 gen8_set_pte(>t_base[i], scratch_pte); 352} 353 354static void gen6_ggtt_insert_page(struct i915_address_space *vm, 355 dma_addr_t addr, 356 u64 offset, 357 unsigned int pat_index, 358 u32 flags) 359{ 360 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 361 gen6_pte_t __iomem *pte = 362 (gen6_pte_t __iomem *)ggtt->gsm + offset / I915_GTT_PAGE_SIZE; 363 364 iowrite32(vm->pte_encode(addr, pat_index, flags), pte); 365 366 ggtt->invalidate(ggtt); 367} 368 369/* 370 * Binds an object into the global gtt with the specified cache level. 371 * The object will be accessible to the GPU via commands whose operands 372 * reference offsets within the global GTT as well as accessible by the GPU 373 * through the GMADR mapped BAR (i915->mm.gtt->gtt). 374 */ 375static void gen6_ggtt_insert_entries(struct i915_address_space *vm, 376 struct i915_vma_resource *vma_res, 377 unsigned int pat_index, 378 u32 flags) 379{ 380 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 381 gen6_pte_t __iomem *gte; 382 gen6_pte_t __iomem *end; 383 struct sgt_iter iter; 384 dma_addr_t addr; 385 386 gte = (gen6_pte_t __iomem *)ggtt->gsm; 387 gte += (vma_res->start - vma_res->guard) / I915_GTT_PAGE_SIZE; 388 389 end = gte + vma_res->guard / I915_GTT_PAGE_SIZE; 390 while (gte < end) 391 iowrite32(vm->scratch[0]->encode, gte++); 392 end += (vma_res->node_size + vma_res->guard) / I915_GTT_PAGE_SIZE; 393 for_each_sgt_daddr(addr, iter, vma_res->bi.pages) 394 iowrite32(vm->pte_encode(addr, pat_index, flags), gte++); 395 GEM_BUG_ON(gte > end); 396 397 /* Fill the allocated but "unused" space beyond the end of the buffer */ 398 while (gte < end) 399 iowrite32(vm->scratch[0]->encode, gte++); 400 401 /* 402 * We want to flush the TLBs only after we're certain all the PTE 403 * updates have finished. 404 */ 405 ggtt->invalidate(ggtt); 406} 407 408static void nop_clear_range(struct i915_address_space *vm, 409 u64 start, u64 length) 410{ 411} 412 413static void bxt_vtd_ggtt_wa(struct i915_address_space *vm) 414{ 415 /* 416 * Make sure the internal GAM fifo has been cleared of all GTT 417 * writes before exiting stop_machine(). This guarantees that 418 * any aperture accesses waiting to start in another process 419 * cannot back up behind the GTT writes causing a hang. 420 * The register can be any arbitrary GAM register. 421 */ 422 intel_uncore_posting_read_fw(vm->gt->uncore, GFX_FLSH_CNTL_GEN6); 423} 424 425struct insert_page { 426 struct i915_address_space *vm; 427 dma_addr_t addr; 428 u64 offset; 429 unsigned int pat_index; 430}; 431 432static int bxt_vtd_ggtt_insert_page__cb(void *_arg) 433{ 434 struct insert_page *arg = _arg; 435 436 gen8_ggtt_insert_page(arg->vm, arg->addr, arg->offset, 437 arg->pat_index, 0); 438 bxt_vtd_ggtt_wa(arg->vm); 439 440 return 0; 441} 442 443static void bxt_vtd_ggtt_insert_page__BKL(struct i915_address_space *vm, 444 dma_addr_t addr, 445 u64 offset, 446 unsigned int pat_index, 447 u32 unused) 448{ 449 struct insert_page arg = { vm, addr, offset, pat_index }; 450 451 stop_machine(bxt_vtd_ggtt_insert_page__cb, &arg, NULL); 452} 453 454struct insert_entries { 455 struct i915_address_space *vm; 456 struct i915_vma_resource *vma_res; 457 unsigned int pat_index; 458 u32 flags; 459}; 460 461static int bxt_vtd_ggtt_insert_entries__cb(void *_arg) 462{ 463 struct insert_entries *arg = _arg; 464 465 gen8_ggtt_insert_entries(arg->vm, arg->vma_res, 466 arg->pat_index, arg->flags); 467 bxt_vtd_ggtt_wa(arg->vm); 468 469 return 0; 470} 471 472static void bxt_vtd_ggtt_insert_entries__BKL(struct i915_address_space *vm, 473 struct i915_vma_resource *vma_res, 474 unsigned int pat_index, 475 u32 flags) 476{ 477 struct insert_entries arg = { vm, vma_res, pat_index, flags }; 478 479 stop_machine(bxt_vtd_ggtt_insert_entries__cb, &arg, NULL); 480} 481 482static void gen6_ggtt_clear_range(struct i915_address_space *vm, 483 u64 start, u64 length) 484{ 485 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 486 unsigned int first_entry = start / I915_GTT_PAGE_SIZE; 487 unsigned int num_entries = length / I915_GTT_PAGE_SIZE; 488 gen6_pte_t scratch_pte, __iomem *gtt_base = 489 (gen6_pte_t __iomem *)ggtt->gsm + first_entry; 490 const int max_entries = ggtt_total_entries(ggtt) - first_entry; 491 int i; 492 493 if (WARN(num_entries > max_entries, 494 "First entry = %d; Num entries = %d (max=%d)\n", 495 first_entry, num_entries, max_entries)) 496 num_entries = max_entries; 497 498 scratch_pte = vm->scratch[0]->encode; 499 for (i = 0; i < num_entries; i++) 500 iowrite32(scratch_pte, >t_base[i]); 501} 502 503void intel_ggtt_bind_vma(struct i915_address_space *vm, 504 struct i915_vm_pt_stash *stash, 505 struct i915_vma_resource *vma_res, 506 unsigned int pat_index, 507 u32 flags) 508{ 509 u32 pte_flags; 510 511 if (vma_res->bound_flags & (~flags & I915_VMA_BIND_MASK)) 512 return; 513 514 vma_res->bound_flags |= flags; 515 516 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 517 pte_flags = 0; 518 if (vma_res->bi.readonly) 519 pte_flags |= PTE_READ_ONLY; 520 if (vma_res->bi.lmem) 521 pte_flags |= PTE_LM; 522 523 vm->insert_entries(vm, vma_res, pat_index, pte_flags); 524 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; 525} 526 527void intel_ggtt_unbind_vma(struct i915_address_space *vm, 528 struct i915_vma_resource *vma_res) 529{ 530 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 531} 532 533/* 534 * Reserve the top of the GuC address space for firmware images. Addresses 535 * beyond GUC_GGTT_TOP in the GuC address space are inaccessible by GuC, 536 * which makes for a suitable range to hold GuC/HuC firmware images if the 537 * size of the GGTT is 4G. However, on a 32-bit platform the size of the GGTT 538 * is limited to 2G, which is less than GUC_GGTT_TOP, but we reserve a chunk 539 * of the same size anyway, which is far more than needed, to keep the logic 540 * in uc_fw_ggtt_offset() simple. 541 */ 542#define GUC_TOP_RESERVE_SIZE (SZ_4G - GUC_GGTT_TOP) 543 544static int ggtt_reserve_guc_top(struct i915_ggtt *ggtt) 545{ 546 u64 offset; 547 int ret; 548 549 if (!intel_uc_uses_guc(&ggtt->vm.gt->uc)) 550 return 0; 551 552 GEM_BUG_ON(ggtt->vm.total <= GUC_TOP_RESERVE_SIZE); 553 offset = ggtt->vm.total - GUC_TOP_RESERVE_SIZE; 554 555 ret = i915_gem_gtt_reserve(&ggtt->vm, NULL, &ggtt->uc_fw, 556 GUC_TOP_RESERVE_SIZE, offset, 557 I915_COLOR_UNEVICTABLE, PIN_NOEVICT); 558 if (ret) 559 drm_dbg(&ggtt->vm.i915->drm, 560 "Failed to reserve top of GGTT for GuC\n"); 561 562 return ret; 563} 564 565static void ggtt_release_guc_top(struct i915_ggtt *ggtt) 566{ 567 if (drm_mm_node_allocated(&ggtt->uc_fw)) 568 drm_mm_remove_node(&ggtt->uc_fw); 569} 570 571static void cleanup_init_ggtt(struct i915_ggtt *ggtt) 572{ 573 ggtt_release_guc_top(ggtt); 574 if (drm_mm_node_allocated(&ggtt->error_capture)) 575 drm_mm_remove_node(&ggtt->error_capture); 576 mutex_destroy(&ggtt->error_mutex); 577} 578 579static int init_ggtt(struct i915_ggtt *ggtt) 580{ 581 /* 582 * Let GEM Manage all of the aperture. 583 * 584 * However, leave one page at the end still bound to the scratch page. 585 * There are a number of places where the hardware apparently prefetches 586 * past the end of the object, and we've seen multiple hangs with the 587 * GPU head pointer stuck in a batchbuffer bound at the last page of the 588 * aperture. One page should be enough to keep any prefetching inside 589 * of the aperture. 590 */ 591 unsigned long hole_start, hole_end; 592 struct drm_mm_node *entry; 593 int ret; 594 595 /* 596 * GuC requires all resources that we're sharing with it to be placed in 597 * non-WOPCM memory. If GuC is not present or not in use we still need a 598 * small bias as ring wraparound at offset 0 sometimes hangs. No idea 599 * why. 600 */ 601 ggtt->pin_bias = max_t(u32, I915_GTT_PAGE_SIZE, 602 intel_wopcm_guc_size(&ggtt->vm.gt->wopcm)); 603 604 ret = intel_vgt_balloon(ggtt); 605 if (ret) 606 return ret; 607 608 mutex_init(&ggtt->error_mutex); 609 if (ggtt->mappable_end) { 610 /* 611 * Reserve a mappable slot for our lockless error capture. 612 * 613 * We strongly prefer taking address 0x0 in order to protect 614 * other critical buffers against accidental overwrites, 615 * as writing to address 0 is a very common mistake. 616 * 617 * Since 0 may already be in use by the system (e.g. the BIOS 618 * framebuffer), we let the reservation fail quietly and hope 619 * 0 remains reserved always. 620 * 621 * If we fail to reserve 0, and then fail to find any space 622 * for an error-capture, remain silent. We can afford not 623 * to reserve an error_capture node as we have fallback 624 * paths, and we trust that 0 will remain reserved. However, 625 * the only likely reason for failure to insert is a driver 626 * bug, which we expect to cause other failures... 627 * 628 * Since CPU can perform speculative reads on error capture 629 * (write-combining allows it) add scratch page after error 630 * capture to avoid DMAR errors. 631 */ 632 ggtt->error_capture.size = 2 * I915_GTT_PAGE_SIZE; 633 ggtt->error_capture.color = I915_COLOR_UNEVICTABLE; 634 if (drm_mm_reserve_node(&ggtt->vm.mm, &ggtt->error_capture)) 635 drm_mm_insert_node_in_range(&ggtt->vm.mm, 636 &ggtt->error_capture, 637 ggtt->error_capture.size, 0, 638 ggtt->error_capture.color, 639 0, ggtt->mappable_end, 640 DRM_MM_INSERT_LOW); 641 } 642 if (drm_mm_node_allocated(&ggtt->error_capture)) { 643 u64 start = ggtt->error_capture.start; 644 u64 size = ggtt->error_capture.size; 645 646 ggtt->vm.scratch_range(&ggtt->vm, start, size); 647 drm_dbg(&ggtt->vm.i915->drm, 648 "Reserved GGTT:[%llx, %llx] for use by error capture\n", 649 start, start + size); 650 } 651 652 /* 653 * The upper portion of the GuC address space has a sizeable hole 654 * (several MB) that is inaccessible by GuC. Reserve this range within 655 * GGTT as it can comfortably hold GuC/HuC firmware images. 656 */ 657 ret = ggtt_reserve_guc_top(ggtt); 658 if (ret) 659 goto err; 660 661 /* Clear any non-preallocated blocks */ 662 drm_mm_for_each_hole(entry, &ggtt->vm.mm, hole_start, hole_end) { 663 drm_dbg(&ggtt->vm.i915->drm, 664 "clearing unused GTT space: [%lx, %lx]\n", 665 hole_start, hole_end); 666 ggtt->vm.clear_range(&ggtt->vm, hole_start, 667 hole_end - hole_start); 668 } 669 670 /* And finally clear the reserved guard page */ 671 ggtt->vm.clear_range(&ggtt->vm, ggtt->vm.total - PAGE_SIZE, PAGE_SIZE); 672 673 return 0; 674 675err: 676 cleanup_init_ggtt(ggtt); 677 return ret; 678} 679 680static void aliasing_gtt_bind_vma(struct i915_address_space *vm, 681 struct i915_vm_pt_stash *stash, 682 struct i915_vma_resource *vma_res, 683 unsigned int pat_index, 684 u32 flags) 685{ 686 u32 pte_flags; 687 688 /* Currently applicable only to VLV */ 689 pte_flags = 0; 690 if (vma_res->bi.readonly) 691 pte_flags |= PTE_READ_ONLY; 692 693 if (flags & I915_VMA_LOCAL_BIND) 694 ppgtt_bind_vma(&i915_vm_to_ggtt(vm)->alias->vm, 695 stash, vma_res, pat_index, flags); 696 697 if (flags & I915_VMA_GLOBAL_BIND) 698 vm->insert_entries(vm, vma_res, pat_index, pte_flags); 699 700 vma_res->bound_flags |= flags; 701} 702 703static void aliasing_gtt_unbind_vma(struct i915_address_space *vm, 704 struct i915_vma_resource *vma_res) 705{ 706 if (vma_res->bound_flags & I915_VMA_GLOBAL_BIND) 707 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 708 709 if (vma_res->bound_flags & I915_VMA_LOCAL_BIND) 710 ppgtt_unbind_vma(&i915_vm_to_ggtt(vm)->alias->vm, vma_res); 711} 712 713static int init_aliasing_ppgtt(struct i915_ggtt *ggtt) 714{ 715 struct i915_vm_pt_stash stash = {}; 716 struct i915_ppgtt *ppgtt; 717 int err; 718 719 ppgtt = i915_ppgtt_create(ggtt->vm.gt, 0); 720 if (IS_ERR(ppgtt)) 721 return PTR_ERR(ppgtt); 722 723 if (GEM_WARN_ON(ppgtt->vm.total < ggtt->vm.total)) { 724 err = -ENODEV; 725 goto err_ppgtt; 726 } 727 728 err = i915_vm_alloc_pt_stash(&ppgtt->vm, &stash, ggtt->vm.total); 729 if (err) 730 goto err_ppgtt; 731 732 i915_gem_object_lock(ppgtt->vm.scratch[0], NULL); 733 err = i915_vm_map_pt_stash(&ppgtt->vm, &stash); 734 i915_gem_object_unlock(ppgtt->vm.scratch[0]); 735 if (err) 736 goto err_stash; 737 738 /* 739 * Note we only pre-allocate as far as the end of the global 740 * GTT. On 48b / 4-level page-tables, the difference is very, 741 * very significant! We have to preallocate as GVT/vgpu does 742 * not like the page directory disappearing. 743 */ 744 ppgtt->vm.allocate_va_range(&ppgtt->vm, &stash, 0, ggtt->vm.total); 745 746 ggtt->alias = ppgtt; 747 ggtt->vm.bind_async_flags |= ppgtt->vm.bind_async_flags; 748 749 GEM_BUG_ON(ggtt->vm.vma_ops.bind_vma != intel_ggtt_bind_vma); 750 ggtt->vm.vma_ops.bind_vma = aliasing_gtt_bind_vma; 751 752 GEM_BUG_ON(ggtt->vm.vma_ops.unbind_vma != intel_ggtt_unbind_vma); 753 ggtt->vm.vma_ops.unbind_vma = aliasing_gtt_unbind_vma; 754 755 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 756 return 0; 757 758err_stash: 759 i915_vm_free_pt_stash(&ppgtt->vm, &stash); 760err_ppgtt: 761 i915_vm_put(&ppgtt->vm); 762 return err; 763} 764 765static void fini_aliasing_ppgtt(struct i915_ggtt *ggtt) 766{ 767 struct i915_ppgtt *ppgtt; 768 769 ppgtt = fetch_and_zero(&ggtt->alias); 770 if (!ppgtt) 771 return; 772 773 i915_vm_put(&ppgtt->vm); 774 775 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 776 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 777} 778 779int i915_init_ggtt(struct drm_i915_private *i915) 780{ 781 int ret; 782 783 ret = init_ggtt(to_gt(i915)->ggtt); 784 if (ret) 785 return ret; 786 787 if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) { 788 ret = init_aliasing_ppgtt(to_gt(i915)->ggtt); 789 if (ret) 790 cleanup_init_ggtt(to_gt(i915)->ggtt); 791 } 792 793 return 0; 794} 795 796static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) 797{ 798 struct i915_vma *vma, *vn; 799 800 flush_workqueue(ggtt->vm.i915->wq); 801 i915_gem_drain_freed_objects(ggtt->vm.i915); 802 803 mutex_lock(&ggtt->vm.mutex); 804 805 ggtt->vm.skip_pte_rewrite = true; 806 807 list_for_each_entry_safe(vma, vn, &ggtt->vm.bound_list, vm_link) { 808 struct drm_i915_gem_object *obj = vma->obj; 809 bool trylock; 810 811 trylock = i915_gem_object_trylock(obj, NULL); 812 WARN_ON(!trylock); 813 814 WARN_ON(__i915_vma_unbind(vma)); 815 if (trylock) 816 i915_gem_object_unlock(obj); 817 } 818 819 if (drm_mm_node_allocated(&ggtt->error_capture)) 820 drm_mm_remove_node(&ggtt->error_capture); 821 mutex_destroy(&ggtt->error_mutex); 822 823 ggtt_release_guc_top(ggtt); 824 intel_vgt_deballoon(ggtt); 825 826 ggtt->vm.cleanup(&ggtt->vm); 827 828 mutex_unlock(&ggtt->vm.mutex); 829 i915_address_space_fini(&ggtt->vm); 830 831 arch_phys_wc_del(ggtt->mtrr); 832 833 if (ggtt->iomap.size) 834 io_mapping_fini(&ggtt->iomap); 835} 836 837/** 838 * i915_ggtt_driver_release - Clean up GGTT hardware initialization 839 * @i915: i915 device 840 */ 841void i915_ggtt_driver_release(struct drm_i915_private *i915) 842{ 843 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 844 845 fini_aliasing_ppgtt(ggtt); 846 847 intel_ggtt_fini_fences(ggtt); 848 ggtt_cleanup_hw(ggtt); 849} 850 851/** 852 * i915_ggtt_driver_late_release - Cleanup of GGTT that needs to be done after 853 * all free objects have been drained. 854 * @i915: i915 device 855 */ 856void i915_ggtt_driver_late_release(struct drm_i915_private *i915) 857{ 858 struct i915_ggtt *ggtt = to_gt(i915)->ggtt; 859 860 GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1); 861 dma_resv_fini(&ggtt->vm._resv); 862} 863 864static unsigned int gen6_get_total_gtt_size(u16 snb_gmch_ctl) 865{ 866 snb_gmch_ctl >>= SNB_GMCH_GGMS_SHIFT; 867 snb_gmch_ctl &= SNB_GMCH_GGMS_MASK; 868 return snb_gmch_ctl << 20; 869} 870 871static unsigned int gen8_get_total_gtt_size(u16 bdw_gmch_ctl) 872{ 873 bdw_gmch_ctl >>= BDW_GMCH_GGMS_SHIFT; 874 bdw_gmch_ctl &= BDW_GMCH_GGMS_MASK; 875 if (bdw_gmch_ctl) 876 bdw_gmch_ctl = 1 << bdw_gmch_ctl; 877 878#ifdef CONFIG_X86_32 879 /* Limit 32b platforms to a 2GB GGTT: 4 << 20 / pte size * I915_GTT_PAGE_SIZE */ 880 if (bdw_gmch_ctl > 4) 881 bdw_gmch_ctl = 4; 882#endif 883 884 return bdw_gmch_ctl << 20; 885} 886 887static unsigned int chv_get_total_gtt_size(u16 gmch_ctrl) 888{ 889 gmch_ctrl >>= SNB_GMCH_GGMS_SHIFT; 890 gmch_ctrl &= SNB_GMCH_GGMS_MASK; 891 892 if (gmch_ctrl) 893 return 1 << (20 + gmch_ctrl); 894 895 return 0; 896} 897 898static unsigned int gen6_gttmmadr_size(struct drm_i915_private *i915) 899{ 900 /* 901 * GEN6: GTTMMADR size is 4MB and GTTADR starts at 2MB offset 902 * GEN8: GTTMMADR size is 16MB and GTTADR starts at 8MB offset 903 */ 904 GEM_BUG_ON(GRAPHICS_VER(i915) < 6); 905 return (GRAPHICS_VER(i915) < 8) ? SZ_4M : SZ_16M; 906} 907 908static unsigned int gen6_gttadr_offset(struct drm_i915_private *i915) 909{ 910 return gen6_gttmmadr_size(i915) / 2; 911} 912 913static int ggtt_probe_common(struct i915_ggtt *ggtt, u64 size) 914{ 915 struct drm_i915_private *i915 = ggtt->vm.i915; 916 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 917 phys_addr_t phys_addr; 918 u32 pte_flags; 919 int ret; 920 921 GEM_WARN_ON(pci_resource_len(pdev, GEN4_GTTMMADR_BAR) != gen6_gttmmadr_size(i915)); 922 phys_addr = pci_resource_start(pdev, GEN4_GTTMMADR_BAR) + gen6_gttadr_offset(i915); 923 924 if (needs_wc_ggtt_mapping(i915)) 925 ggtt->gsm = ioremap_wc(phys_addr, size); 926 else 927 ggtt->gsm = ioremap(phys_addr, size); 928 929 if (!ggtt->gsm) { 930 drm_err(&i915->drm, "Failed to map the ggtt page table\n"); 931 return -ENOMEM; 932 } 933 934 kref_init(&ggtt->vm.resv_ref); 935 ret = setup_scratch_page(&ggtt->vm); 936 if (ret) { 937 drm_err(&i915->drm, "Scratch setup failed\n"); 938 /* iounmap will also get called at remove, but meh */ 939 iounmap(ggtt->gsm); 940 return ret; 941 } 942 943 pte_flags = 0; 944 if (i915_gem_object_is_lmem(ggtt->vm.scratch[0])) 945 pte_flags |= PTE_LM; 946 947 ggtt->vm.scratch[0]->encode = 948 ggtt->vm.pte_encode(px_dma(ggtt->vm.scratch[0]), 949 i915_gem_get_pat_index(i915, 950 I915_CACHE_NONE), 951 pte_flags); 952 953 return 0; 954} 955 956static void gen6_gmch_remove(struct i915_address_space *vm) 957{ 958 struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm); 959 960 iounmap(ggtt->gsm); 961 free_scratch(vm); 962} 963 964static struct resource pci_resource(struct pci_dev *pdev, int bar) 965{ 966 return DEFINE_RES_MEM(pci_resource_start(pdev, bar), 967 pci_resource_len(pdev, bar)); 968} 969 970static int gen8_gmch_probe(struct i915_ggtt *ggtt) 971{ 972 struct drm_i915_private *i915 = ggtt->vm.i915; 973 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 974 unsigned int size; 975 u16 snb_gmch_ctl; 976 977 if (!HAS_LMEM(i915) && !HAS_LMEMBAR_SMEM_STOLEN(i915)) { 978 if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR)) 979 return -ENXIO; 980 981 ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR); 982 ggtt->mappable_end = resource_size(&ggtt->gmadr); 983 } 984 985 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 986 if (IS_CHERRYVIEW(i915)) 987 size = chv_get_total_gtt_size(snb_gmch_ctl); 988 else 989 size = gen8_get_total_gtt_size(snb_gmch_ctl); 990 991 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 992 ggtt->vm.alloc_scratch_dma = alloc_pt_dma; 993 ggtt->vm.lmem_pt_obj_flags = I915_BO_ALLOC_PM_EARLY; 994 995 ggtt->vm.total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 996 ggtt->vm.cleanup = gen6_gmch_remove; 997 ggtt->vm.insert_page = gen8_ggtt_insert_page; 998 ggtt->vm.clear_range = nop_clear_range; 999 ggtt->vm.scratch_range = gen8_ggtt_clear_range; 1000 1001 ggtt->vm.insert_entries = gen8_ggtt_insert_entries; 1002 1003 /* 1004 * Serialize GTT updates with aperture access on BXT if VT-d is on, 1005 * and always on CHV. 1006 */ 1007 if (intel_vm_no_concurrent_access_wa(i915)) { 1008 ggtt->vm.insert_entries = bxt_vtd_ggtt_insert_entries__BKL; 1009 ggtt->vm.insert_page = bxt_vtd_ggtt_insert_page__BKL; 1010 1011 /* 1012 * Calling stop_machine() version of GGTT update function 1013 * at error capture/reset path will raise lockdep warning. 1014 * Allow calling gen8_ggtt_insert_* directly at reset path 1015 * which is safe from parallel GGTT updates. 1016 */ 1017 ggtt->vm.raw_insert_page = gen8_ggtt_insert_page; 1018 ggtt->vm.raw_insert_entries = gen8_ggtt_insert_entries; 1019 1020 ggtt->vm.bind_async_flags = 1021 I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 1022 } 1023 1024 if (intel_uc_wants_guc(&ggtt->vm.gt->uc)) 1025 ggtt->invalidate = guc_ggtt_invalidate; 1026 else 1027 ggtt->invalidate = gen8_ggtt_invalidate; 1028 1029 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 1030 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 1031 1032 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 70)) 1033 ggtt->vm.pte_encode = mtl_ggtt_pte_encode; 1034 else 1035 ggtt->vm.pte_encode = gen8_ggtt_pte_encode; 1036 1037 return ggtt_probe_common(ggtt, size); 1038} 1039 1040/* 1041 * For pre-gen8 platforms pat_index is the same as enum i915_cache_level, 1042 * so the switch-case statements in these PTE encode functions are still valid. 1043 * See translation table LEGACY_CACHELEVEL. 1044 */ 1045static u64 snb_pte_encode(dma_addr_t addr, 1046 unsigned int pat_index, 1047 u32 flags) 1048{ 1049 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1050 1051 switch (pat_index) { 1052 case I915_CACHE_L3_LLC: 1053 case I915_CACHE_LLC: 1054 pte |= GEN6_PTE_CACHE_LLC; 1055 break; 1056 case I915_CACHE_NONE: 1057 pte |= GEN6_PTE_UNCACHED; 1058 break; 1059 default: 1060 MISSING_CASE(pat_index); 1061 } 1062 1063 return pte; 1064} 1065 1066static u64 ivb_pte_encode(dma_addr_t addr, 1067 unsigned int pat_index, 1068 u32 flags) 1069{ 1070 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1071 1072 switch (pat_index) { 1073 case I915_CACHE_L3_LLC: 1074 pte |= GEN7_PTE_CACHE_L3_LLC; 1075 break; 1076 case I915_CACHE_LLC: 1077 pte |= GEN6_PTE_CACHE_LLC; 1078 break; 1079 case I915_CACHE_NONE: 1080 pte |= GEN6_PTE_UNCACHED; 1081 break; 1082 default: 1083 MISSING_CASE(pat_index); 1084 } 1085 1086 return pte; 1087} 1088 1089static u64 byt_pte_encode(dma_addr_t addr, 1090 unsigned int pat_index, 1091 u32 flags) 1092{ 1093 gen6_pte_t pte = GEN6_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1094 1095 if (!(flags & PTE_READ_ONLY)) 1096 pte |= BYT_PTE_WRITEABLE; 1097 1098 if (pat_index != I915_CACHE_NONE) 1099 pte |= BYT_PTE_SNOOPED_BY_CPU_CACHES; 1100 1101 return pte; 1102} 1103 1104static u64 hsw_pte_encode(dma_addr_t addr, 1105 unsigned int pat_index, 1106 u32 flags) 1107{ 1108 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1109 1110 if (pat_index != I915_CACHE_NONE) 1111 pte |= HSW_WB_LLC_AGE3; 1112 1113 return pte; 1114} 1115 1116static u64 iris_pte_encode(dma_addr_t addr, 1117 unsigned int pat_index, 1118 u32 flags) 1119{ 1120 gen6_pte_t pte = HSW_PTE_ADDR_ENCODE(addr) | GEN6_PTE_VALID; 1121 1122 switch (pat_index) { 1123 case I915_CACHE_NONE: 1124 break; 1125 case I915_CACHE_WT: 1126 pte |= HSW_WT_ELLC_LLC_AGE3; 1127 break; 1128 default: 1129 pte |= HSW_WB_ELLC_LLC_AGE3; 1130 break; 1131 } 1132 1133 return pte; 1134} 1135 1136static int gen6_gmch_probe(struct i915_ggtt *ggtt) 1137{ 1138 struct drm_i915_private *i915 = ggtt->vm.i915; 1139 struct pci_dev *pdev = to_pci_dev(i915->drm.dev); 1140 unsigned int size; 1141 u16 snb_gmch_ctl; 1142 1143 if (!i915_pci_resource_valid(pdev, GEN4_GMADR_BAR)) 1144 return -ENXIO; 1145 1146 ggtt->gmadr = pci_resource(pdev, GEN4_GMADR_BAR); 1147 ggtt->mappable_end = resource_size(&ggtt->gmadr); 1148 1149 /* 1150 * 64/512MB is the current min/max we actually know of, but this is 1151 * just a coarse sanity check. 1152 */ 1153 if (ggtt->mappable_end < (64 << 20) || 1154 ggtt->mappable_end > (512 << 20)) { 1155 drm_err(&i915->drm, "Unknown GMADR size (%pa)\n", 1156 &ggtt->mappable_end); 1157 return -ENXIO; 1158 } 1159 1160 pci_read_config_word(pdev, SNB_GMCH_CTRL, &snb_gmch_ctl); 1161 1162 size = gen6_get_total_gtt_size(snb_gmch_ctl); 1163 ggtt->vm.total = (size / sizeof(gen6_pte_t)) * I915_GTT_PAGE_SIZE; 1164 1165 ggtt->vm.alloc_pt_dma = alloc_pt_dma; 1166 ggtt->vm.alloc_scratch_dma = alloc_pt_dma; 1167 1168 ggtt->vm.clear_range = nop_clear_range; 1169 if (!HAS_FULL_PPGTT(i915)) 1170 ggtt->vm.clear_range = gen6_ggtt_clear_range; 1171 ggtt->vm.scratch_range = gen6_ggtt_clear_range; 1172 ggtt->vm.insert_page = gen6_ggtt_insert_page; 1173 ggtt->vm.insert_entries = gen6_ggtt_insert_entries; 1174 ggtt->vm.cleanup = gen6_gmch_remove; 1175 1176 ggtt->invalidate = gen6_ggtt_invalidate; 1177 1178 if (HAS_EDRAM(i915)) 1179 ggtt->vm.pte_encode = iris_pte_encode; 1180 else if (IS_HASWELL(i915)) 1181 ggtt->vm.pte_encode = hsw_pte_encode; 1182 else if (IS_VALLEYVIEW(i915)) 1183 ggtt->vm.pte_encode = byt_pte_encode; 1184 else if (GRAPHICS_VER(i915) >= 7) 1185 ggtt->vm.pte_encode = ivb_pte_encode; 1186 else 1187 ggtt->vm.pte_encode = snb_pte_encode; 1188 1189 ggtt->vm.vma_ops.bind_vma = intel_ggtt_bind_vma; 1190 ggtt->vm.vma_ops.unbind_vma = intel_ggtt_unbind_vma; 1191 1192 return ggtt_probe_common(ggtt, size); 1193} 1194 1195static int ggtt_probe_hw(struct i915_ggtt *ggtt, struct intel_gt *gt) 1196{ 1197 struct drm_i915_private *i915 = gt->i915; 1198 int ret; 1199 1200 ggtt->vm.gt = gt; 1201 ggtt->vm.i915 = i915; 1202 ggtt->vm.dma = i915->drm.dev; 1203 dma_resv_init(&ggtt->vm._resv); 1204 1205 if (GRAPHICS_VER(i915) >= 8) 1206 ret = gen8_gmch_probe(ggtt); 1207 else if (GRAPHICS_VER(i915) >= 6) 1208 ret = gen6_gmch_probe(ggtt); 1209 else 1210 ret = intel_ggtt_gmch_probe(ggtt); 1211 1212 if (ret) { 1213 dma_resv_fini(&ggtt->vm._resv); 1214 return ret; 1215 } 1216 1217 if ((ggtt->vm.total - 1) >> 32) { 1218 drm_err(&i915->drm, 1219 "We never expected a Global GTT with more than 32bits" 1220 " of address space! Found %lldM!\n", 1221 ggtt->vm.total >> 20); 1222 ggtt->vm.total = 1ULL << 32; 1223 ggtt->mappable_end = 1224 min_t(u64, ggtt->mappable_end, ggtt->vm.total); 1225 } 1226 1227 if (ggtt->mappable_end > ggtt->vm.total) { 1228 drm_err(&i915->drm, 1229 "mappable aperture extends past end of GGTT," 1230 " aperture=%pa, total=%llx\n", 1231 &ggtt->mappable_end, ggtt->vm.total); 1232 ggtt->mappable_end = ggtt->vm.total; 1233 } 1234 1235 /* GMADR is the PCI mmio aperture into the global GTT. */ 1236 drm_dbg(&i915->drm, "GGTT size = %lluM\n", ggtt->vm.total >> 20); 1237 drm_dbg(&i915->drm, "GMADR size = %lluM\n", 1238 (u64)ggtt->mappable_end >> 20); 1239 drm_dbg(&i915->drm, "DSM size = %lluM\n", 1240 (u64)resource_size(&intel_graphics_stolen_res) >> 20); 1241 1242 return 0; 1243} 1244 1245/** 1246 * i915_ggtt_probe_hw - Probe GGTT hardware location 1247 * @i915: i915 device 1248 */ 1249int i915_ggtt_probe_hw(struct drm_i915_private *i915) 1250{ 1251 struct intel_gt *gt; 1252 int ret, i; 1253 1254 for_each_gt(gt, i915, i) { 1255 ret = intel_gt_assign_ggtt(gt); 1256 if (ret) 1257 return ret; 1258 } 1259 1260 ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915)); 1261 if (ret) 1262 return ret; 1263 1264 if (i915_vtd_active(i915)) 1265 drm_info(&i915->drm, "VT-d active for gfx access\n"); 1266 1267 return 0; 1268} 1269 1270struct i915_ggtt *i915_ggtt_create(struct drm_i915_private *i915) 1271{ 1272 struct i915_ggtt *ggtt; 1273 1274 ggtt = drmm_kzalloc(&i915->drm, sizeof(*ggtt), GFP_KERNEL); 1275 if (!ggtt) 1276 return ERR_PTR(-ENOMEM); 1277 1278 INIT_LIST_HEAD(&ggtt->gt_list); 1279 1280 return ggtt; 1281} 1282 1283int i915_ggtt_enable_hw(struct drm_i915_private *i915) 1284{ 1285 if (GRAPHICS_VER(i915) < 6) 1286 return intel_ggtt_gmch_enable_hw(i915); 1287 1288 return 0; 1289} 1290 1291/** 1292 * i915_ggtt_resume_vm - Restore the memory mappings for a GGTT or DPT VM 1293 * @vm: The VM to restore the mappings for 1294 * 1295 * Restore the memory mappings for all objects mapped to HW via the GGTT or a 1296 * DPT page table. 1297 * 1298 * Returns %true if restoring the mapping for any object that was in a write 1299 * domain before suspend. 1300 */ 1301bool i915_ggtt_resume_vm(struct i915_address_space *vm) 1302{ 1303 struct i915_vma *vma; 1304 bool write_domain_objs = false; 1305 1306 drm_WARN_ON(&vm->i915->drm, !vm->is_ggtt && !vm->is_dpt); 1307 1308 /* First fill our portion of the GTT with scratch pages */ 1309 vm->clear_range(vm, 0, vm->total); 1310 1311 /* clflush objects bound into the GGTT and rebind them. */ 1312 list_for_each_entry(vma, &vm->bound_list, vm_link) { 1313 struct drm_i915_gem_object *obj = vma->obj; 1314 unsigned int was_bound = 1315 atomic_read(&vma->flags) & I915_VMA_BIND_MASK; 1316 1317 GEM_BUG_ON(!was_bound); 1318 1319 /* 1320 * Clear the bound flags of the vma resource to allow 1321 * ptes to be repopulated. 1322 */ 1323 vma->resource->bound_flags = 0; 1324 vma->ops->bind_vma(vm, NULL, vma->resource, 1325 obj ? obj->pat_index : 1326 i915_gem_get_pat_index(vm->i915, 1327 I915_CACHE_NONE), 1328 was_bound); 1329 1330 if (obj) { /* only used during resume => exclusive access */ 1331 write_domain_objs |= fetch_and_zero(&obj->write_domain); 1332 obj->read_domains |= I915_GEM_DOMAIN_GTT; 1333 } 1334 } 1335 1336 return write_domain_objs; 1337} 1338 1339void i915_ggtt_resume(struct i915_ggtt *ggtt) 1340{ 1341 struct intel_gt *gt; 1342 bool flush; 1343 1344 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 1345 intel_gt_check_and_clear_faults(gt); 1346 1347 flush = i915_ggtt_resume_vm(&ggtt->vm); 1348 1349 if (drm_mm_node_allocated(&ggtt->error_capture)) 1350 ggtt->vm.scratch_range(&ggtt->vm, ggtt->error_capture.start, 1351 ggtt->error_capture.size); 1352 1353 list_for_each_entry(gt, &ggtt->gt_list, ggtt_link) 1354 intel_uc_resume_mappings(>->uc); 1355 1356 ggtt->invalidate(ggtt); 1357 1358 if (flush) 1359 wbinvd_on_all_cpus(); 1360 1361 intel_ggtt_restore_fences(ggtt); 1362} 1363