1/* SPDX-License-Identifier: MIT */
2/*
3 * Copyright © 2020 Intel Corporation
4 *
5 * Please try to maintain the following order within this file unless it makes
6 * sense to do otherwise. From top to bottom:
7 * 1. typedefs
8 * 2. #defines, and macros
9 * 3. structure definitions
10 * 4. function prototypes
11 *
12 * Within each section, please try to order by generation in ascending order,
13 * from top to bottom (ie. gen6 on the top, gen8 on the bottom).
14 */
15
16#ifndef __INTEL_GTT_H__
17#define __INTEL_GTT_H__
18
19#include <linux/io-mapping.h>
20#include <linux/kref.h>
21#include <linux/mm.h>
22#include <linux/pagevec.h>
23#include <linux/scatterlist.h>
24#include <linux/workqueue.h>
25
26#include <drm/drm_mm.h>
27
28#include "gt/intel_reset.h"
29#include "i915_selftest.h"
30#include "i915_vma_types.h"
31
32#define I915_GFP_ALLOW_FAIL (GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN)
33
34#if IS_ENABLED(CONFIG_DRM_I915_TRACE_GTT)
35#define DBG(...) trace_printk(__VA_ARGS__)
36#else
37#define DBG(...)
38#endif
39
40#define NALLOC 3 /* 1 normal, 1 for concurrent threads, 1 for preallocation */
41
42#define I915_GTT_PAGE_SIZE_4K	BIT_ULL(12)
43#define I915_GTT_PAGE_SIZE_64K	BIT_ULL(16)
44#define I915_GTT_PAGE_SIZE_2M	BIT_ULL(21)
45
46#define I915_GTT_PAGE_SIZE I915_GTT_PAGE_SIZE_4K
47#define I915_GTT_MAX_PAGE_SIZE I915_GTT_PAGE_SIZE_2M
48
49#define I915_GTT_PAGE_MASK -I915_GTT_PAGE_SIZE
50
51#define I915_GTT_MIN_ALIGNMENT I915_GTT_PAGE_SIZE
52
53#define I915_FENCE_REG_NONE -1
54#define I915_MAX_NUM_FENCES 32
55/* 32 fences + sign bit for FENCE_REG_NONE */
56#define I915_MAX_NUM_FENCE_BITS 6
57
58typedef u32 gen6_pte_t;
59typedef u64 gen8_pte_t;
60
61#define ggtt_total_entries(ggtt) ((ggtt)->vm.total >> PAGE_SHIFT)
62
63#define I915_PTES(pte_len)		((unsigned int)(PAGE_SIZE / (pte_len)))
64#define I915_PTE_MASK(pte_len)		(I915_PTES(pte_len) - 1)
65#define I915_PDES			512
66#define I915_PDE_MASK			(I915_PDES - 1)
67
68/* gen6-hsw has bit 11-4 for physical addr bit 39-32 */
69#define GEN6_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0xff0))
70#define GEN6_PTE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
71#define GEN6_PDE_ADDR_ENCODE(addr)	GEN6_GTT_ADDR_ENCODE(addr)
72#define GEN6_PTE_CACHE_LLC		(2 << 1)
73#define GEN6_PTE_UNCACHED		(1 << 1)
74#define GEN6_PTE_VALID			REG_BIT(0)
75
76#define GEN6_PTES			I915_PTES(sizeof(gen6_pte_t))
77#define GEN6_PD_SIZE		        (I915_PDES * PAGE_SIZE)
78#define GEN6_PD_ALIGN			(PAGE_SIZE * 16)
79#define GEN6_PDE_SHIFT			22
80#define GEN6_PDE_VALID			REG_BIT(0)
81#define NUM_PTE(pde_shift)     (1 << (pde_shift - PAGE_SHIFT))
82
83#define GEN7_PTE_CACHE_L3_LLC		(3 << 1)
84
85#define BYT_PTE_SNOOPED_BY_CPU_CACHES	REG_BIT(2)
86#define BYT_PTE_WRITEABLE		REG_BIT(1)
87
88/*
89 * Cacheability Control is a 4-bit value. The low three bits are stored in bits
90 * 3:1 of the PTE, while the fourth bit is stored in bit 11 of the PTE.
91 */
92#define HSW_CACHEABILITY_CONTROL(bits)	((((bits) & 0x7) << 1) | \
93					 (((bits) & 0x8) << (11 - 3)))
94#define HSW_WB_LLC_AGE3			HSW_CACHEABILITY_CONTROL(0x2)
95#define HSW_WB_LLC_AGE0			HSW_CACHEABILITY_CONTROL(0x3)
96#define HSW_WB_ELLC_LLC_AGE3		HSW_CACHEABILITY_CONTROL(0x8)
97#define HSW_WB_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0xb)
98#define HSW_WT_ELLC_LLC_AGE3		HSW_CACHEABILITY_CONTROL(0x7)
99#define HSW_WT_ELLC_LLC_AGE0		HSW_CACHEABILITY_CONTROL(0x6)
100#define HSW_PTE_UNCACHED		(0)
101#define HSW_GTT_ADDR_ENCODE(addr)	((addr) | (((addr) >> 28) & 0x7f0))
102#define HSW_PTE_ADDR_ENCODE(addr)	HSW_GTT_ADDR_ENCODE(addr)
103
104/*
105 * GEN8 32b style address is defined as a 3 level page table:
106 * 31:30 | 29:21 | 20:12 |  11:0
107 * PDPE  |  PDE  |  PTE  | offset
108 * The difference as compared to normal x86 3 level page table is the PDPEs are
109 * programmed via register.
110 *
111 * GEN8 48b style address is defined as a 4 level page table:
112 * 47:39 | 38:30 | 29:21 | 20:12 |  11:0
113 * PML4E | PDPE  |  PDE  |  PTE  | offset
114 */
115#define GEN8_3LVL_PDPES			4
116
117#define PPAT_UNCACHED			(_PAGE_PWT | _PAGE_PCD)
118#define PPAT_CACHED_PDE			0 /* WB LLC */
119#define PPAT_CACHED			_PAGE_PAT /* WB LLCeLLC */
120#define PPAT_DISPLAY_ELLC		_PAGE_PCD /* WT eLLC */
121
122#define CHV_PPAT_SNOOP			REG_BIT(6)
123#define GEN8_PPAT_AGE(x)		((x)<<4)
124#define GEN8_PPAT_LLCeLLC		(3<<2)
125#define GEN8_PPAT_LLCELLC		(2<<2)
126#define GEN8_PPAT_LLC			(1<<2)
127#define GEN8_PPAT_WB			(3<<0)
128#define GEN8_PPAT_WT			(2<<0)
129#define GEN8_PPAT_WC			(1<<0)
130#define GEN8_PPAT_UC			(0<<0)
131#define GEN8_PPAT_ELLC_OVERRIDE		(0<<2)
132#define GEN8_PPAT(i, x)			((u64)(x) << ((i) * 8))
133
134#define GEN8_PDE_IPS_64K BIT(11)
135#define GEN8_PDE_PS_2M   BIT(7)
136
137enum i915_cache_level;
138
139struct drm_i915_file_private;
140struct drm_i915_gem_object;
141struct i915_fence_reg;
142struct i915_vma;
143struct intel_gt;
144
145#define for_each_sgt_daddr(__dp, __iter, __sgt) \
146	__for_each_sgt_daddr(__dp, __iter, __sgt, I915_GTT_PAGE_SIZE)
147
148struct i915_page_table {
149	struct drm_i915_gem_object *base;
150	union {
151		atomic_t used;
152		struct i915_page_table *stash;
153	};
154};
155
156struct i915_page_directory {
157	struct i915_page_table pt;
158	spinlock_t lock;
159	void **entry;
160};
161
162#define __px_choose_expr(x, type, expr, other) \
163	__builtin_choose_expr( \
164	__builtin_types_compatible_p(typeof(x), type) || \
165	__builtin_types_compatible_p(typeof(x), const type), \
166	({ type __x = (type)(x); expr; }), \
167	other)
168
169#define px_base(px) \
170	__px_choose_expr(px, struct drm_i915_gem_object *, __x, \
171	__px_choose_expr(px, struct i915_page_table *, __x->base, \
172	__px_choose_expr(px, struct i915_page_directory *, __x->pt.base, \
173	(void)0)))
174
175struct page *__px_page(struct drm_i915_gem_object *p);
176dma_addr_t __px_dma(struct drm_i915_gem_object *p);
177#define px_dma(px) (__px_dma(px_base(px)))
178
179#define px_pt(px) \
180	__px_choose_expr(px, struct i915_page_table *, __x, \
181	__px_choose_expr(px, struct i915_page_directory *, &__x->pt, \
182	(void)0))
183#define px_used(px) (&px_pt(px)->used)
184
185struct i915_vm_pt_stash {
186	/* preallocated chains of page tables/directories */
187	struct i915_page_table *pt[2];
188};
189
190struct i915_vma_ops {
191	/* Map an object into an address space with the given cache flags. */
192	void (*bind_vma)(struct i915_address_space *vm,
193			 struct i915_vm_pt_stash *stash,
194			 struct i915_vma *vma,
195			 enum i915_cache_level cache_level,
196			 u32 flags);
197	/*
198	 * Unmap an object from an address space. This usually consists of
199	 * setting the valid PTE entries to a reserved scratch page.
200	 */
201	void (*unbind_vma)(struct i915_address_space *vm,
202			   struct i915_vma *vma);
203
204	int (*set_pages)(struct i915_vma *vma);
205	void (*clear_pages)(struct i915_vma *vma);
206};
207
208struct i915_address_space {
209	struct kref ref;
210	struct rcu_work rcu;
211
212	struct drm_mm mm;
213	struct intel_gt *gt;
214	struct drm_i915_private *i915;
215	struct device *dma;
216	/*
217	 * Every address space belongs to a struct file - except for the global
218	 * GTT that is owned by the driver (and so @file is set to NULL). In
219	 * principle, no information should leak from one context to another
220	 * (or between files/processes etc) unless explicitly shared by the
221	 * owner. Tracking the owner is important in order to free up per-file
222	 * objects along with the file, to aide resource tracking, and to
223	 * assign blame.
224	 */
225	struct drm_i915_file_private *file;
226	u64 total;		/* size addr space maps (ex. 2GB for ggtt) */
227	u64 reserved;		/* size addr space reserved */
228
229	unsigned int bind_async_flags;
230
231	/*
232	 * Each active user context has its own address space (in full-ppgtt).
233	 * Since the vm may be shared between multiple contexts, we count how
234	 * many contexts keep us "open". Once open hits zero, we are closed
235	 * and do not allow any new attachments, and proceed to shutdown our
236	 * vma and page directories.
237	 */
238	atomic_t open;
239
240	struct mutex mutex; /* protects vma and our lists */
241#define VM_CLASS_GGTT 0
242#define VM_CLASS_PPGTT 1
243
244	struct drm_i915_gem_object *scratch[4];
245	/**
246	 * List of vma currently bound.
247	 */
248	struct list_head bound_list;
249
250	/* Global GTT */
251	bool is_ggtt:1;
252
253	/* Some systems support read-only mappings for GGTT and/or PPGTT */
254	bool has_read_only:1;
255
256	u8 top;
257	u8 pd_shift;
258	u8 scratch_order;
259
260	struct drm_i915_gem_object *
261		(*alloc_pt_dma)(struct i915_address_space *vm, int sz);
262
263	u64 (*pte_encode)(dma_addr_t addr,
264			  enum i915_cache_level level,
265			  u32 flags); /* Create a valid PTE */
266#define PTE_READ_ONLY	BIT(0)
267
268	void (*allocate_va_range)(struct i915_address_space *vm,
269				  struct i915_vm_pt_stash *stash,
270				  u64 start, u64 length);
271	void (*clear_range)(struct i915_address_space *vm,
272			    u64 start, u64 length);
273	void (*insert_page)(struct i915_address_space *vm,
274			    dma_addr_t addr,
275			    u64 offset,
276			    enum i915_cache_level cache_level,
277			    u32 flags);
278	void (*insert_entries)(struct i915_address_space *vm,
279			       struct i915_vma *vma,
280			       enum i915_cache_level cache_level,
281			       u32 flags);
282	void (*cleanup)(struct i915_address_space *vm);
283
284	struct i915_vma_ops vma_ops;
285
286	I915_SELFTEST_DECLARE(struct fault_attr fault_attr);
287	I915_SELFTEST_DECLARE(bool scrub_64K);
288};
289
290/*
291 * The Graphics Translation Table is the way in which GEN hardware translates a
292 * Graphics Virtual Address into a Physical Address. In addition to the normal
293 * collateral associated with any va->pa translations GEN hardware also has a
294 * portion of the GTT which can be mapped by the CPU and remain both coherent
295 * and correct (in cases like swizzling). That region is referred to as GMADR in
296 * the spec.
297 */
298struct i915_ggtt {
299	struct i915_address_space vm;
300
301	struct io_mapping iomap;	/* Mapping to our CPU mappable region */
302	struct resource gmadr;          /* GMADR resource */
303	resource_size_t mappable_end;	/* End offset that we can CPU map */
304
305	/** "Graphics Stolen Memory" holds the global PTEs */
306	void __iomem *gsm;
307	void (*invalidate)(struct i915_ggtt *ggtt);
308
309	/** PPGTT used for aliasing the PPGTT with the GTT */
310	struct i915_ppgtt *alias;
311
312	bool do_idle_maps;
313
314	int mtrr;
315
316	/** Bit 6 swizzling required for X tiling */
317	u32 bit_6_swizzle_x;
318	/** Bit 6 swizzling required for Y tiling */
319	u32 bit_6_swizzle_y;
320
321	u32 pin_bias;
322
323	unsigned int num_fences;
324	struct i915_fence_reg *fence_regs;
325	struct list_head fence_list;
326
327	/**
328	 * List of all objects in gtt_space, currently mmaped by userspace.
329	 * All objects within this list must also be on bound_list.
330	 */
331	struct list_head userfault_list;
332
333	/* Manual runtime pm autosuspend delay for user GGTT mmaps */
334	struct intel_wakeref_auto userfault_wakeref;
335
336	struct mutex error_mutex;
337	struct drm_mm_node error_capture;
338	struct drm_mm_node uc_fw;
339};
340
341struct i915_ppgtt {
342	struct i915_address_space vm;
343
344	struct i915_page_directory *pd;
345};
346
347#define i915_is_ggtt(vm) ((vm)->is_ggtt)
348
349static inline bool
350i915_vm_is_4lvl(const struct i915_address_space *vm)
351{
352	return (vm->total - 1) >> 32;
353}
354
355static inline bool
356i915_vm_has_scratch_64K(struct i915_address_space *vm)
357{
358	return vm->scratch_order == get_order(I915_GTT_PAGE_SIZE_64K);
359}
360
361static inline bool
362i915_vm_has_cache_coloring(struct i915_address_space *vm)
363{
364	return i915_is_ggtt(vm) && vm->mm.color_adjust;
365}
366
367static inline struct i915_ggtt *
368i915_vm_to_ggtt(struct i915_address_space *vm)
369{
370	BUILD_BUG_ON(offsetof(struct i915_ggtt, vm));
371	GEM_BUG_ON(!i915_is_ggtt(vm));
372	return container_of(vm, struct i915_ggtt, vm);
373}
374
375static inline struct i915_ppgtt *
376i915_vm_to_ppgtt(struct i915_address_space *vm)
377{
378	BUILD_BUG_ON(offsetof(struct i915_ppgtt, vm));
379	GEM_BUG_ON(i915_is_ggtt(vm));
380	return container_of(vm, struct i915_ppgtt, vm);
381}
382
383static inline struct i915_address_space *
384i915_vm_get(struct i915_address_space *vm)
385{
386	kref_get(&vm->ref);
387	return vm;
388}
389
390void i915_vm_release(struct kref *kref);
391
392static inline void i915_vm_put(struct i915_address_space *vm)
393{
394	kref_put(&vm->ref, i915_vm_release);
395}
396
397static inline struct i915_address_space *
398i915_vm_open(struct i915_address_space *vm)
399{
400	GEM_BUG_ON(!atomic_read(&vm->open));
401	atomic_inc(&vm->open);
402	return i915_vm_get(vm);
403}
404
405static inline bool
406i915_vm_tryopen(struct i915_address_space *vm)
407{
408	if (atomic_add_unless(&vm->open, 1, 0))
409		return i915_vm_get(vm);
410
411	return false;
412}
413
414void __i915_vm_close(struct i915_address_space *vm);
415
416static inline void
417i915_vm_close(struct i915_address_space *vm)
418{
419	GEM_BUG_ON(!atomic_read(&vm->open));
420	__i915_vm_close(vm);
421
422	i915_vm_put(vm);
423}
424
425void i915_address_space_init(struct i915_address_space *vm, int subclass);
426void i915_address_space_fini(struct i915_address_space *vm);
427
428static inline u32 i915_pte_index(u64 address, unsigned int pde_shift)
429{
430	const u32 mask = NUM_PTE(pde_shift) - 1;
431
432	return (address >> PAGE_SHIFT) & mask;
433}
434
435/*
436 * Helper to counts the number of PTEs within the given length. This count
437 * does not cross a page table boundary, so the max value would be
438 * GEN6_PTES for GEN6, and GEN8_PTES for GEN8.
439 */
440static inline u32 i915_pte_count(u64 addr, u64 length, unsigned int pde_shift)
441{
442	const u64 mask = ~((1ULL << pde_shift) - 1);
443	u64 end;
444
445	GEM_BUG_ON(length == 0);
446	GEM_BUG_ON(offset_in_page(addr | length));
447
448	end = addr + length;
449
450	if ((addr & mask) != (end & mask))
451		return NUM_PTE(pde_shift) - i915_pte_index(addr, pde_shift);
452
453	return i915_pte_index(end, pde_shift) - i915_pte_index(addr, pde_shift);
454}
455
456static inline u32 i915_pde_index(u64 addr, u32 shift)
457{
458	return (addr >> shift) & I915_PDE_MASK;
459}
460
461static inline struct i915_page_table *
462i915_pt_entry(const struct i915_page_directory * const pd,
463	      const unsigned short n)
464{
465	return pd->entry[n];
466}
467
468static inline struct i915_page_directory *
469i915_pd_entry(const struct i915_page_directory * const pdp,
470	      const unsigned short n)
471{
472	return pdp->entry[n];
473}
474
475static inline dma_addr_t
476i915_page_dir_dma_addr(const struct i915_ppgtt *ppgtt, const unsigned int n)
477{
478	struct i915_page_table *pt = ppgtt->pd->entry[n];
479
480	return __px_dma(pt ? px_base(pt) : ppgtt->vm.scratch[ppgtt->vm.top]);
481}
482
483void ppgtt_init(struct i915_ppgtt *ppgtt, struct intel_gt *gt);
484
485int i915_ggtt_probe_hw(struct drm_i915_private *i915);
486int i915_ggtt_init_hw(struct drm_i915_private *i915);
487int i915_ggtt_enable_hw(struct drm_i915_private *i915);
488void i915_ggtt_enable_guc(struct i915_ggtt *ggtt);
489void i915_ggtt_disable_guc(struct i915_ggtt *ggtt);
490int i915_init_ggtt(struct drm_i915_private *i915);
491void i915_ggtt_driver_release(struct drm_i915_private *i915);
492
493static inline bool i915_ggtt_has_aperture(const struct i915_ggtt *ggtt)
494{
495	return ggtt->mappable_end > 0;
496}
497
498int i915_ppgtt_init_hw(struct intel_gt *gt);
499
500struct i915_ppgtt *i915_ppgtt_create(struct intel_gt *gt);
501
502void i915_ggtt_suspend(struct i915_ggtt *gtt);
503void i915_ggtt_resume(struct i915_ggtt *ggtt);
504
505#define kmap_atomic_px(px) kmap_atomic(__px_page(px_base(px)))
506
507void
508fill_page_dma(struct drm_i915_gem_object *p, const u64 val, unsigned int count);
509
510#define fill_px(px, v) fill_page_dma(px_base(px), (v), PAGE_SIZE / sizeof(u64))
511#define fill32_px(px, v) do {						\
512	u64 v__ = lower_32_bits(v);					\
513	fill_px((px), v__ << 32 | v__);					\
514} while (0)
515
516int setup_scratch_page(struct i915_address_space *vm);
517void free_scratch(struct i915_address_space *vm);
518
519struct drm_i915_gem_object *alloc_pt_dma(struct i915_address_space *vm, int sz);
520struct i915_page_table *alloc_pt(struct i915_address_space *vm);
521struct i915_page_directory *alloc_pd(struct i915_address_space *vm);
522struct i915_page_directory *__alloc_pd(int npde);
523
524int pin_pt_dma(struct i915_address_space *vm, struct drm_i915_gem_object *obj);
525
526void free_px(struct i915_address_space *vm,
527	     struct i915_page_table *pt, int lvl);
528#define free_pt(vm, px) free_px(vm, px, 0)
529#define free_pd(vm, px) free_px(vm, px_pt(px), 1)
530
531void
532__set_pd_entry(struct i915_page_directory * const pd,
533	       const unsigned short idx,
534	       struct i915_page_table *pt,
535	       u64 (*encode)(const dma_addr_t, const enum i915_cache_level));
536
537#define set_pd_entry(pd, idx, to) \
538	__set_pd_entry((pd), (idx), px_pt(to), gen8_pde_encode)
539
540void
541clear_pd_entry(struct i915_page_directory * const pd,
542	       const unsigned short idx,
543	       const struct drm_i915_gem_object * const scratch);
544
545bool
546release_pd_entry(struct i915_page_directory * const pd,
547		 const unsigned short idx,
548		 struct i915_page_table * const pt,
549		 const struct drm_i915_gem_object * const scratch);
550void gen6_ggtt_invalidate(struct i915_ggtt *ggtt);
551
552int ggtt_set_pages(struct i915_vma *vma);
553int ppgtt_set_pages(struct i915_vma *vma);
554void clear_pages(struct i915_vma *vma);
555
556void ppgtt_bind_vma(struct i915_address_space *vm,
557		    struct i915_vm_pt_stash *stash,
558		    struct i915_vma *vma,
559		    enum i915_cache_level cache_level,
560		    u32 flags);
561void ppgtt_unbind_vma(struct i915_address_space *vm,
562		      struct i915_vma *vma);
563
564void gtt_write_workarounds(struct intel_gt *gt);
565
566void setup_private_pat(struct intel_uncore *uncore);
567
568int i915_vm_alloc_pt_stash(struct i915_address_space *vm,
569			   struct i915_vm_pt_stash *stash,
570			   u64 size);
571int i915_vm_pin_pt_stash(struct i915_address_space *vm,
572			 struct i915_vm_pt_stash *stash);
573void i915_vm_free_pt_stash(struct i915_address_space *vm,
574			   struct i915_vm_pt_stash *stash);
575
576static inline struct sgt_dma {
577	struct scatterlist *sg;
578	dma_addr_t dma, max;
579} sgt_dma(struct i915_vma *vma) {
580	struct scatterlist *sg = vma->pages->sgl;
581	dma_addr_t addr = sg_dma_address(sg);
582
583	return (struct sgt_dma){ sg, addr, addr + sg->length };
584}
585
586#endif
587