| /kernel/linux/linux-6.6/drivers/accel/ivpu/ |
| H A D | ivpu_mmu_context.c | 42 static int ivpu_mmu_pgtable_init(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) in ivpu_mmu_pgtable_init() argument
46 pgtable->pgd_dma_ptr = dma_alloc_coherent(vdev->drm.dev, IVPU_MMU_PGTABLE_SIZE, &pgd_dma, in ivpu_mmu_pgtable_init()
48 if (!pgtable->pgd_dma_ptr) in ivpu_mmu_pgtable_init()
51 pgtable->pgd_dma = pgd_dma; in ivpu_mmu_pgtable_init()
63 static void ivpu_mmu_pgtables_free(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable) in ivpu_mmu_pgtables_free() argument
70 pud_dma_ptr = pgtable->pud_ptrs[pgd_idx]; in ivpu_mmu_pgtables_free()
71 pud_dma = pgtable->pgd_dma_ptr[pgd_idx]; in ivpu_mmu_pgtables_free()
77 pmd_dma_ptr = pgtable->pmd_ptrs[pgd_idx][pud_idx]; in ivpu_mmu_pgtables_free()
78 pmd_dma = pgtable->pud_ptrs[pgd_idx][pud_idx]; in ivpu_mmu_pgtables_free()
84 pte_dma_ptr = pgtable in ivpu_mmu_pgtables_free()
103 ivpu_mmu_ensure_pud(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx) ivpu_mmu_ensure_pud() argument
139 ivpu_mmu_ensure_pmd(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx, int pud_idx) ivpu_mmu_ensure_pmd() argument
168 ivpu_mmu_ensure_pte(struct ivpu_device *vdev, struct ivpu_mmu_pgtable *pgtable, int pgd_idx, int pud_idx, int pmd_idx) ivpu_mmu_ensure_pte() argument
255 struct ivpu_mmu_pgtable *pgtable = &ctx->pgtable; ivpu_mmu_context_flush_page_tables() local
[all...] |
| /kernel/linux/linux-6.6/drivers/iommu/amd/ |
| H A D | io_pgtable_v2.c | 14 #include <linux/io-pgtable.h>
198 static u64 *fetch_pte(struct amd_io_pgtable *pgtable, in fetch_pte() argument
205 pte = &pgtable->pgd[PM_LEVEL_INDEX(level, iova)]; in fetch_pte()
293 struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); in iommu_v2_unmap_pages() local
294 struct io_pgtable_cfg *cfg = &pgtable->iop.cfg; in iommu_v2_unmap_pages()
304 pte = fetch_pte(pgtable, iova, &unmap_size); in iommu_v2_unmap_pages()
319 struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); in iommu_v2_iova_to_phys() local
323 pte = fetch_pte(pgtable, iova, &pte_pgsize); in iommu_v2_iova_to_phys()
360 struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); in v2_free_pgtable() local
362 pdom = container_of(pgtable, struc in v2_free_pgtable()
378 struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); v2_alloc_pgtable() local
[all...] |
| H A D | io_pgtable.c | 14 #include <linux/io-pgtable.h>
139 /* lowest 3 bits encode pgtable mode */ in amd_iommu_domain_set_pgtable()
290 static u64 *fetch_pte(struct amd_io_pgtable *pgtable, in fetch_pte() argument
299 if (address > PM_LEVEL_SIZE(pgtable->mode)) in fetch_pte()
302 level = pgtable->mode - 1; in fetch_pte()
303 pte = &pgtable->root[PM_LEVEL_INDEX(level, address)]; in fetch_pte()
443 struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); in iommu_v1_unmap_pages() local
454 pte = fetch_pte(pgtable, iova, &unmap_size); in iommu_v1_unmap_pages()
474 struct amd_io_pgtable *pgtable = io_pgtable_ops_to_data(ops); in iommu_v1_iova_to_phys() local
478 pte = fetch_pte(pgtable, iov in iommu_v1_iova_to_phys()
494 struct amd_io_pgtable *pgtable = container_of(iop, struct amd_io_pgtable, iop); v1_free_pgtable() local
520 struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); v1_alloc_pgtable() local
[all...] |
| /kernel/linux/linux-5.10/mm/ |
| H A D | pgtable-generic.c | 3 * mm/pgtable-generic.c
5 * Generic pgtable methods declared in linux/pgtable.h
12 #include <linux/pgtable.h>
162 pgtable_t pgtable) in pgtable_trans_huge_deposit()
168 INIT_LIST_HEAD(&pgtable->lru); in pgtable_trans_huge_deposit()
170 list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); in pgtable_trans_huge_deposit()
171 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
179 pgtable_t pgtable; in pgtable_trans_huge_withdraw() local
184 pgtable in pgtable_trans_huge_withdraw()
161 pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) pgtable_trans_huge_deposit() argument
[all...] |
| H A D | huge_memory.c | 585 pgtable_t pgtable; in __do_huge_pmd_anonymous_page() local
599 pgtable = pte_alloc_one(vma->vm_mm); in __do_huge_pmd_anonymous_page()
600 if (unlikely(!pgtable)) { in __do_huge_pmd_anonymous_page()
629 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
639 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
652 if (pgtable) in __do_huge_pmd_anonymous_page()
653 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
695 static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, in set_huge_zero_page() argument
704 if (pgtable) in set_huge_zero_page()
705 pgtable_trans_huge_deposit(mm, pmd, pgtable); in set_huge_zero_page()
727 pgtable_t pgtable; do_huge_pmd_anonymous_page() local
772 insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, pgtable_t pgtable) insert_pfn_pmd() argument
837 pgtable_t pgtable = NULL; vmf_insert_pfn_pmd_prot() local
1020 pgtable_t pgtable = NULL; copy_huge_pmd() local
1643 pgtable_t pgtable; zap_deposited_table() local
1773 pgtable_t pgtable; move_huge_pmd() local
1996 pgtable_t pgtable; __split_huge_zero_page_pmd() local
2033 pgtable_t pgtable; __split_huge_pmd_locked() local
[all...] |
| /kernel/linux/linux-6.6/mm/ |
| H A D | pgtable-generic.c | 3 * mm/pgtable-generic.c
5 * Generic pgtable methods declared in linux/pgtable.h
12 #include <linux/pgtable.h>
166 pgtable_t pgtable) in pgtable_trans_huge_deposit()
172 INIT_LIST_HEAD(&pgtable->lru); in pgtable_trans_huge_deposit()
174 list_add(&pgtable->lru, &pmd_huge_pte(mm, pmdp)->lru); in pgtable_trans_huge_deposit()
175 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
183 pgtable_t pgtable; in pgtable_trans_huge_withdraw() local
188 pgtable in pgtable_trans_huge_withdraw()
165 pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) pgtable_trans_huge_deposit() argument
245 pte_free_defer(struct mm_struct *mm, pgtable_t pgtable) pte_free_defer() argument
[all...] |
| H A D | huge_memory.c | 651 pgtable_t pgtable; in __do_huge_pmd_anonymous_page() local
665 pgtable = pte_alloc_one(vma->vm_mm); in __do_huge_pmd_anonymous_page()
666 if (unlikely(!pgtable)) { in __do_huge_pmd_anonymous_page()
693 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
703 pgtable_trans_huge_deposit(vma->vm_mm, vmf->pmd, pgtable); in __do_huge_pmd_anonymous_page()
717 if (pgtable) in __do_huge_pmd_anonymous_page()
718 pte_free(vma->vm_mm, pgtable); in __do_huge_pmd_anonymous_page()
760 static void set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm, in set_huge_zero_page() argument
769 pgtable_trans_huge_deposit(mm, pmd, pgtable); in set_huge_zero_page()
790 pgtable_t pgtable; in do_huge_pmd_anonymous_page() local
835 insert_pfn_pmd(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmd, pfn_t pfn, pgprot_t prot, bool write, pgtable_t pgtable) insert_pfn_pmd() argument
897 pgtable_t pgtable = NULL; vmf_insert_pfn_pmd() local
1069 pgtable_t pgtable = NULL; copy_huge_pmd() local
1652 pgtable_t pgtable; zap_deposited_table() local
1781 pgtable_t pgtable; move_huge_pmd() local
2029 pgtable_t pgtable; __split_huge_zero_page_pmd() local
2071 pgtable_t pgtable; __split_huge_pmd_locked() local
[all...] |
| /kernel/linux/linux-5.10/drivers/gpu/drm/etnaviv/ |
| H A D | etnaviv_iommu.c | 93 u32 pgtable; in etnaviv_iommuv1_restore() local
107 pgtable = (u32)v1_context->pgtable_dma; in etnaviv_iommuv1_restore()
109 gpu_write(gpu, VIVS_MC_MMU_FE_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
110 gpu_write(gpu, VIVS_MC_MMU_TX_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
111 gpu_write(gpu, VIVS_MC_MMU_PE_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
112 gpu_write(gpu, VIVS_MC_MMU_PEZ_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
113 gpu_write(gpu, VIVS_MC_MMU_RA_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
|
| /kernel/linux/linux-6.6/drivers/gpu/drm/etnaviv/ |
| H A D | etnaviv_iommu.c | 93 u32 pgtable; in etnaviv_iommuv1_restore() local
107 pgtable = (u32)v1_context->pgtable_dma; in etnaviv_iommuv1_restore()
109 gpu_write(gpu, VIVS_MC_MMU_FE_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
110 gpu_write(gpu, VIVS_MC_MMU_TX_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
111 gpu_write(gpu, VIVS_MC_MMU_PE_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
112 gpu_write(gpu, VIVS_MC_MMU_PEZ_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
113 gpu_write(gpu, VIVS_MC_MMU_RA_PAGE_TABLE, pgtable); in etnaviv_iommuv1_restore()
|
| /kernel/linux/linux-5.10/sound/core/ |
| H A D | sgbuf.c | 62 struct page **pgtable; in snd_malloc_sgbuf_pages() local
84 pgtable = kcalloc(sgbuf->tblsize, sizeof(*pgtable), GFP_KERNEL); in snd_malloc_sgbuf_pages()
85 if (!pgtable) in snd_malloc_sgbuf_pages()
87 sgbuf->page_table = pgtable; in snd_malloc_sgbuf_pages()
113 *pgtable++ = virt_to_page(tmpb.area); in snd_malloc_sgbuf_pages()
|
| /kernel/linux/linux-5.10/arch/sparc/mm/ |
| H A D | tlb.c | 264 pgtable_t pgtable) in pgtable_trans_huge_deposit()
266 struct list_head *lh = (struct list_head *) pgtable; in pgtable_trans_huge_deposit()
275 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
281 pgtable_t pgtable; in pgtable_trans_huge_withdraw() local
286 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw()
287 lh = (struct list_head *) pgtable; in pgtable_trans_huge_withdraw()
294 pte_val(pgtable[0]) = 0; in pgtable_trans_huge_withdraw()
295 pte_val(pgtable[1]) = 0; in pgtable_trans_huge_withdraw()
297 return pgtable; in pgtable_trans_huge_withdraw()
263 pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) pgtable_trans_huge_deposit() argument
|
| /kernel/linux/linux-5.10/arch/powerpc/mm/book3s64/ |
| H A D | hash_pgtable.c | 263 * We want to put the pgtable in pmd and use pgtable for tracking
267 pgtable_t pgtable) in hash__pgtable_trans_huge_deposit()
273 * we store the pgtable in the second half of PMD in hash__pgtable_trans_huge_deposit()
276 *pgtable_slot = pgtable; in hash__pgtable_trans_huge_deposit()
278 * expose the deposited pgtable to other cpus. in hash__pgtable_trans_huge_deposit()
280 * hash fault code looks at the deposted pgtable in hash__pgtable_trans_huge_deposit()
288 pgtable_t pgtable; in hash__pgtable_trans_huge_withdraw() local
294 pgtable = *pgtable_slot; in hash__pgtable_trans_huge_withdraw()
303 memset(pgtable, in hash__pgtable_trans_huge_withdraw()
266 hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) hash__pgtable_trans_huge_deposit() argument
348 pgtable_t pgtable; hash__pmdp_huge_get_and_clear() local
[all...] |
| /kernel/linux/linux-6.6/arch/powerpc/mm/book3s64/ |
| H A D | hash_pgtable.c | 264 * We want to put the pgtable in pmd and use pgtable for tracking
268 pgtable_t pgtable) in hash__pgtable_trans_huge_deposit()
274 * we store the pgtable in the second half of PMD in hash__pgtable_trans_huge_deposit()
277 *pgtable_slot = pgtable; in hash__pgtable_trans_huge_deposit()
279 * expose the deposited pgtable to other cpus. in hash__pgtable_trans_huge_deposit()
281 * hash fault code looks at the deposted pgtable in hash__pgtable_trans_huge_deposit()
289 pgtable_t pgtable; in hash__pgtable_trans_huge_withdraw() local
295 pgtable = *pgtable_slot; in hash__pgtable_trans_huge_withdraw()
304 memset(pgtable, in hash__pgtable_trans_huge_withdraw()
267 hash__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) hash__pgtable_trans_huge_deposit() argument
349 pgtable_t pgtable; hash__pmdp_huge_get_and_clear() local
[all...] |
| /kernel/linux/linux-6.6/arch/sparc/mm/ |
| H A D | tlb.c | 269 pgtable_t pgtable) in pgtable_trans_huge_deposit()
271 struct list_head *lh = (struct list_head *) pgtable; in pgtable_trans_huge_deposit()
280 pmd_huge_pte(mm, pmdp) = pgtable; in pgtable_trans_huge_deposit()
286 pgtable_t pgtable; in pgtable_trans_huge_withdraw() local
291 pgtable = pmd_huge_pte(mm, pmdp); in pgtable_trans_huge_withdraw()
292 lh = (struct list_head *) pgtable; in pgtable_trans_huge_withdraw()
299 pte_val(pgtable[0]) = 0; in pgtable_trans_huge_withdraw()
300 pte_val(pgtable[1]) = 0; in pgtable_trans_huge_withdraw()
302 return pgtable; in pgtable_trans_huge_withdraw()
268 pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, pgtable_t pgtable) pgtable_trans_huge_deposit() argument
|
| /kernel/linux/linux-5.10/arch/m68k/include/asm/ |
| H A D | motorola_pgalloc.h | 42 static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) in pte_free() argument
44 free_pointer_table(pgtable, TABLE_PTE); in pte_free()
47 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, in __pte_free_tlb() argument
50 free_pointer_table(pgtable, TABLE_PTE); in __pte_free_tlb()
|
| H A D | mcf_pgalloc.h | 37 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, in __pte_free_tlb() argument
40 struct page *page = virt_to_page(pgtable); in __pte_free_tlb()
64 static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) in pte_free() argument
66 struct page *page = virt_to_page(pgtable); in pte_free()
|
| /kernel/linux/linux-6.6/arch/m68k/include/asm/ |
| H A D | mcf_pgalloc.h | 35 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, in __pte_free_tlb() argument
38 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in __pte_free_tlb()
60 static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) in pte_free() argument
62 struct ptdesc *ptdesc = virt_to_ptdesc(pgtable); in pte_free()
|
| H A D | motorola_pgalloc.h | 42 static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) in pte_free() argument
44 free_pointer_table(pgtable, TABLE_PTE); in pte_free()
47 static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, in __pte_free_tlb() argument
50 free_pointer_table(pgtable, TABLE_PTE); in __pte_free_tlb()
|
| /kernel/linux/linux-5.10/drivers/iommu/ |
| H A D | exynos-iommu.c | 183 static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova) in section_entry() argument
185 return pgtable + lv1ent_offset(iova); in section_entry()
248 sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ member
251 spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
275 phys_addr_t pgtable; /* assigned page table structure */ member
395 dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable); in show_fault_information()
396 ent = section_entry(phys_to_virt(data->pgtable), fault_addr); in show_fault_information()
500 __sysmmu_set_ptbase(data, data->pgtable); in __sysmmu_enable()
757 domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); in exynos_iommu_domain_alloc()
758 if (!domain->pgtable) in exynos_iommu_domain_alloc()
[all...] |
| /kernel/linux/linux-6.6/drivers/iommu/ |
| H A D | exynos-iommu.c | 177 static sysmmu_pte_t *section_entry(sysmmu_pte_t *pgtable, sysmmu_iova_t iova) in section_entry() argument
179 return pgtable + lv1ent_offset(iova); in section_entry()
247 sysmmu_pte_t *pgtable; /* lv1 page table, 16KB */ member
250 spinlock_t pgtablelock; /* lock for modifying page table @ pgtable */
297 phys_addr_t pgtable; /* assigned page table structure */ member
554 dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable); in show_fault_information()
555 ent = section_entry(phys_to_virt(data->pgtable), fault->addr); in show_fault_information()
653 __sysmmu_set_ptbase(data, data->pgtable); in __sysmmu_enable()
905 domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); in exynos_iommu_domain_alloc()
906 if (!domain->pgtable) in exynos_iommu_domain_alloc()
[all...] |
| /kernel/linux/linux-5.10/drivers/iommu/amd/ |
| H A D | iommu.c | 103 struct domain_pgtable *pgtable);
156 struct domain_pgtable *pgtable) in amd_iommu_domain_get_pgtable()
160 pgtable->root = (u64 *)(pt_root & PAGE_MASK); in amd_iommu_domain_get_pgtable()
161 pgtable->mode = pt_root & 7; /* lowest 3 bits encode pgtable mode */ in amd_iommu_domain_get_pgtable()
179 /* lowest 3 bits encode pgtable mode */ in amd_iommu_domain_set_pgtable()
1482 static void free_pagetable(struct domain_pgtable *pgtable) in free_pagetable() argument
1487 if (pgtable->mode == PAGE_MODE_NONE) in free_pagetable()
1490 BUG_ON(pgtable->mode < PAGE_MODE_NONE || in free_pagetable()
1491 pgtable in free_pagetable()
155 amd_iommu_domain_get_pgtable(struct protection_domain *domain, struct domain_pgtable *pgtable) amd_iommu_domain_get_pgtable() argument
1508 struct domain_pgtable pgtable; increase_address_space() local
1558 struct domain_pgtable pgtable; alloc_pte() local
1656 struct domain_pgtable pgtable; fetch_pte() local
1908 set_dte_entry(u16 devid, struct protection_domain *domain, struct domain_pgtable *pgtable, bool ats, bool ppr) set_dte_entry() argument
1992 struct domain_pgtable pgtable; do_attach() local
2316 update_device_table(struct protection_domain *domain, struct domain_pgtable *pgtable) update_device_table() argument
2328 update_and_flush_device_table(struct protection_domain *domain, struct domain_pgtable *pgtable) update_and_flush_device_table() argument
2337 struct domain_pgtable pgtable; update_domain() local
2413 struct domain_pgtable pgtable; protection_domain_free() local
2593 struct domain_pgtable pgtable; amd_iommu_map() local
2618 struct domain_pgtable pgtable; amd_iommu_unmap() local
2632 struct domain_pgtable pgtable; amd_iommu_iova_to_phys() local
2812 struct domain_pgtable pgtable; amd_iommu_domain_direct_map() local
3016 struct domain_pgtable pgtable; __set_gcr3() local
3034 struct domain_pgtable pgtable; __clear_gcr3() local
[all...] |
| /kernel/linux/linux-5.10/arch/arm64/include/asm/ |
| H A D | pgtable-types.h | 48 #include <asm-generic/pgtable-nopmd.h>
50 #include <asm-generic/pgtable-nopud.h>
52 #include <asm-generic/pgtable-nop4d.h>
|
| /kernel/linux/linux-6.6/arch/arm64/include/asm/ |
| H A D | pgtable-types.h | 48 #include <asm-generic/pgtable-nopmd.h>
50 #include <asm-generic/pgtable-nopud.h>
52 #include <asm-generic/pgtable-nop4d.h>
|
| /kernel/linux/linux-5.10/arch/arm/include/asm/ |
| H A D | pgtable-hwdef.h | 3 * arch/arm/include/asm/pgtable-hwdef.h
11 #include <asm/pgtable-3level-hwdef.h>
13 #include <asm/pgtable-2level-hwdef.h>
|
| /kernel/linux/linux-6.6/arch/arm/include/asm/ |
| H A D | pgtable-hwdef.h | 3 * arch/arm/include/asm/pgtable-hwdef.h
11 #include <asm/pgtable-3level-hwdef.h>
13 #include <asm/pgtable-2level-hwdef.h>
|