| /kernel/linux/linux-6.6/mm/ |
| H A D | userfaultfd.c | 77 pte_t _dst_pte, *dst_pte; in mfill_atomic_install_pte() local
94 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_install_pte()
95 if (!dst_pte) in mfill_atomic_install_pte()
109 if (!pte_none_mostly(ptep_get(dst_pte))) in mfill_atomic_install_pte()
129 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mfill_atomic_install_pte()
132 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mfill_atomic_install_pte()
135 pte_unmap_unlock(dst_pte, ptl); in mfill_atomic_install_pte()
220 pte_t _dst_pte, *dst_pte; in mfill_atomic_pte_zeropage() local
227 dst_pte = pte_offset_map_lock(dst_vma->vm_mm, dst_pmd, dst_addr, &ptl); in mfill_atomic_pte_zeropage()
228 if (!dst_pte) in mfill_atomic_pte_zeropage()
299 pte_t _dst_pte, *dst_pte; mfill_atomic_pte_poison() local
366 pte_t *dst_pte; mfill_atomic_hugetlb() local
[all...] |
| H A D | hugetlb.c | 5039 pte_t *src_pte, *dst_pte, entry; in copy_hugetlb_page_range() local
5075 dst_pte = huge_pte_alloc(dst, dst_vma, addr, sz); in copy_hugetlb_page_range()
5076 if (!dst_pte) { in copy_hugetlb_page_range()
5084 * dst_pte == src_pte is the common case of src/dest sharing. in copy_hugetlb_page_range()
5089 if (page_count(virt_to_page(dst_pte)) > 1) { in copy_hugetlb_page_range()
5094 dst_ptl = huge_pte_lock(h, dst, dst_pte); in copy_hugetlb_page_range()
5107 set_huge_pte_at(dst, addr, dst_pte, entry, sz); in copy_hugetlb_page_range()
5126 set_huge_pte_at(dst, addr, dst_pte, entry, sz); in copy_hugetlb_page_range()
5132 set_huge_pte_at(dst, addr, dst_pte, in copy_hugetlb_page_range()
5175 dst_ptl = huge_pte_lock(h, dst, dst_pte); in copy_hugetlb_page_range()
5225 move_huge_pte(struct vm_area_struct *vma, unsigned long old_addr, unsigned long new_addr, pte_t *src_pte, pte_t *dst_pte, unsigned long sz) move_huge_pte() argument
5263 pte_t *src_pte, *dst_pte; move_hugetlb_page_tables() local
6284 hugetlb_mfill_atomic_pte(pte_t *dst_pte, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, uffd_flags_t flags, struct folio **foliop) hugetlb_mfill_atomic_pte() argument
[all...] |
| H A D | memory.c | 769 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma, in copy_nonpresent_pte()
865 set_pte_at(dst_mm, addr, dst_pte, in copy_nonpresent_pte()
871 set_pte_at(dst_mm, addr, dst_pte, pte); in copy_nonpresent_pte()
889 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss, in copy_present_page()
916 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); in copy_present_page()
926 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss, in copy_present_pte()
949 return copy_present_page(dst_vma, src_vma, dst_pte, src_pte, in copy_present_pte()
980 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); in copy_present_pte()
1010 pte_t *src_pte, *dst_pte; in copy_pte_range() local
1030 dst_pte in copy_pte_range()
768 copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, unsigned long addr, int *rss) copy_nonpresent_pte() argument
888 copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss, struct folio **prealloc, struct page *page) copy_present_page() argument
925 copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss, struct folio **prealloc) copy_present_pte() argument
[all...] |
| /kernel/linux/linux-5.10/mm/ |
| H A D | userfaultfd.c | 59 pte_t _dst_pte, *dst_pte; in mcopy_atomic_pte() local
112 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mcopy_atomic_pte()
123 if (!pte_none(*dst_pte)) in mcopy_atomic_pte()
130 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in mcopy_atomic_pte()
133 update_mmu_cache(dst_vma, dst_addr, dst_pte); in mcopy_atomic_pte()
135 pte_unmap_unlock(dst_pte, ptl); in mcopy_atomic_pte()
140 pte_unmap_unlock(dst_pte, ptl); in mcopy_atomic_pte()
151 pte_t _dst_pte, *dst_pte; in mfill_zeropage_pte() local
159 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in mfill_zeropage_pte()
170 if (!pte_none(*dst_pte)) in mfill_zeropage_pte()
217 pte_t *dst_pte; __mcopy_atomic_hugetlb() local
[all...] |
| H A D | hugetlb.c | 3814 pte_t *src_pte, *dst_pte, entry, dst_entry; in copy_hugetlb_page_range() local
3846 dst_pte = huge_pte_alloc(dst, addr, sz); in copy_hugetlb_page_range()
3847 if (!dst_pte) { in copy_hugetlb_page_range()
3854 * dst_pte == src_pte is the common case of src/dest sharing. in copy_hugetlb_page_range()
3857 * another vma. If dst_pte !none, this implies sharing. in copy_hugetlb_page_range()
3861 dst_entry = huge_ptep_get(dst_pte); in copy_hugetlb_page_range()
3862 if ((dst_pte == src_pte) || !huge_pte_none(dst_entry)) in copy_hugetlb_page_range()
3865 dst_ptl = huge_pte_lock(h, dst, dst_pte); in copy_hugetlb_page_range()
3869 dst_entry = huge_ptep_get(dst_pte); in copy_hugetlb_page_range()
3891 set_huge_swap_pte_at(dst, addr, dst_pte, entr in copy_hugetlb_page_range()
4715 hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, struct page **pagep) hugetlb_mcopy_atomic_pte() argument
[all...] |
| H A D | memory.c | 701 pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma, in copy_nonpresent_pte()
775 set_pte_at(dst_mm, addr, dst_pte, pte); in copy_nonpresent_pte()
801 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss, in copy_present_page()
860 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); in copy_present_page()
870 pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss, in copy_present_pte()
882 retval = copy_present_page(dst_vma, src_vma, dst_pte, src_pte, in copy_present_pte()
912 set_pte_at(dst_vma->vm_mm, addr, dst_pte, pte); in copy_present_pte()
943 pte_t *src_pte, *dst_pte; in copy_pte_range() local
954 dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl); in copy_pte_range()
955 if (!dst_pte) { in copy_pte_range()
700 copy_nonpresent_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm, pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, unsigned long addr, int *rss) copy_nonpresent_pte() argument
800 copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss, struct page **prealloc, pte_t pte, struct page *page) copy_present_page() argument
869 copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, pte_t *dst_pte, pte_t *src_pte, unsigned long addr, int *rss, struct page **prealloc) copy_present_pte() argument
[all...] |
| H A D | shmem.c | 2367 pte_t _dst_pte, *dst_pte; in shmem_mfill_atomic_pte() local
2442 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl); in shmem_mfill_atomic_pte()
2450 if (!pte_none(*dst_pte)) in shmem_mfill_atomic_pte()
2463 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte); in shmem_mfill_atomic_pte()
2466 update_mmu_cache(dst_vma, dst_addr, dst_pte); in shmem_mfill_atomic_pte()
2467 pte_unmap_unlock(dst_pte, ptl); in shmem_mfill_atomic_pte()
2473 pte_unmap_unlock(dst_pte, ptl); in shmem_mfill_atomic_pte()
|
| /kernel/linux/linux-5.10/include/linux/ |
| H A D | shmem_fs.h | 132 #define shmem_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma, dst_addr, \
|
| H A D | hugetlb.h | 138 int hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte,
313 pte_t *dst_pte, in hugetlb_mcopy_atomic_pte()
312 hugetlb_mcopy_atomic_pte(struct mm_struct *dst_mm, pte_t *dst_pte, struct vm_area_struct *dst_vma, unsigned long dst_addr, unsigned long src_addr, struct page **pagep) hugetlb_mcopy_atomic_pte() argument
|
| /kernel/linux/linux-6.6/include/linux/ |
| H A D | hugetlb.h | 153 int hugetlb_mfill_atomic_pte(pte_t *dst_pte,
425 static inline int hugetlb_mfill_atomic_pte(pte_t *dst_pte, in hugetlb_mfill_atomic_pte() argument
|