Lines Matching refs:dst_mm
22 struct vm_area_struct *find_dst_vma(struct mm_struct *dst_mm,
32 dst_vma = find_vma(dst_mm, dst_start);
51 static int mcopy_atomic_pte(struct mm_struct *dst_mm,
101 if (mem_cgroup_charge(page, dst_mm, GFP_KERNEL))
112 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
126 inc_mm_counter(dst_mm, MM_ANONPAGES);
130 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
146 static int mfill_zeropage_pte(struct mm_struct *dst_mm,
159 dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
172 set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
207 static __always_inline ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
233 mmap_read_unlock(dst_mm);
257 dst_vma = find_dst_vma(dst_mm, dst_start, len);
295 dst_pte = huge_pte_alloc(dst_mm, dst_addr, vma_hpagesize);
310 err = hugetlb_mcopy_atomic_pte(dst_mm, dst_pte, dst_vma,
320 mmap_read_unlock(dst_mm);
331 mmap_read_lock(dst_mm);
351 mmap_read_unlock(dst_mm);
408 extern ssize_t __mcopy_atomic_hugetlb(struct mm_struct *dst_mm,
416 static __always_inline ssize_t mfill_atomic_pte(struct mm_struct *dst_mm,
439 err = mcopy_atomic_pte(dst_mm, dst_pmd, dst_vma,
443 err = mfill_zeropage_pte(dst_mm, dst_pmd,
448 err = shmem_mcopy_atomic_pte(dst_mm, dst_pmd,
452 err = shmem_mfill_zeropage_pte(dst_mm, dst_pmd,
459 static __always_inline ssize_t __mcopy_atomic(struct mm_struct *dst_mm,
490 mmap_read_lock(dst_mm);
506 dst_vma = find_dst_vma(dst_mm, dst_start, len);
531 return __mcopy_atomic_hugetlb(dst_mm, dst_vma, dst_start,
552 dst_pmd = mm_alloc_pmd(dst_mm, dst_addr);
568 unlikely(__pte_alloc(dst_mm, dst_pmd))) {
581 err = mfill_atomic_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
588 mmap_read_unlock(dst_mm);
618 mmap_read_unlock(dst_mm);
628 ssize_t mcopy_atomic(struct mm_struct *dst_mm, unsigned long dst_start,
632 return __mcopy_atomic(dst_mm, dst_start, src_start, len, false,
636 ssize_t mfill_zeropage(struct mm_struct *dst_mm, unsigned long start,
639 return __mcopy_atomic(dst_mm, start, 0, len, true, mmap_changing, 0);
642 int mwriteprotect_range(struct mm_struct *dst_mm, unsigned long start,
658 mmap_read_lock(dst_mm);
670 dst_vma = find_dst_vma(dst_mm, start, len);
692 mmap_read_unlock(dst_mm);