Lines Matching refs:addr
1096 unsigned long addr)
1099 return vma_lookup(mm, addr);
1105 vma = find_vma(mm, addr);
1106 if (!vma || (addr >= vma->vm_start))
1112 if (vma->vm_start - addr > 65536)
1124 vma->vm_start, vma->vm_end, addr);
1284 * e.g. when start addr is not thp-size aligned.
1663 * We made sure addr is within a VMA, so the following will
1999 * @addr: user address
2012 struct page *get_dump_page(unsigned long addr)
2018 ret = __get_user_pages_locked(current->mm, addr, 1, &page, &locked,
2555 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
2563 ptem = ptep = pte_offset_map(&pmd, addr);
2640 } while (ptep++, addr += PAGE_SIZE, addr != end);
2661 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
2670 static int __gup_device_huge(unsigned long pfn, unsigned long addr,
2699 } while (addr += PAGE_SIZE, addr != end);
2702 return addr == end;
2705 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2712 fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2713 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2723 static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2730 fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2731 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2741 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2749 static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
2758 static int record_subpages(struct page *page, unsigned long addr,
2763 for (nr = 0; addr != end; nr++, addr += PAGE_SIZE)
2770 static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
2773 unsigned long __boundary = (addr + sz) & ~(sz-1);
2777 static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
2787 pte_end = (addr + sz) & ~(sz-1);
2799 page = nth_page(pte_page(pte), (addr & (sz - 1)) >> PAGE_SHIFT);
2800 refs = record_subpages(page, addr, end, pages + *nr);
2826 static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2834 ptep = hugepte_offset(hugepd, addr, pdshift);
2836 next = hugepte_addr_end(addr, end, sz);
2837 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
2839 } while (ptep++, addr = next, addr != end);
2844 static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2852 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2866 return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
2870 page = nth_page(pmd_page(orig), (addr & ~PMD_MASK) >> PAGE_SHIFT);
2871 refs = record_subpages(page, addr, end, pages + *nr);
2896 static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2910 return __gup_device_huge_pud(orig, pudp, addr, end, flags,
2914 page = nth_page(pud_page(orig), (addr & ~PUD_MASK) >> PAGE_SHIFT);
2915 refs = record_subpages(page, addr, end, pages + *nr);
2941 static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
2954 page = nth_page(pgd_page(orig), (addr & ~PGDIR_MASK) >> PAGE_SHIFT);
2955 refs = record_subpages(page, addr, end, pages + *nr);
2981 static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
2987 pmdp = pmd_offset_lockless(pudp, pud, addr);
2991 next = pmd_addr_end(addr, end);
3001 if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
3010 if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
3013 } else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))
3015 } while (pmdp++, addr = next, addr != end);
3020 static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
3026 pudp = pud_offset_lockless(p4dp, p4d, addr);
3030 next = pud_addr_end(addr, end);
3034 if (!gup_huge_pud(pud, pudp, addr, next, flags,
3038 if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
3041 } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
3043 } while (pudp++, addr = next, addr != end);
3048 static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
3054 p4dp = p4d_offset_lockless(pgdp, pgd, addr);
3058 next = p4d_addr_end(addr, end);
3063 if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
3066 } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
3068 } while (p4dp++, addr = next, addr != end);
3073 static void gup_pgd_range(unsigned long addr, unsigned long end,
3079 pgdp = pgd_offset(current->mm, addr);
3083 next = pgd_addr_end(addr, end);
3087 if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
3091 if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
3094 } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
3096 } while (pgdp++, addr = next, addr != end);
3099 static inline void gup_pgd_range(unsigned long addr, unsigned long end,