Lines Matching refs:addr
1433 * We made sure addr is within a VMA, so the following will
1546 * @addr: user address
1559 struct page *get_dump_page(unsigned long addr)
1568 ret = __get_user_pages_locked(mm, addr, 1, &page, NULL, &locked,
2162 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
2170 ptem = ptep = pte_offset_map(&pmd, addr);
2229 } while (ptep++, addr += PAGE_SIZE, addr != end);
2250 static int gup_pte_range(pmd_t pmd, pmd_t *pmdp, unsigned long addr,
2259 static int __gup_device_huge(unsigned long pfn, unsigned long addr,
2282 } while (addr += PAGE_SIZE, addr != end);
2289 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2296 fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2297 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2307 static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2314 fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2315 if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
2325 static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2333 static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
2342 static int record_subpages(struct page *page, unsigned long addr,
2347 for (nr = 0; addr != end; addr += PAGE_SIZE)
2354 static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
2357 unsigned long __boundary = (addr + sz) & ~(sz-1);
2361 static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
2370 pte_end = (addr + sz) & ~(sz-1);
2383 page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
2384 refs = record_subpages(page, addr, end, pages + *nr);
2400 static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2408 ptep = hugepte_offset(hugepd, addr, pdshift);
2410 next = hugepte_addr_end(addr, end, sz);
2411 if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
2413 } while (ptep++, addr = next, addr != end);
2418 static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
2426 static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
2439 return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
2443 page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
2444 refs = record_subpages(page, addr, end, pages + *nr);
2460 static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
2473 return __gup_device_huge_pud(orig, pudp, addr, end, flags,
2477 page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
2478 refs = record_subpages(page, addr, end, pages + *nr);
2494 static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
2506 page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
2507 refs = record_subpages(page, addr, end, pages + *nr);
2523 static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
2529 pmdp = pmd_offset_lockless(pudp, pud, addr);
2533 next = pmd_addr_end(addr, end);
2547 if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
2556 if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
2559 } else if (!gup_pte_range(pmd, pmdp, addr, next, flags, pages, nr))
2561 } while (pmdp++, addr = next, addr != end);
2566 static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
2572 pudp = pud_offset_lockless(p4dp, p4d, addr);
2576 next = pud_addr_end(addr, end);
2580 if (!gup_huge_pud(pud, pudp, addr, next, flags,
2584 if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
2587 } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
2589 } while (pudp++, addr = next, addr != end);
2594 static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
2600 p4dp = p4d_offset_lockless(pgdp, pgd, addr);
2604 next = p4d_addr_end(addr, end);
2609 if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
2612 } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
2614 } while (p4dp++, addr = next, addr != end);
2619 static void gup_pgd_range(unsigned long addr, unsigned long end,
2625 pgdp = pgd_offset(current->mm, addr);
2629 next = pgd_addr_end(addr, end);
2633 if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
2637 if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
2640 } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
2642 } while (pgdp++, addr = next, addr != end);
2645 static inline void gup_pgd_range(unsigned long addr, unsigned long end,