Lines Matching defs:gup_flags
809 unsigned int gup_flags, struct vm_area_struct **vma,
820 if (gup_flags & FOLL_WRITE)
846 if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
850 if (unlikely(!try_grab_page(*page, gup_flags))) {
920 static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
923 int write = (gup_flags & FOLL_WRITE);
924 int foreign = (gup_flags & FOLL_REMOTE);
929 if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma))
934 if (!(gup_flags & FOLL_FORCE))
949 if (!(gup_flags & FOLL_FORCE))
972 * @gup_flags: flags modifying pin behaviour
1009 * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
1015 * released by an up_read(). That can happen if @gup_flags does not
1018 * A caller using such a combination of @locked and @gup_flags
1025 * you need some special @gup_flags.
1029 unsigned int gup_flags, struct page **pages,
1041 VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
1048 if (!(gup_flags & FOLL_FORCE))
1049 gup_flags |= FOLL_NUMA;
1053 unsigned int foll_flags = gup_flags;
1061 gup_flags, &vma,
1069 if (!vma || check_vma_flags(vma, gup_flags)) {
1076 gup_flags, locked);
1083 BUG_ON(gup_flags & FOLL_NOWAIT);
1406 int gup_flags;
1414 gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
1416 gup_flags &= ~FOLL_POPULATE;
1423 gup_flags |= FOLL_WRITE;
1430 gup_flags |= FOLL_FORCE;
1436 return __get_user_pages(mm, start, nr_pages, gup_flags,
1602 unsigned int gup_flags)
1662 if (gup_flags & FOLL_PIN)
1679 NULL, gup_flags);
1697 unsigned int gup_flags)
1712 unsigned int gup_flags)
1718 if (gup_flags & FOLL_LONGTERM) {
1733 vmas_tmp, NULL, gup_flags);
1735 if (gup_flags & FOLL_LONGTERM) {
1740 if (gup_flags & FOLL_PIN)
1750 vmas_tmp, gup_flags);
1772 static bool is_valid_gup_flags(unsigned int gup_flags)
1778 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1785 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1794 unsigned int gup_flags, struct page **pages,
1804 if (gup_flags & FOLL_LONGTERM) {
1812 vmas, gup_flags | FOLL_TOUCH |
1818 gup_flags | FOLL_TOUCH | FOLL_REMOTE);
1826 * @gup_flags: flags modifying lookup behaviour
1864 * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
1883 unsigned int gup_flags, struct page **pages,
1886 if (!is_valid_gup_flags(gup_flags))
1889 return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
1897 unsigned int gup_flags, struct page **pages,
1905 unsigned int gup_flags, struct page **pages,
1916 * @gup_flags: flags modifying lookup behaviour
1929 unsigned int gup_flags, struct page **pages,
1932 if (!is_valid_gup_flags(gup_flags))
1936 pages, vmas, gup_flags | FOLL_TOUCH);
1959 * @gup_flags: flags modifying lookup behaviour
1973 unsigned int gup_flags, struct page **pages,
1982 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
1988 if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
1993 gup_flags | FOLL_TOUCH);
2009 * get_user_pages_fast should be used instead if specific gup_flags
2013 struct page **pages, unsigned int gup_flags)
2025 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
2030 &locked, gup_flags | FOLL_TOUCH);
2663 unsigned int gup_flags, struct page **pages)
2671 if (gup_flags & FOLL_LONGTERM) {
2675 pages, NULL, gup_flags);
2679 pages, gup_flags);
2687 unsigned int gup_flags,
2698 if (gup_flags & FOLL_PIN) {
2716 gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
2723 if (gup_flags & FOLL_PIN) {
2734 unsigned int gup_flags,
2741 if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
2746 if (gup_flags & FOLL_PIN)
2749 if (!(gup_flags & FOLL_FAST_ONLY))
2759 nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);
2760 if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
2766 ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags,
2784 * @gup_flags: flags modifying pin behaviour
2801 unsigned int gup_flags, struct page **pages)
2811 gup_flags |= FOLL_GET | FOLL_FAST_ONLY;
2813 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2833 * @gup_flags: flags modifying pin behaviour
2846 unsigned int gup_flags, struct page **pages)
2848 if (!is_valid_gup_flags(gup_flags))
2857 gup_flags |= FOLL_GET;
2858 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2867 * @gup_flags: flags modifying pin behaviour
2879 unsigned int gup_flags, struct page **pages)
2882 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2885 gup_flags |= FOLL_PIN;
2886 return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
2897 unsigned int gup_flags, struct page **pages)
2905 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2911 gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY);
2912 nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
2932 * @gup_flags: flags modifying lookup behaviour
2951 unsigned int gup_flags, struct page **pages,
2955 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2958 gup_flags |= FOLL_PIN;
2959 return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
2969 * @gup_flags: flags modifying lookup behaviour
2983 unsigned int gup_flags, struct page **pages,
2987 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
2990 gup_flags |= FOLL_PIN;
2992 pages, vmas, gup_flags);
3002 struct page **pages, unsigned int gup_flags)
3005 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
3008 gup_flags |= FOLL_PIN;
3009 return get_user_pages_unlocked(start, nr_pages, pages, gup_flags);
3019 unsigned int gup_flags, struct page **pages,
3028 if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
3032 if (WARN_ON_ONCE(gup_flags & FOLL_GET))
3035 gup_flags |= FOLL_PIN;
3038 gup_flags | FOLL_TOUCH);