Lines Matching defs:ops
57 struct mem_ops *ops, bool expect);
600 static void *alloc_hpage(struct mem_ops *ops)
602 void *p = ops->setup_area(1);
604 ops->fault(p, 0, hpage_pmd_size);
618 if (!ops->check_huge(p, 1)) {
793 struct mem_ops *ops, bool expect)
813 else if (!ops->check_huge(p, expect ? nr_hpages : 0))
822 struct mem_ops *ops, bool expect)
825 if (!ops->check_huge(p, 0)) {
829 __madvise_collapse(msg, p, nr_hpages, ops, expect);
834 struct mem_ops *ops)
840 if (!ops->check_huge(p, 0)) {
852 if (ops->check_huge(p, nr_hpages))
866 struct mem_ops *ops, bool expect)
868 if (wait_for_scan(msg, p, nr_hpages, ops)) {
881 if (ops != &__anon_ops)
882 ops->fault(p, 0, nr_hpages * hpage_pmd_size);
884 if (ops->check_huge(p, expect ? nr_hpages : 0))
902 static bool is_tmpfs(struct mem_ops *ops)
904 return ops == &__file_ops && finfo.type == VMA_SHMEM;
934 static void collapse_full(struct collapse_context *c, struct mem_ops *ops)
940 p = ops->setup_area(nr_hpages);
941 ops->fault(p, 0, size);
943 ops, true);
945 ops->cleanup_area(p, size);
948 static void collapse_empty(struct collapse_context *c, struct mem_ops *ops)
952 p = ops->setup_area(1);
953 c->collapse("Do not collapse empty PTE table", p, 1, ops, false);
954 ops->cleanup_area(p, hpage_pmd_size);
957 static void collapse_single_pte_entry(struct collapse_context *c, struct mem_ops *ops)
961 p = ops->setup_area(1);
962 ops->fault(p, 0, page_size);
964 1, ops, true);
965 ops->cleanup_area(p, hpage_pmd_size);
968 static void collapse_max_ptes_none(struct collapse_context *c, struct mem_ops *ops)
977 p = ops->setup_area(1);
979 if (is_tmpfs(ops)) {
986 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none - 1) * page_size);
988 ops, !c->enforce_pte_scan_limits);
992 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_none) * page_size);
993 c->collapse("Collapse with max_ptes_none PTEs empty", p, 1, ops,
999 ops->cleanup_area(p, hpage_pmd_size);
1003 static void collapse_swapin_single_pte(struct collapse_context *c, struct mem_ops *ops)
1007 p = ops->setup_area(1);
1008 ops->fault(p, 0, hpage_pmd_size);
1022 c->collapse("Collapse with swapping in single PTE entry", p, 1, ops,
1026 ops->cleanup_area(p, hpage_pmd_size);
1029 static void collapse_max_ptes_swap(struct collapse_context *c, struct mem_ops *ops)
1034 p = ops->setup_area(1);
1035 ops->fault(p, 0, hpage_pmd_size);
1049 c->collapse("Maybe collapse with max_ptes_swap exceeded", p, 1, ops,
1054 ops->fault(p, 0, hpage_pmd_size);
1069 1, ops, true);
1073 ops->cleanup_area(p, hpage_pmd_size);
1076 static void collapse_single_pte_entry_compound(struct collapse_context *c, struct mem_ops *ops)
1080 p = alloc_hpage(ops);
1082 if (is_tmpfs(ops)) {
1092 if (ops->check_huge(p, 0))
1098 p, 1, ops, true);
1101 ops->cleanup_area(p, hpage_pmd_size);
1104 static void collapse_full_of_compound(struct collapse_context *c, struct mem_ops *ops)
1108 p = alloc_hpage(ops);
1112 if (ops->check_huge(p, 0))
1117 c->collapse("Collapse PTE table full of compound pages", p, 1, ops,
1120 ops->cleanup_area(p, hpage_pmd_size);
1123 static void collapse_compound_extreme(struct collapse_context *c, struct mem_ops *ops)
1128 p = ops->setup_area(1);
1134 ops->fault(BASE_ADDR, 0, hpage_pmd_size);
1135 if (!ops->check_huge(BASE_ADDR, 1)) {
1162 ops->cleanup_area(BASE_ADDR, hpage_pmd_size);
1163 ops->fault(p, 0, hpage_pmd_size);
1164 if (!ops->check_huge(p, 1))
1170 ops, true);
1173 ops->cleanup_area(p, hpage_pmd_size);
1176 static void collapse_fork(struct collapse_context *c, struct mem_ops *ops)
1181 p = ops->setup_area(1);
1184 ops->fault(p, 0, page_size);
1185 if (ops->check_huge(p, 0))
1196 if (ops->check_huge(p, 0))
1201 ops->fault(p, page_size, 2 * page_size);
1203 p, 1, ops, true);
1206 ops->cleanup_area(p, hpage_pmd_size);
1214 if (ops->check_huge(p, 0))
1219 ops->cleanup_area(p, hpage_pmd_size);
1222 static void collapse_fork_compound(struct collapse_context *c, struct mem_ops *ops)
1227 p = alloc_hpage(ops);
1234 if (ops->check_huge(p, 1))
1242 if (ops->check_huge(p, 0))
1246 ops->fault(p, 0, page_size);
1250 p, 1, ops, true);
1255 ops->cleanup_area(p, hpage_pmd_size);
1263 if (ops->check_huge(p, 1))
1268 ops->cleanup_area(p, hpage_pmd_size);
1271 static void collapse_max_ptes_shared(struct collapse_context *c, struct mem_ops *ops)
1277 p = alloc_hpage(ops);
1284 if (ops->check_huge(p, 1))
1291 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared - 1) * page_size);
1292 if (ops->check_huge(p, 0))
1298 1, ops, !c->enforce_pte_scan_limits);
1303 ops->fault(p, 0, (hpage_pmd_nr - max_ptes_shared) *
1305 if (ops->check_huge(p, 0))
1311 p, 1, ops, true);
1315 ops->cleanup_area(p, hpage_pmd_size);
1323 if (ops->check_huge(p, 1))
1328 ops->cleanup_area(p, hpage_pmd_size);
1332 struct mem_ops *ops)
1336 p = ops->setup_area(1);
1337 ops->fault(p, 0, hpage_pmd_size);
1338 c->collapse("Collapse fully populated PTE table...", p, 1, ops, true);
1342 __madvise_collapse("Re-collapse PMD-mapped hugepage", p, 1, ops, true);
1344 ops->cleanup_area(p, hpage_pmd_size);
1352 struct mem_ops *ops)
1358 p = ops->setup_area(nr_hpages);
1359 ops->fault(p, 0, size);
1363 ops)) {
1368 c->collapse("Install huge PMD from page cache", p, nr_hpages, ops,
1371 ops->cleanup_area(p, size);