Lines Matching refs:pgmap
45 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
47 if (pgmap->type == MEMORY_DEVICE_FS_DAX)
51 static void devmap_managed_enable_get(struct dev_pagemap *pgmap)
53 if (pgmap->type == MEMORY_DEVICE_FS_DAX)
57 static void devmap_managed_enable_get(struct dev_pagemap *pgmap)
60 static void devmap_managed_enable_put(struct dev_pagemap *pgmap)
72 static unsigned long pfn_first(struct dev_pagemap *pgmap, int range_id)
74 struct range *range = &pgmap->ranges[range_id];
79 return pfn + vmem_altmap_offset(pgmap_altmap(pgmap));
82 bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn)
86 for (i = 0; i < pgmap->nr_range; i++) {
87 struct range *range = &pgmap->ranges[i];
91 return pfn >= pfn_first(pgmap, i);
97 static unsigned long pfn_end(struct dev_pagemap *pgmap, int range_id)
99 const struct range *range = &pgmap->ranges[range_id];
104 static unsigned long pfn_len(struct dev_pagemap *pgmap, unsigned long range_id)
106 return (pfn_end(pgmap, range_id) -
107 pfn_first(pgmap, range_id)) >> pgmap->vmemmap_shift;
110 static void pageunmap_range(struct dev_pagemap *pgmap, int range_id)
112 struct range *range = &pgmap->ranges[range_id];
116 first_page = pfn_to_page(pfn_first(pgmap, range_id));
122 if (pgmap->type == MEMORY_DEVICE_PRIVATE) {
127 pgmap_altmap(pgmap));
136 void memunmap_pages(struct dev_pagemap *pgmap)
140 percpu_ref_kill(&pgmap->ref);
141 if (pgmap->type != MEMORY_DEVICE_PRIVATE &&
142 pgmap->type != MEMORY_DEVICE_COHERENT)
143 for (i = 0; i < pgmap->nr_range; i++)
144 percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i));
146 wait_for_completion(&pgmap->done);
148 for (i = 0; i < pgmap->nr_range; i++)
149 pageunmap_range(pgmap, i);
150 percpu_ref_exit(&pgmap->ref);
152 WARN_ONCE(pgmap->altmap.alloc, "failed to free all reserved pages\n");
153 devmap_managed_enable_put(pgmap);
164 struct dev_pagemap *pgmap = container_of(ref, struct dev_pagemap, ref);
166 complete(&pgmap->done);
169 static int pagemap_range(struct dev_pagemap *pgmap, struct mhp_params *params,
172 const bool is_private = pgmap->type == MEMORY_DEVICE_PRIVATE;
173 struct range *range = &pgmap->ranges[range_id];
177 if (WARN_ONCE(pgmap_altmap(pgmap) && range_id > 0,
206 PHYS_PFN(range->end), pgmap, GFP_KERNEL));
269 PHYS_PFN(range_len(range)), pgmap);
270 if (pgmap->type != MEMORY_DEVICE_PRIVATE &&
271 pgmap->type != MEMORY_DEVICE_COHERENT)
272 percpu_ref_get_many(&pgmap->ref, pfn_len(pgmap, range_id));
291 void *memremap_pages(struct dev_pagemap *pgmap, int nid)
294 .altmap = pgmap_altmap(pgmap),
295 .pgmap = pgmap,
298 const int nr_range = pgmap->nr_range;
304 switch (pgmap->type) {
310 if (!pgmap->ops || !pgmap->ops->migrate_to_ram) {
314 if (!pgmap->ops->page_free) {
318 if (!pgmap->owner) {
324 if (!pgmap->ops->page_free) {
328 if (!pgmap->owner) {
346 WARN(1, "Invalid pgmap type %d\n", pgmap->type);
350 init_completion(&pgmap->done);
351 error = percpu_ref_init(&pgmap->ref, dev_pagemap_percpu_release, 0,
356 devmap_managed_enable_get(pgmap);
359 * Clear the pgmap nr_range as it will be incremented for each
363 pgmap->nr_range = 0;
366 error = pagemap_range(pgmap, ¶ms, i, nid);
369 pgmap->nr_range++;
373 memunmap_pages(pgmap);
374 pgmap->nr_range = nr_range;
378 return __va(pgmap->ranges[0].start);
385 * @pgmap: pointer to a struct dev_pagemap
388 * 1/ At a minimum the range and type members of @pgmap must be initialized
392 * PGMAP_ALTMAP_VALID must be set in pgmap->flags.
394 * 3/ The ref field may optionally be provided, in which pgmap->ref must be
402 void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap)
407 ret = memremap_pages(pgmap, dev_to_node(dev));
412 pgmap);
419 void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap)
421 devm_release_action(dev, devm_memremap_pages_release, pgmap);
441 * @pgmap: optional known pgmap that already has a reference
443 * If @pgmap is non-NULL and covers @pfn it will be returned as-is. If @pgmap
447 struct dev_pagemap *pgmap)
454 if (pgmap) {
455 if (phys >= pgmap->range.start && phys <= pgmap->range.end)
456 return pgmap;
457 put_dev_pagemap(pgmap);
462 pgmap = xa_load(&pgmap_array, PHYS_PFN(phys));
463 if (pgmap && !percpu_ref_tryget_live_rcu(&pgmap->ref))
464 pgmap = NULL;
467 return pgmap;
473 if (WARN_ON_ONCE(!page->pgmap->ops || !page->pgmap->ops->page_free))
509 page->pgmap->ops->page_free(page);
511 if (page->pgmap->type != MEMORY_DEVICE_PRIVATE &&
512 page->pgmap->type != MEMORY_DEVICE_COHERENT)
519 put_dev_pagemap(page->pgmap);
528 WARN_ON_ONCE(!percpu_ref_tryget_live(&page->pgmap->ref));
537 if (page->pgmap->type != MEMORY_DEVICE_FS_DAX)