Lines Matching refs:pmem
30 #include "pmem.h"
35 static struct device *to_dev(struct pmem_device *pmem)
41 return pmem->bb.dev;
44 static struct nd_region *to_region(struct pmem_device *pmem)
46 return to_nd_region(to_dev(pmem)->parent);
49 static phys_addr_t pmem_to_phys(struct pmem_device *pmem, phys_addr_t offset)
51 return pmem->phys_addr + offset;
54 static sector_t to_sect(struct pmem_device *pmem, phys_addr_t offset)
56 return (offset - pmem->data_offset) >> SECTOR_SHIFT;
59 static phys_addr_t to_offset(struct pmem_device *pmem, sector_t sector)
61 return (sector << SECTOR_SHIFT) + pmem->data_offset;
64 static void pmem_mkpage_present(struct pmem_device *pmem, phys_addr_t offset,
67 phys_addr_t phys = pmem_to_phys(pmem, offset);
70 /* only pmem in the linear map supports HWPoison */
71 if (is_vmalloc_addr(pmem->virt_addr))
89 static void pmem_clear_bb(struct pmem_device *pmem, sector_t sector, long blks)
93 badblocks_clear(&pmem->bb, sector, blks);
94 if (pmem->bb_state)
95 sysfs_notify_dirent(pmem->bb_state);
98 static long __pmem_clear_poison(struct pmem_device *pmem,
101 phys_addr_t phys = pmem_to_phys(pmem, offset);
102 long cleared = nvdimm_clear_poison(to_dev(pmem), phys, len);
105 pmem_mkpage_present(pmem, offset, cleared);
106 arch_invalidate_pmem(pmem->virt_addr + offset, len);
111 static blk_status_t pmem_clear_poison(struct pmem_device *pmem,
114 long cleared = __pmem_clear_poison(pmem, offset, len);
119 pmem_clear_bb(pmem, to_sect(pmem, offset), cleared >> SECTOR_SHIFT);
165 static blk_status_t pmem_do_read(struct pmem_device *pmem,
170 phys_addr_t pmem_off = to_offset(pmem, sector);
171 void *pmem_addr = pmem->virt_addr + pmem_off;
173 if (unlikely(is_bad_pmem(&pmem->bb, sector, len)))
181 static blk_status_t pmem_do_write(struct pmem_device *pmem,
185 phys_addr_t pmem_off = to_offset(pmem, sector);
186 void *pmem_addr = pmem->virt_addr + pmem_off;
188 if (unlikely(is_bad_pmem(&pmem->bb, sector, len))) {
189 blk_status_t rc = pmem_clear_poison(pmem, pmem_off, len);
209 struct pmem_device *pmem = bio->bi_bdev->bd_disk->private_data;
210 struct nd_region *nd_region = to_region(pmem);
220 rc = pmem_do_write(pmem, bvec.bv_page, bvec.bv_offset,
223 rc = pmem_do_read(pmem, bvec.bv_page, bvec.bv_offset,
242 /* see "strong" declaration in tools/testing/nvdimm/pmem-dax.c */
243 __weak long __pmem_direct_access(struct pmem_device *pmem, pgoff_t pgoff,
247 resource_size_t offset = PFN_PHYS(pgoff) + pmem->data_offset;
250 struct badblocks *bb = &pmem->bb;
255 *kaddr = pmem->virt_addr + offset;
257 *pfn = phys_to_pfn_t(pmem->phys_addr + offset, pmem->pfn_flags);
273 dev_dbg(pmem->bb.dev, "start sector(%llu), nr_pages(%ld), first_bad(%llu), actual_nr(%ld)\n",
286 return PHYS_PFN(pmem->size - pmem->pfn_pad - offset);
297 struct pmem_device *pmem = dax_get_private(dax_dev);
299 return blk_status_to_errno(pmem_do_write(pmem, ZERO_PAGE(0), 0,
308 struct pmem_device *pmem = dax_get_private(dax_dev);
310 return __pmem_direct_access(pmem, pgoff, nr_pages, mode, kaddr, pfn);
329 struct pmem_device *pmem = dax_get_private(dax_dev);
332 struct device *dev = pmem->bb.dev;
337 if (!is_bad_pmem(&pmem->bb, PFN_PHYS(pgoff) >> SECTOR_SHIFT, len))
350 pmem_off = PFN_PHYS(pgoff) + pmem->data_offset;
351 cleared = __pmem_clear_poison(pmem, pmem_off, len);
363 pmem_clear_bb(pmem, to_sect(pmem, pmem_off), cleared >> SECTOR_SHIFT);
377 struct pmem_device *pmem = dev_to_disk(dev)->private_data;
379 return sprintf(buf, "%d\n", !!dax_write_cache_enabled(pmem->dax_dev));
385 struct pmem_device *pmem = dev_to_disk(dev)->private_data;
392 dax_write_cache(pmem->dax_dev, write_cache);
424 struct pmem_device *pmem = __pmem;
426 dax_remove_host(pmem->disk);
427 kill_dax(pmem->dax_dev);
428 put_dax(pmem->dax_dev);
429 del_gendisk(pmem->disk);
431 put_disk(pmem->disk);
437 struct pmem_device *pmem =
439 u64 offset = PFN_PHYS(pfn) - pmem->phys_addr - pmem->data_offset;
442 return dax_holder_notify_failure(pmem->dax_dev, offset, len, mf_flags);
460 struct pmem_device *pmem;
466 pmem = devm_kzalloc(dev, sizeof(*pmem), GFP_KERNEL);
467 if (!pmem)
477 rc = nvdimm_setup_pfn(nd_pfn, &pmem->pgmap);
485 dev_set_drvdata(dev, pmem);
486 pmem->phys_addr = res->start;
487 pmem->size = resource_size(res);
505 pmem->disk = disk;
506 pmem->pgmap.owner = pmem;
507 pmem->pfn_flags = PFN_DEV;
509 pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;
510 pmem->pgmap.ops = &fsdax_pagemap_ops;
511 addr = devm_memremap_pages(dev, &pmem->pgmap);
513 pmem->data_offset = le64_to_cpu(pfn_sb->dataoff);
514 pmem->pfn_pad = resource_size(res) -
515 range_len(&pmem->pgmap.range);
516 pmem->pfn_flags |= PFN_MAP;
517 bb_range = pmem->pgmap.range;
518 bb_range.start += pmem->data_offset;
520 pmem->pgmap.range.start = res->start;
521 pmem->pgmap.range.end = res->end;
522 pmem->pgmap.nr_range = 1;
523 pmem->pgmap.type = MEMORY_DEVICE_FS_DAX;
524 pmem->pgmap.ops = &fsdax_pagemap_ops;
525 addr = devm_memremap_pages(dev, &pmem->pgmap);
526 pmem->pfn_flags |= PFN_MAP;
527 bb_range = pmem->pgmap.range;
529 addr = devm_memremap(dev, pmem->phys_addr,
530 pmem->size, ARCH_MEMREMAP_PMEM);
539 pmem->virt_addr = addr;
547 if (pmem->pfn_flags & PFN_MAP)
551 disk->private_data = pmem;
553 set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset)
555 if (devm_init_badblocks(dev, &pmem->bb))
557 nvdimm_badblocks_populate(nd_region, &pmem->bb, &bb_range);
558 disk->bb = &pmem->bb;
560 dax_dev = alloc_dax(pmem, &pmem_dax_ops);
573 pmem->dax_dev = dax_dev;
578 if (devm_add_action_or_reset(dev, pmem_release_disk, pmem))
583 pmem->bb_state = sysfs_get_dirent(disk_to_dev(disk)->kobj.sd,
585 if (!pmem->bb_state)
590 dax_remove_host(pmem->disk);
592 kill_dax(pmem->dax_dev);
593 put_dax(pmem->dax_dev);
595 put_disk(pmem->disk);
627 * For the first case consider namespace as a raw pmem namespace
653 struct pmem_device *pmem = dev_get_drvdata(dev);
662 sysfs_put(pmem->bb_state);
663 pmem->bb_state = NULL;
692 struct pmem_device *pmem = dev_get_drvdata(dev);
694 nd_region = to_region(pmem);
695 bb = &pmem->bb;
696 bb_state = pmem->bb_state;
703 offset = pmem->data_offset +
722 struct pmem_device *pmem;
732 pmem = dev_get_drvdata(dev);
733 nvdimm_check_and_set_ro(pmem->disk);
751 MODULE_ALIAS("pmem");