1// SPDX-License-Identifier: GPL-2.0 2/* Copyright(c) 2016-2019 Intel Corporation. All rights reserved. */ 3#include <linux/memremap.h> 4#include <linux/pagemap.h> 5#include <linux/memory.h> 6#include <linux/module.h> 7#include <linux/device.h> 8#include <linux/pfn_t.h> 9#include <linux/slab.h> 10#include <linux/dax.h> 11#include <linux/fs.h> 12#include <linux/mm.h> 13#include <linux/mman.h> 14#include "dax-private.h" 15#include "bus.h" 16 17/* Memory resource name used for add_memory_driver_managed(). */ 18static const char *kmem_name; 19/* Set if any memory will remain added when the driver will be unloaded. */ 20static bool any_hotremove_failed; 21 22static int dax_kmem_range(struct dev_dax *dev_dax, int i, struct range *r) 23{ 24 struct dev_dax_range *dax_range = &dev_dax->ranges[i]; 25 struct range *range = &dax_range->range; 26 27 /* memory-block align the hotplug range */ 28 r->start = ALIGN(range->start, memory_block_size_bytes()); 29 r->end = ALIGN_DOWN(range->end + 1, memory_block_size_bytes()) - 1; 30 if (r->start >= r->end) { 31 r->start = range->start; 32 r->end = range->end; 33 return -ENOSPC; 34 } 35 return 0; 36} 37 38struct dax_kmem_data { 39 const char *res_name; 40 struct resource *res[]; 41}; 42 43static int dev_dax_kmem_probe(struct dev_dax *dev_dax) 44{ 45 struct device *dev = &dev_dax->dev; 46 struct dax_kmem_data *data; 47 int rc = -ENOMEM; 48 int i, mapped = 0; 49 int numa_node; 50 51 /* 52 * Ensure good NUMA information for the persistent memory. 53 * Without this check, there is a risk that slow memory 54 * could be mixed in a node with faster memory, causing 55 * unavoidable performance issues. 56 */ 57 numa_node = dev_dax->target_node; 58 if (numa_node < 0) { 59 dev_warn(dev, "rejecting DAX region with invalid node: %d\n", 60 numa_node); 61 return -EINVAL; 62 } 63 64 data = kzalloc(sizeof(*data) + sizeof(struct resource *) * dev_dax->nr_range, GFP_KERNEL); 65 if (!data) 66 return -ENOMEM; 67 68 data->res_name = kstrdup(dev_name(dev), GFP_KERNEL); 69 if (!data->res_name) 70 goto err_res_name; 71 72 for (i = 0; i < dev_dax->nr_range; i++) { 73 struct resource *res; 74 struct range range; 75 76 rc = dax_kmem_range(dev_dax, i, &range); 77 if (rc) { 78 dev_info(dev, "mapping%d: %#llx-%#llx too small after alignment\n", 79 i, range.start, range.end); 80 continue; 81 } 82 83 /* Region is permanently reserved if hotremove fails. */ 84 res = request_mem_region(range.start, range_len(&range), data->res_name); 85 if (!res) { 86 dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve region\n", 87 i, range.start, range.end); 88 /* 89 * Once some memory has been onlined we can't 90 * assume that it can be un-onlined safely. 91 */ 92 if (mapped) 93 continue; 94 rc = -EBUSY; 95 goto err_request_mem; 96 } 97 data->res[i] = res; 98 99 /* 100 * Set flags appropriate for System RAM. Leave ..._BUSY clear 101 * so that add_memory() can add a child resource. Do not 102 * inherit flags from the parent since it may set new flags 103 * unknown to us that will break add_memory() below. 104 */ 105 res->flags = IORESOURCE_SYSTEM_RAM; 106 107 /* 108 * Ensure that future kexec'd kernels will not treat 109 * this as RAM automatically. 110 */ 111 rc = add_memory_driver_managed(numa_node, range.start, 112 range_len(&range), kmem_name, MHP_NONE); 113 114 if (rc) { 115 dev_warn(dev, "mapping%d: %#llx-%#llx memory add failed\n", 116 i, range.start, range.end); 117 remove_resource(res); 118 kfree(res); 119 data->res[i] = NULL; 120 if (mapped) 121 continue; 122 goto err_request_mem; 123 } 124 mapped++; 125 } 126 127 dev_set_drvdata(dev, data); 128 129 return 0; 130 131err_request_mem: 132 kfree(data->res_name); 133err_res_name: 134 kfree(data); 135 return rc; 136} 137 138#ifdef CONFIG_MEMORY_HOTREMOVE 139static int dev_dax_kmem_remove(struct dev_dax *dev_dax) 140{ 141 int i, success = 0; 142 struct device *dev = &dev_dax->dev; 143 struct dax_kmem_data *data = dev_get_drvdata(dev); 144 145 /* 146 * We have one shot for removing memory, if some memory blocks were not 147 * offline prior to calling this function remove_memory() will fail, and 148 * there is no way to hotremove this memory until reboot because device 149 * unbind will succeed even if we return failure. 150 */ 151 for (i = 0; i < dev_dax->nr_range; i++) { 152 struct range range; 153 int rc; 154 155 rc = dax_kmem_range(dev_dax, i, &range); 156 if (rc) 157 continue; 158 159 rc = remove_memory(dev_dax->target_node, range.start, 160 range_len(&range)); 161 if (rc == 0) { 162 remove_resource(data->res[i]); 163 kfree(data->res[i]); 164 data->res[i] = NULL; 165 success++; 166 continue; 167 } 168 any_hotremove_failed = true; 169 dev_err(dev, 170 "mapping%d: %#llx-%#llx cannot be hotremoved until the next reboot\n", 171 i, range.start, range.end); 172 } 173 174 if (success >= dev_dax->nr_range) { 175 kfree(data->res_name); 176 kfree(data); 177 dev_set_drvdata(dev, NULL); 178 } 179 180 return 0; 181} 182#else 183static int dev_dax_kmem_remove(struct dev_dax *dev_dax) 184{ 185 /* 186 * Without hotremove purposely leak the request_mem_region() for the 187 * device-dax range and return '0' to ->remove() attempts. The removal 188 * of the device from the driver always succeeds, but the region is 189 * permanently pinned as reserved by the unreleased 190 * request_mem_region(). 191 */ 192 any_hotremove_failed = true; 193 return 0; 194} 195#endif /* CONFIG_MEMORY_HOTREMOVE */ 196 197static struct dax_device_driver device_dax_kmem_driver = { 198 .probe = dev_dax_kmem_probe, 199 .remove = dev_dax_kmem_remove, 200}; 201 202static int __init dax_kmem_init(void) 203{ 204 int rc; 205 206 /* Resource name is permanently allocated if any hotremove fails. */ 207 kmem_name = kstrdup_const("System RAM (kmem)", GFP_KERNEL); 208 if (!kmem_name) 209 return -ENOMEM; 210 211 rc = dax_driver_register(&device_dax_kmem_driver); 212 if (rc) 213 kfree_const(kmem_name); 214 return rc; 215} 216 217static void __exit dax_kmem_exit(void) 218{ 219 dax_driver_unregister(&device_dax_kmem_driver); 220 if (!any_hotremove_failed) 221 kfree_const(kmem_name); 222} 223 224MODULE_AUTHOR("Intel Corporation"); 225MODULE_LICENSE("GPL v2"); 226module_init(dax_kmem_init); 227module_exit(dax_kmem_exit); 228MODULE_ALIAS_DAX_DEVICE(0); 229