1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved. 4 */ 5#include <linux/scatterlist.h> 6#include <linux/memregion.h> 7#include <linux/highmem.h> 8#include <linux/sched.h> 9#include <linux/slab.h> 10#include <linux/hash.h> 11#include <linux/sort.h> 12#include <linux/io.h> 13#include <linux/nd.h> 14#include "nd-core.h" 15#include "nd.h" 16 17/* 18 * For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is 19 * irrelevant. 20 */ 21#include <linux/io-64-nonatomic-hi-lo.h> 22 23static DEFINE_PER_CPU(int, flush_idx); 24 25static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm, 26 struct nd_region_data *ndrd) 27{ 28 int i, j; 29 30 dev_dbg(dev, "%s: map %d flush address%s\n", nvdimm_name(nvdimm), 31 nvdimm->num_flush, nvdimm->num_flush == 1 ? "" : "es"); 32 for (i = 0; i < (1 << ndrd->hints_shift); i++) { 33 struct resource *res = &nvdimm->flush_wpq[i]; 34 unsigned long pfn = PHYS_PFN(res->start); 35 void __iomem *flush_page; 36 37 /* check if flush hints share a page */ 38 for (j = 0; j < i; j++) { 39 struct resource *res_j = &nvdimm->flush_wpq[j]; 40 unsigned long pfn_j = PHYS_PFN(res_j->start); 41 42 if (pfn == pfn_j) 43 break; 44 } 45 46 if (j < i) 47 flush_page = (void __iomem *) ((unsigned long) 48 ndrd_get_flush_wpq(ndrd, dimm, j) 49 & PAGE_MASK); 50 else 51 flush_page = devm_nvdimm_ioremap(dev, 52 PFN_PHYS(pfn), PAGE_SIZE); 53 if (!flush_page) 54 return -ENXIO; 55 ndrd_set_flush_wpq(ndrd, dimm, i, flush_page 56 + (res->start & ~PAGE_MASK)); 57 } 58 59 return 0; 60} 61 62int nd_region_activate(struct nd_region *nd_region) 63{ 64 int i, j, num_flush = 0; 65 struct nd_region_data *ndrd; 66 struct device *dev = &nd_region->dev; 67 size_t flush_data_size = sizeof(void *); 68 69 nvdimm_bus_lock(&nd_region->dev); 70 for (i = 0; i < nd_region->ndr_mappings; i++) { 71 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 72 struct nvdimm *nvdimm = nd_mapping->nvdimm; 73 74 if (test_bit(NDD_SECURITY_OVERWRITE, &nvdimm->flags)) { 75 nvdimm_bus_unlock(&nd_region->dev); 76 return -EBUSY; 77 } 78 79 /* at least one null hint slot per-dimm for the "no-hint" case */ 80 flush_data_size += sizeof(void *); 81 num_flush = min_not_zero(num_flush, nvdimm->num_flush); 82 if (!nvdimm->num_flush) 83 continue; 84 flush_data_size += nvdimm->num_flush * sizeof(void *); 85 } 86 nvdimm_bus_unlock(&nd_region->dev); 87 88 ndrd = devm_kzalloc(dev, sizeof(*ndrd) + flush_data_size, GFP_KERNEL); 89 if (!ndrd) 90 return -ENOMEM; 91 dev_set_drvdata(dev, ndrd); 92 93 if (!num_flush) 94 return 0; 95 96 ndrd->hints_shift = ilog2(num_flush); 97 for (i = 0; i < nd_region->ndr_mappings; i++) { 98 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 99 struct nvdimm *nvdimm = nd_mapping->nvdimm; 100 int rc = nvdimm_map_flush(&nd_region->dev, nvdimm, i, ndrd); 101 102 if (rc) 103 return rc; 104 } 105 106 /* 107 * Clear out entries that are duplicates. This should prevent the 108 * extra flushings. 109 */ 110 for (i = 0; i < nd_region->ndr_mappings - 1; i++) { 111 /* ignore if NULL already */ 112 if (!ndrd_get_flush_wpq(ndrd, i, 0)) 113 continue; 114 115 for (j = i + 1; j < nd_region->ndr_mappings; j++) 116 if (ndrd_get_flush_wpq(ndrd, i, 0) == 117 ndrd_get_flush_wpq(ndrd, j, 0)) 118 ndrd_set_flush_wpq(ndrd, j, 0, NULL); 119 } 120 121 return 0; 122} 123 124static void nd_region_release(struct device *dev) 125{ 126 struct nd_region *nd_region = to_nd_region(dev); 127 u16 i; 128 129 for (i = 0; i < nd_region->ndr_mappings; i++) { 130 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 131 struct nvdimm *nvdimm = nd_mapping->nvdimm; 132 133 put_device(&nvdimm->dev); 134 } 135 free_percpu(nd_region->lane); 136 memregion_free(nd_region->id); 137 if (is_nd_blk(dev)) 138 kfree(to_nd_blk_region(dev)); 139 else 140 kfree(nd_region); 141} 142 143struct nd_region *to_nd_region(struct device *dev) 144{ 145 struct nd_region *nd_region = container_of(dev, struct nd_region, dev); 146 147 WARN_ON(dev->type->release != nd_region_release); 148 return nd_region; 149} 150EXPORT_SYMBOL_GPL(to_nd_region); 151 152struct device *nd_region_dev(struct nd_region *nd_region) 153{ 154 if (!nd_region) 155 return NULL; 156 return &nd_region->dev; 157} 158EXPORT_SYMBOL_GPL(nd_region_dev); 159 160struct nd_blk_region *to_nd_blk_region(struct device *dev) 161{ 162 struct nd_region *nd_region = to_nd_region(dev); 163 164 WARN_ON(!is_nd_blk(dev)); 165 return container_of(nd_region, struct nd_blk_region, nd_region); 166} 167EXPORT_SYMBOL_GPL(to_nd_blk_region); 168 169void *nd_region_provider_data(struct nd_region *nd_region) 170{ 171 return nd_region->provider_data; 172} 173EXPORT_SYMBOL_GPL(nd_region_provider_data); 174 175void *nd_blk_region_provider_data(struct nd_blk_region *ndbr) 176{ 177 return ndbr->blk_provider_data; 178} 179EXPORT_SYMBOL_GPL(nd_blk_region_provider_data); 180 181void nd_blk_region_set_provider_data(struct nd_blk_region *ndbr, void *data) 182{ 183 ndbr->blk_provider_data = data; 184} 185EXPORT_SYMBOL_GPL(nd_blk_region_set_provider_data); 186 187/** 188 * nd_region_to_nstype() - region to an integer namespace type 189 * @nd_region: region-device to interrogate 190 * 191 * This is the 'nstype' attribute of a region as well, an input to the 192 * MODALIAS for namespace devices, and bit number for a nvdimm_bus to match 193 * namespace devices with namespace drivers. 194 */ 195int nd_region_to_nstype(struct nd_region *nd_region) 196{ 197 if (is_memory(&nd_region->dev)) { 198 u16 i, label; 199 200 for (i = 0, label = 0; i < nd_region->ndr_mappings; i++) { 201 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 202 struct nvdimm *nvdimm = nd_mapping->nvdimm; 203 204 if (test_bit(NDD_LABELING, &nvdimm->flags)) 205 label++; 206 } 207 if (label) 208 return ND_DEVICE_NAMESPACE_PMEM; 209 else 210 return ND_DEVICE_NAMESPACE_IO; 211 } else if (is_nd_blk(&nd_region->dev)) { 212 return ND_DEVICE_NAMESPACE_BLK; 213 } 214 215 return 0; 216} 217EXPORT_SYMBOL(nd_region_to_nstype); 218 219static unsigned long long region_size(struct nd_region *nd_region) 220{ 221 if (is_memory(&nd_region->dev)) { 222 return nd_region->ndr_size; 223 } else if (nd_region->ndr_mappings == 1) { 224 struct nd_mapping *nd_mapping = &nd_region->mapping[0]; 225 226 return nd_mapping->size; 227 } 228 229 return 0; 230} 231 232static ssize_t size_show(struct device *dev, 233 struct device_attribute *attr, char *buf) 234{ 235 struct nd_region *nd_region = to_nd_region(dev); 236 237 return sprintf(buf, "%llu\n", region_size(nd_region)); 238} 239static DEVICE_ATTR_RO(size); 240 241static ssize_t deep_flush_show(struct device *dev, 242 struct device_attribute *attr, char *buf) 243{ 244 struct nd_region *nd_region = to_nd_region(dev); 245 246 /* 247 * NOTE: in the nvdimm_has_flush() error case this attribute is 248 * not visible. 249 */ 250 return sprintf(buf, "%d\n", nvdimm_has_flush(nd_region)); 251} 252 253static ssize_t deep_flush_store(struct device *dev, struct device_attribute *attr, 254 const char *buf, size_t len) 255{ 256 bool flush; 257 int rc = strtobool(buf, &flush); 258 struct nd_region *nd_region = to_nd_region(dev); 259 260 if (rc) 261 return rc; 262 if (!flush) 263 return -EINVAL; 264 rc = nvdimm_flush(nd_region, NULL); 265 if (rc) 266 return rc; 267 268 return len; 269} 270static DEVICE_ATTR_RW(deep_flush); 271 272static ssize_t mappings_show(struct device *dev, 273 struct device_attribute *attr, char *buf) 274{ 275 struct nd_region *nd_region = to_nd_region(dev); 276 277 return sprintf(buf, "%d\n", nd_region->ndr_mappings); 278} 279static DEVICE_ATTR_RO(mappings); 280 281static ssize_t nstype_show(struct device *dev, 282 struct device_attribute *attr, char *buf) 283{ 284 struct nd_region *nd_region = to_nd_region(dev); 285 286 return sprintf(buf, "%d\n", nd_region_to_nstype(nd_region)); 287} 288static DEVICE_ATTR_RO(nstype); 289 290static ssize_t set_cookie_show(struct device *dev, 291 struct device_attribute *attr, char *buf) 292{ 293 struct nd_region *nd_region = to_nd_region(dev); 294 struct nd_interleave_set *nd_set = nd_region->nd_set; 295 ssize_t rc = 0; 296 297 if (is_memory(dev) && nd_set) 298 /* pass, should be precluded by region_visible */; 299 else 300 return -ENXIO; 301 302 /* 303 * The cookie to show depends on which specification of the 304 * labels we are using. If there are not labels then default to 305 * the v1.1 namespace label cookie definition. To read all this 306 * data we need to wait for probing to settle. 307 */ 308 nd_device_lock(dev); 309 nvdimm_bus_lock(dev); 310 wait_nvdimm_bus_probe_idle(dev); 311 if (nd_region->ndr_mappings) { 312 struct nd_mapping *nd_mapping = &nd_region->mapping[0]; 313 struct nvdimm_drvdata *ndd = to_ndd(nd_mapping); 314 315 if (ndd) { 316 struct nd_namespace_index *nsindex; 317 318 nsindex = to_namespace_index(ndd, ndd->ns_current); 319 rc = sprintf(buf, "%#llx\n", 320 nd_region_interleave_set_cookie(nd_region, 321 nsindex)); 322 } 323 } 324 nvdimm_bus_unlock(dev); 325 nd_device_unlock(dev); 326 327 if (rc) 328 return rc; 329 return sprintf(buf, "%#llx\n", nd_set->cookie1); 330} 331static DEVICE_ATTR_RO(set_cookie); 332 333resource_size_t nd_region_available_dpa(struct nd_region *nd_region) 334{ 335 resource_size_t blk_max_overlap = 0, available, overlap; 336 int i; 337 338 WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev)); 339 340 retry: 341 available = 0; 342 overlap = blk_max_overlap; 343 for (i = 0; i < nd_region->ndr_mappings; i++) { 344 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 345 struct nvdimm_drvdata *ndd = to_ndd(nd_mapping); 346 347 /* if a dimm is disabled the available capacity is zero */ 348 if (!ndd) 349 return 0; 350 351 if (is_memory(&nd_region->dev)) { 352 available += nd_pmem_available_dpa(nd_region, 353 nd_mapping, &overlap); 354 if (overlap > blk_max_overlap) { 355 blk_max_overlap = overlap; 356 goto retry; 357 } 358 } else if (is_nd_blk(&nd_region->dev)) 359 available += nd_blk_available_dpa(nd_region); 360 } 361 362 return available; 363} 364 365resource_size_t nd_region_allocatable_dpa(struct nd_region *nd_region) 366{ 367 resource_size_t available = 0; 368 int i; 369 370 if (is_memory(&nd_region->dev)) 371 available = PHYS_ADDR_MAX; 372 373 WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev)); 374 for (i = 0; i < nd_region->ndr_mappings; i++) { 375 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 376 377 if (is_memory(&nd_region->dev)) 378 available = min(available, 379 nd_pmem_max_contiguous_dpa(nd_region, 380 nd_mapping)); 381 else if (is_nd_blk(&nd_region->dev)) 382 available += nd_blk_available_dpa(nd_region); 383 } 384 if (is_memory(&nd_region->dev)) 385 return available * nd_region->ndr_mappings; 386 return available; 387} 388 389static ssize_t available_size_show(struct device *dev, 390 struct device_attribute *attr, char *buf) 391{ 392 struct nd_region *nd_region = to_nd_region(dev); 393 unsigned long long available = 0; 394 395 /* 396 * Flush in-flight updates and grab a snapshot of the available 397 * size. Of course, this value is potentially invalidated the 398 * memory nvdimm_bus_lock() is dropped, but that's userspace's 399 * problem to not race itself. 400 */ 401 nd_device_lock(dev); 402 nvdimm_bus_lock(dev); 403 wait_nvdimm_bus_probe_idle(dev); 404 available = nd_region_available_dpa(nd_region); 405 nvdimm_bus_unlock(dev); 406 nd_device_unlock(dev); 407 408 return sprintf(buf, "%llu\n", available); 409} 410static DEVICE_ATTR_RO(available_size); 411 412static ssize_t max_available_extent_show(struct device *dev, 413 struct device_attribute *attr, char *buf) 414{ 415 struct nd_region *nd_region = to_nd_region(dev); 416 unsigned long long available = 0; 417 418 nd_device_lock(dev); 419 nvdimm_bus_lock(dev); 420 wait_nvdimm_bus_probe_idle(dev); 421 available = nd_region_allocatable_dpa(nd_region); 422 nvdimm_bus_unlock(dev); 423 nd_device_unlock(dev); 424 425 return sprintf(buf, "%llu\n", available); 426} 427static DEVICE_ATTR_RO(max_available_extent); 428 429static ssize_t init_namespaces_show(struct device *dev, 430 struct device_attribute *attr, char *buf) 431{ 432 struct nd_region_data *ndrd = dev_get_drvdata(dev); 433 ssize_t rc; 434 435 nvdimm_bus_lock(dev); 436 if (ndrd) 437 rc = sprintf(buf, "%d/%d\n", ndrd->ns_active, ndrd->ns_count); 438 else 439 rc = -ENXIO; 440 nvdimm_bus_unlock(dev); 441 442 return rc; 443} 444static DEVICE_ATTR_RO(init_namespaces); 445 446static ssize_t namespace_seed_show(struct device *dev, 447 struct device_attribute *attr, char *buf) 448{ 449 struct nd_region *nd_region = to_nd_region(dev); 450 ssize_t rc; 451 452 nvdimm_bus_lock(dev); 453 if (nd_region->ns_seed) 454 rc = sprintf(buf, "%s\n", dev_name(nd_region->ns_seed)); 455 else 456 rc = sprintf(buf, "\n"); 457 nvdimm_bus_unlock(dev); 458 return rc; 459} 460static DEVICE_ATTR_RO(namespace_seed); 461 462static ssize_t btt_seed_show(struct device *dev, 463 struct device_attribute *attr, char *buf) 464{ 465 struct nd_region *nd_region = to_nd_region(dev); 466 ssize_t rc; 467 468 nvdimm_bus_lock(dev); 469 if (nd_region->btt_seed) 470 rc = sprintf(buf, "%s\n", dev_name(nd_region->btt_seed)); 471 else 472 rc = sprintf(buf, "\n"); 473 nvdimm_bus_unlock(dev); 474 475 return rc; 476} 477static DEVICE_ATTR_RO(btt_seed); 478 479static ssize_t pfn_seed_show(struct device *dev, 480 struct device_attribute *attr, char *buf) 481{ 482 struct nd_region *nd_region = to_nd_region(dev); 483 ssize_t rc; 484 485 nvdimm_bus_lock(dev); 486 if (nd_region->pfn_seed) 487 rc = sprintf(buf, "%s\n", dev_name(nd_region->pfn_seed)); 488 else 489 rc = sprintf(buf, "\n"); 490 nvdimm_bus_unlock(dev); 491 492 return rc; 493} 494static DEVICE_ATTR_RO(pfn_seed); 495 496static ssize_t dax_seed_show(struct device *dev, 497 struct device_attribute *attr, char *buf) 498{ 499 struct nd_region *nd_region = to_nd_region(dev); 500 ssize_t rc; 501 502 nvdimm_bus_lock(dev); 503 if (nd_region->dax_seed) 504 rc = sprintf(buf, "%s\n", dev_name(nd_region->dax_seed)); 505 else 506 rc = sprintf(buf, "\n"); 507 nvdimm_bus_unlock(dev); 508 509 return rc; 510} 511static DEVICE_ATTR_RO(dax_seed); 512 513static ssize_t read_only_show(struct device *dev, 514 struct device_attribute *attr, char *buf) 515{ 516 struct nd_region *nd_region = to_nd_region(dev); 517 518 return sprintf(buf, "%d\n", nd_region->ro); 519} 520 521static ssize_t read_only_store(struct device *dev, 522 struct device_attribute *attr, const char *buf, size_t len) 523{ 524 bool ro; 525 int rc = strtobool(buf, &ro); 526 struct nd_region *nd_region = to_nd_region(dev); 527 528 if (rc) 529 return rc; 530 531 nd_region->ro = ro; 532 return len; 533} 534static DEVICE_ATTR_RW(read_only); 535 536static ssize_t align_show(struct device *dev, 537 struct device_attribute *attr, char *buf) 538{ 539 struct nd_region *nd_region = to_nd_region(dev); 540 541 return sprintf(buf, "%#lx\n", nd_region->align); 542} 543 544static ssize_t align_store(struct device *dev, 545 struct device_attribute *attr, const char *buf, size_t len) 546{ 547 struct nd_region *nd_region = to_nd_region(dev); 548 unsigned long val, dpa; 549 u32 remainder; 550 int rc; 551 552 rc = kstrtoul(buf, 0, &val); 553 if (rc) 554 return rc; 555 556 if (!nd_region->ndr_mappings) 557 return -ENXIO; 558 559 /* 560 * Ensure space-align is evenly divisible by the region 561 * interleave-width because the kernel typically has no facility 562 * to determine which DIMM(s), dimm-physical-addresses, would 563 * contribute to the tail capacity in system-physical-address 564 * space for the namespace. 565 */ 566 dpa = div_u64_rem(val, nd_region->ndr_mappings, &remainder); 567 if (!is_power_of_2(dpa) || dpa < PAGE_SIZE 568 || val > region_size(nd_region) || remainder) 569 return -EINVAL; 570 571 /* 572 * Given that space allocation consults this value multiple 573 * times ensure it does not change for the duration of the 574 * allocation. 575 */ 576 nvdimm_bus_lock(dev); 577 nd_region->align = val; 578 nvdimm_bus_unlock(dev); 579 580 return len; 581} 582static DEVICE_ATTR_RW(align); 583 584static ssize_t region_badblocks_show(struct device *dev, 585 struct device_attribute *attr, char *buf) 586{ 587 struct nd_region *nd_region = to_nd_region(dev); 588 ssize_t rc; 589 590 nd_device_lock(dev); 591 if (dev->driver) 592 rc = badblocks_show(&nd_region->bb, buf, 0); 593 else 594 rc = -ENXIO; 595 nd_device_unlock(dev); 596 597 return rc; 598} 599static DEVICE_ATTR(badblocks, 0444, region_badblocks_show, NULL); 600 601static ssize_t resource_show(struct device *dev, 602 struct device_attribute *attr, char *buf) 603{ 604 struct nd_region *nd_region = to_nd_region(dev); 605 606 return sprintf(buf, "%#llx\n", nd_region->ndr_start); 607} 608static DEVICE_ATTR_ADMIN_RO(resource); 609 610static ssize_t persistence_domain_show(struct device *dev, 611 struct device_attribute *attr, char *buf) 612{ 613 struct nd_region *nd_region = to_nd_region(dev); 614 615 if (test_bit(ND_REGION_PERSIST_CACHE, &nd_region->flags)) 616 return sprintf(buf, "cpu_cache\n"); 617 else if (test_bit(ND_REGION_PERSIST_MEMCTRL, &nd_region->flags)) 618 return sprintf(buf, "memory_controller\n"); 619 else 620 return sprintf(buf, "\n"); 621} 622static DEVICE_ATTR_RO(persistence_domain); 623 624static struct attribute *nd_region_attributes[] = { 625 &dev_attr_size.attr, 626 &dev_attr_align.attr, 627 &dev_attr_nstype.attr, 628 &dev_attr_mappings.attr, 629 &dev_attr_btt_seed.attr, 630 &dev_attr_pfn_seed.attr, 631 &dev_attr_dax_seed.attr, 632 &dev_attr_deep_flush.attr, 633 &dev_attr_read_only.attr, 634 &dev_attr_set_cookie.attr, 635 &dev_attr_available_size.attr, 636 &dev_attr_max_available_extent.attr, 637 &dev_attr_namespace_seed.attr, 638 &dev_attr_init_namespaces.attr, 639 &dev_attr_badblocks.attr, 640 &dev_attr_resource.attr, 641 &dev_attr_persistence_domain.attr, 642 NULL, 643}; 644 645static umode_t region_visible(struct kobject *kobj, struct attribute *a, int n) 646{ 647 struct device *dev = container_of(kobj, typeof(*dev), kobj); 648 struct nd_region *nd_region = to_nd_region(dev); 649 struct nd_interleave_set *nd_set = nd_region->nd_set; 650 int type = nd_region_to_nstype(nd_region); 651 652 if (!is_memory(dev) && a == &dev_attr_pfn_seed.attr) 653 return 0; 654 655 if (!is_memory(dev) && a == &dev_attr_dax_seed.attr) 656 return 0; 657 658 if (!is_memory(dev) && a == &dev_attr_badblocks.attr) 659 return 0; 660 661 if (a == &dev_attr_resource.attr && !is_memory(dev)) 662 return 0; 663 664 if (a == &dev_attr_deep_flush.attr) { 665 int has_flush = nvdimm_has_flush(nd_region); 666 667 if (has_flush == 1) 668 return a->mode; 669 else if (has_flush == 0) 670 return 0444; 671 else 672 return 0; 673 } 674 675 if (a == &dev_attr_persistence_domain.attr) { 676 if ((nd_region->flags & (BIT(ND_REGION_PERSIST_CACHE) 677 | BIT(ND_REGION_PERSIST_MEMCTRL))) == 0) 678 return 0; 679 return a->mode; 680 } 681 682 if (a == &dev_attr_align.attr) 683 return a->mode; 684 685 if (a != &dev_attr_set_cookie.attr 686 && a != &dev_attr_available_size.attr) 687 return a->mode; 688 689 if ((type == ND_DEVICE_NAMESPACE_PMEM 690 || type == ND_DEVICE_NAMESPACE_BLK) 691 && a == &dev_attr_available_size.attr) 692 return a->mode; 693 else if (is_memory(dev) && nd_set) 694 return a->mode; 695 696 return 0; 697} 698 699static ssize_t mappingN(struct device *dev, char *buf, int n) 700{ 701 struct nd_region *nd_region = to_nd_region(dev); 702 struct nd_mapping *nd_mapping; 703 struct nvdimm *nvdimm; 704 705 if (n >= nd_region->ndr_mappings) 706 return -ENXIO; 707 nd_mapping = &nd_region->mapping[n]; 708 nvdimm = nd_mapping->nvdimm; 709 710 return sprintf(buf, "%s,%llu,%llu,%d\n", dev_name(&nvdimm->dev), 711 nd_mapping->start, nd_mapping->size, 712 nd_mapping->position); 713} 714 715#define REGION_MAPPING(idx) \ 716static ssize_t mapping##idx##_show(struct device *dev, \ 717 struct device_attribute *attr, char *buf) \ 718{ \ 719 return mappingN(dev, buf, idx); \ 720} \ 721static DEVICE_ATTR_RO(mapping##idx) 722 723/* 724 * 32 should be enough for a while, even in the presence of socket 725 * interleave a 32-way interleave set is a degenerate case. 726 */ 727REGION_MAPPING(0); 728REGION_MAPPING(1); 729REGION_MAPPING(2); 730REGION_MAPPING(3); 731REGION_MAPPING(4); 732REGION_MAPPING(5); 733REGION_MAPPING(6); 734REGION_MAPPING(7); 735REGION_MAPPING(8); 736REGION_MAPPING(9); 737REGION_MAPPING(10); 738REGION_MAPPING(11); 739REGION_MAPPING(12); 740REGION_MAPPING(13); 741REGION_MAPPING(14); 742REGION_MAPPING(15); 743REGION_MAPPING(16); 744REGION_MAPPING(17); 745REGION_MAPPING(18); 746REGION_MAPPING(19); 747REGION_MAPPING(20); 748REGION_MAPPING(21); 749REGION_MAPPING(22); 750REGION_MAPPING(23); 751REGION_MAPPING(24); 752REGION_MAPPING(25); 753REGION_MAPPING(26); 754REGION_MAPPING(27); 755REGION_MAPPING(28); 756REGION_MAPPING(29); 757REGION_MAPPING(30); 758REGION_MAPPING(31); 759 760static umode_t mapping_visible(struct kobject *kobj, struct attribute *a, int n) 761{ 762 struct device *dev = container_of(kobj, struct device, kobj); 763 struct nd_region *nd_region = to_nd_region(dev); 764 765 if (n < nd_region->ndr_mappings) 766 return a->mode; 767 return 0; 768} 769 770static struct attribute *mapping_attributes[] = { 771 &dev_attr_mapping0.attr, 772 &dev_attr_mapping1.attr, 773 &dev_attr_mapping2.attr, 774 &dev_attr_mapping3.attr, 775 &dev_attr_mapping4.attr, 776 &dev_attr_mapping5.attr, 777 &dev_attr_mapping6.attr, 778 &dev_attr_mapping7.attr, 779 &dev_attr_mapping8.attr, 780 &dev_attr_mapping9.attr, 781 &dev_attr_mapping10.attr, 782 &dev_attr_mapping11.attr, 783 &dev_attr_mapping12.attr, 784 &dev_attr_mapping13.attr, 785 &dev_attr_mapping14.attr, 786 &dev_attr_mapping15.attr, 787 &dev_attr_mapping16.attr, 788 &dev_attr_mapping17.attr, 789 &dev_attr_mapping18.attr, 790 &dev_attr_mapping19.attr, 791 &dev_attr_mapping20.attr, 792 &dev_attr_mapping21.attr, 793 &dev_attr_mapping22.attr, 794 &dev_attr_mapping23.attr, 795 &dev_attr_mapping24.attr, 796 &dev_attr_mapping25.attr, 797 &dev_attr_mapping26.attr, 798 &dev_attr_mapping27.attr, 799 &dev_attr_mapping28.attr, 800 &dev_attr_mapping29.attr, 801 &dev_attr_mapping30.attr, 802 &dev_attr_mapping31.attr, 803 NULL, 804}; 805 806static const struct attribute_group nd_mapping_attribute_group = { 807 .is_visible = mapping_visible, 808 .attrs = mapping_attributes, 809}; 810 811static const struct attribute_group nd_region_attribute_group = { 812 .attrs = nd_region_attributes, 813 .is_visible = region_visible, 814}; 815 816static const struct attribute_group *nd_region_attribute_groups[] = { 817 &nd_device_attribute_group, 818 &nd_region_attribute_group, 819 &nd_numa_attribute_group, 820 &nd_mapping_attribute_group, 821 NULL, 822}; 823 824static const struct device_type nd_blk_device_type = { 825 .name = "nd_blk", 826 .release = nd_region_release, 827 .groups = nd_region_attribute_groups, 828}; 829 830static const struct device_type nd_pmem_device_type = { 831 .name = "nd_pmem", 832 .release = nd_region_release, 833 .groups = nd_region_attribute_groups, 834}; 835 836static const struct device_type nd_volatile_device_type = { 837 .name = "nd_volatile", 838 .release = nd_region_release, 839 .groups = nd_region_attribute_groups, 840}; 841 842bool is_nd_pmem(struct device *dev) 843{ 844 return dev ? dev->type == &nd_pmem_device_type : false; 845} 846 847bool is_nd_blk(struct device *dev) 848{ 849 return dev ? dev->type == &nd_blk_device_type : false; 850} 851 852bool is_nd_volatile(struct device *dev) 853{ 854 return dev ? dev->type == &nd_volatile_device_type : false; 855} 856 857u64 nd_region_interleave_set_cookie(struct nd_region *nd_region, 858 struct nd_namespace_index *nsindex) 859{ 860 struct nd_interleave_set *nd_set = nd_region->nd_set; 861 862 if (!nd_set) 863 return 0; 864 865 if (nsindex && __le16_to_cpu(nsindex->major) == 1 866 && __le16_to_cpu(nsindex->minor) == 1) 867 return nd_set->cookie1; 868 return nd_set->cookie2; 869} 870 871u64 nd_region_interleave_set_altcookie(struct nd_region *nd_region) 872{ 873 struct nd_interleave_set *nd_set = nd_region->nd_set; 874 875 if (nd_set) 876 return nd_set->altcookie; 877 return 0; 878} 879 880void nd_mapping_free_labels(struct nd_mapping *nd_mapping) 881{ 882 struct nd_label_ent *label_ent, *e; 883 884 lockdep_assert_held(&nd_mapping->lock); 885 list_for_each_entry_safe(label_ent, e, &nd_mapping->labels, list) { 886 list_del(&label_ent->list); 887 kfree(label_ent); 888 } 889} 890 891/* 892 * When a namespace is activated create new seeds for the next 893 * namespace, or namespace-personality to be configured. 894 */ 895void nd_region_advance_seeds(struct nd_region *nd_region, struct device *dev) 896{ 897 nvdimm_bus_lock(dev); 898 if (nd_region->ns_seed == dev) { 899 nd_region_create_ns_seed(nd_region); 900 } else if (is_nd_btt(dev)) { 901 struct nd_btt *nd_btt = to_nd_btt(dev); 902 903 if (nd_region->btt_seed == dev) 904 nd_region_create_btt_seed(nd_region); 905 if (nd_region->ns_seed == &nd_btt->ndns->dev) 906 nd_region_create_ns_seed(nd_region); 907 } else if (is_nd_pfn(dev)) { 908 struct nd_pfn *nd_pfn = to_nd_pfn(dev); 909 910 if (nd_region->pfn_seed == dev) 911 nd_region_create_pfn_seed(nd_region); 912 if (nd_region->ns_seed == &nd_pfn->ndns->dev) 913 nd_region_create_ns_seed(nd_region); 914 } else if (is_nd_dax(dev)) { 915 struct nd_dax *nd_dax = to_nd_dax(dev); 916 917 if (nd_region->dax_seed == dev) 918 nd_region_create_dax_seed(nd_region); 919 if (nd_region->ns_seed == &nd_dax->nd_pfn.ndns->dev) 920 nd_region_create_ns_seed(nd_region); 921 } 922 nvdimm_bus_unlock(dev); 923} 924 925int nd_blk_region_init(struct nd_region *nd_region) 926{ 927 struct device *dev = &nd_region->dev; 928 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(dev); 929 930 if (!is_nd_blk(dev)) 931 return 0; 932 933 if (nd_region->ndr_mappings < 1) { 934 dev_dbg(dev, "invalid BLK region\n"); 935 return -ENXIO; 936 } 937 938 return to_nd_blk_region(dev)->enable(nvdimm_bus, dev); 939} 940 941/** 942 * nd_region_acquire_lane - allocate and lock a lane 943 * @nd_region: region id and number of lanes possible 944 * 945 * A lane correlates to a BLK-data-window and/or a log slot in the BTT. 946 * We optimize for the common case where there are 256 lanes, one 947 * per-cpu. For larger systems we need to lock to share lanes. For now 948 * this implementation assumes the cost of maintaining an allocator for 949 * free lanes is on the order of the lock hold time, so it implements a 950 * static lane = cpu % num_lanes mapping. 951 * 952 * In the case of a BTT instance on top of a BLK namespace a lane may be 953 * acquired recursively. We lock on the first instance. 954 * 955 * In the case of a BTT instance on top of PMEM, we only acquire a lane 956 * for the BTT metadata updates. 957 */ 958unsigned int nd_region_acquire_lane(struct nd_region *nd_region) 959{ 960 unsigned int cpu, lane; 961 962 migrate_disable(); 963 cpu = smp_processor_id(); 964 if (nd_region->num_lanes < nr_cpu_ids) { 965 struct nd_percpu_lane *ndl_lock, *ndl_count; 966 967 lane = cpu % nd_region->num_lanes; 968 ndl_count = per_cpu_ptr(nd_region->lane, cpu); 969 ndl_lock = per_cpu_ptr(nd_region->lane, lane); 970 if (ndl_count->count++ == 0) 971 spin_lock(&ndl_lock->lock); 972 } else 973 lane = cpu; 974 975 return lane; 976} 977EXPORT_SYMBOL(nd_region_acquire_lane); 978 979void nd_region_release_lane(struct nd_region *nd_region, unsigned int lane) 980{ 981 if (nd_region->num_lanes < nr_cpu_ids) { 982 unsigned int cpu = smp_processor_id(); 983 struct nd_percpu_lane *ndl_lock, *ndl_count; 984 985 ndl_count = per_cpu_ptr(nd_region->lane, cpu); 986 ndl_lock = per_cpu_ptr(nd_region->lane, lane); 987 if (--ndl_count->count == 0) 988 spin_unlock(&ndl_lock->lock); 989 } 990 migrate_enable(); 991} 992EXPORT_SYMBOL(nd_region_release_lane); 993 994/* 995 * PowerPC requires this alignment for memremap_pages(). All other archs 996 * should be ok with SUBSECTION_SIZE (see memremap_compat_align()). 997 */ 998#define MEMREMAP_COMPAT_ALIGN_MAX SZ_16M 999 1000static unsigned long default_align(struct nd_region *nd_region) 1001{ 1002 unsigned long align; 1003 int i, mappings; 1004 u32 remainder; 1005 1006 if (is_nd_blk(&nd_region->dev)) 1007 align = PAGE_SIZE; 1008 else 1009 align = MEMREMAP_COMPAT_ALIGN_MAX; 1010 1011 for (i = 0; i < nd_region->ndr_mappings; i++) { 1012 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 1013 struct nvdimm *nvdimm = nd_mapping->nvdimm; 1014 1015 if (test_bit(NDD_ALIASING, &nvdimm->flags)) { 1016 align = MEMREMAP_COMPAT_ALIGN_MAX; 1017 break; 1018 } 1019 } 1020 1021 if (nd_region->ndr_size < MEMREMAP_COMPAT_ALIGN_MAX) 1022 align = PAGE_SIZE; 1023 1024 mappings = max_t(u16, 1, nd_region->ndr_mappings); 1025 div_u64_rem(align, mappings, &remainder); 1026 if (remainder) 1027 align *= mappings; 1028 1029 return align; 1030} 1031 1032static struct nd_region *nd_region_create(struct nvdimm_bus *nvdimm_bus, 1033 struct nd_region_desc *ndr_desc, 1034 const struct device_type *dev_type, const char *caller) 1035{ 1036 struct nd_region *nd_region; 1037 struct device *dev; 1038 void *region_buf; 1039 unsigned int i; 1040 int ro = 0; 1041 1042 for (i = 0; i < ndr_desc->num_mappings; i++) { 1043 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 1044 struct nvdimm *nvdimm = mapping->nvdimm; 1045 1046 if ((mapping->start | mapping->size) % PAGE_SIZE) { 1047 dev_err(&nvdimm_bus->dev, 1048 "%s: %s mapping%d is not %ld aligned\n", 1049 caller, dev_name(&nvdimm->dev), i, PAGE_SIZE); 1050 return NULL; 1051 } 1052 1053 if (test_bit(NDD_UNARMED, &nvdimm->flags)) 1054 ro = 1; 1055 1056 if (test_bit(NDD_NOBLK, &nvdimm->flags) 1057 && dev_type == &nd_blk_device_type) { 1058 dev_err(&nvdimm_bus->dev, "%s: %s mapping%d is not BLK capable\n", 1059 caller, dev_name(&nvdimm->dev), i); 1060 return NULL; 1061 } 1062 } 1063 1064 if (dev_type == &nd_blk_device_type) { 1065 struct nd_blk_region_desc *ndbr_desc; 1066 struct nd_blk_region *ndbr; 1067 1068 ndbr_desc = to_blk_region_desc(ndr_desc); 1069 ndbr = kzalloc(sizeof(*ndbr) + sizeof(struct nd_mapping) 1070 * ndr_desc->num_mappings, 1071 GFP_KERNEL); 1072 if (ndbr) { 1073 nd_region = &ndbr->nd_region; 1074 ndbr->enable = ndbr_desc->enable; 1075 ndbr->do_io = ndbr_desc->do_io; 1076 } 1077 region_buf = ndbr; 1078 } else { 1079 nd_region = kzalloc(struct_size(nd_region, mapping, 1080 ndr_desc->num_mappings), 1081 GFP_KERNEL); 1082 region_buf = nd_region; 1083 } 1084 1085 if (!region_buf) 1086 return NULL; 1087 nd_region->id = memregion_alloc(GFP_KERNEL); 1088 if (nd_region->id < 0) 1089 goto err_id; 1090 1091 nd_region->lane = alloc_percpu(struct nd_percpu_lane); 1092 if (!nd_region->lane) 1093 goto err_percpu; 1094 1095 for (i = 0; i < nr_cpu_ids; i++) { 1096 struct nd_percpu_lane *ndl; 1097 1098 ndl = per_cpu_ptr(nd_region->lane, i); 1099 spin_lock_init(&ndl->lock); 1100 ndl->count = 0; 1101 } 1102 1103 for (i = 0; i < ndr_desc->num_mappings; i++) { 1104 struct nd_mapping_desc *mapping = &ndr_desc->mapping[i]; 1105 struct nvdimm *nvdimm = mapping->nvdimm; 1106 1107 nd_region->mapping[i].nvdimm = nvdimm; 1108 nd_region->mapping[i].start = mapping->start; 1109 nd_region->mapping[i].size = mapping->size; 1110 nd_region->mapping[i].position = mapping->position; 1111 INIT_LIST_HEAD(&nd_region->mapping[i].labels); 1112 mutex_init(&nd_region->mapping[i].lock); 1113 1114 get_device(&nvdimm->dev); 1115 } 1116 nd_region->ndr_mappings = ndr_desc->num_mappings; 1117 nd_region->provider_data = ndr_desc->provider_data; 1118 nd_region->nd_set = ndr_desc->nd_set; 1119 nd_region->num_lanes = ndr_desc->num_lanes; 1120 nd_region->flags = ndr_desc->flags; 1121 nd_region->ro = ro; 1122 nd_region->numa_node = ndr_desc->numa_node; 1123 nd_region->target_node = ndr_desc->target_node; 1124 ida_init(&nd_region->ns_ida); 1125 ida_init(&nd_region->btt_ida); 1126 ida_init(&nd_region->pfn_ida); 1127 ida_init(&nd_region->dax_ida); 1128 dev = &nd_region->dev; 1129 dev_set_name(dev, "region%d", nd_region->id); 1130 dev->parent = &nvdimm_bus->dev; 1131 dev->type = dev_type; 1132 dev->groups = ndr_desc->attr_groups; 1133 dev->of_node = ndr_desc->of_node; 1134 nd_region->ndr_size = resource_size(ndr_desc->res); 1135 nd_region->ndr_start = ndr_desc->res->start; 1136 nd_region->align = default_align(nd_region); 1137 if (ndr_desc->flush) 1138 nd_region->flush = ndr_desc->flush; 1139 else 1140 nd_region->flush = NULL; 1141 1142 nd_device_register(dev); 1143 1144 return nd_region; 1145 1146 err_percpu: 1147 memregion_free(nd_region->id); 1148 err_id: 1149 kfree(region_buf); 1150 return NULL; 1151} 1152 1153struct nd_region *nvdimm_pmem_region_create(struct nvdimm_bus *nvdimm_bus, 1154 struct nd_region_desc *ndr_desc) 1155{ 1156 ndr_desc->num_lanes = ND_MAX_LANES; 1157 return nd_region_create(nvdimm_bus, ndr_desc, &nd_pmem_device_type, 1158 __func__); 1159} 1160EXPORT_SYMBOL_GPL(nvdimm_pmem_region_create); 1161 1162struct nd_region *nvdimm_blk_region_create(struct nvdimm_bus *nvdimm_bus, 1163 struct nd_region_desc *ndr_desc) 1164{ 1165 if (ndr_desc->num_mappings > 1) 1166 return NULL; 1167 ndr_desc->num_lanes = min(ndr_desc->num_lanes, ND_MAX_LANES); 1168 return nd_region_create(nvdimm_bus, ndr_desc, &nd_blk_device_type, 1169 __func__); 1170} 1171EXPORT_SYMBOL_GPL(nvdimm_blk_region_create); 1172 1173struct nd_region *nvdimm_volatile_region_create(struct nvdimm_bus *nvdimm_bus, 1174 struct nd_region_desc *ndr_desc) 1175{ 1176 ndr_desc->num_lanes = ND_MAX_LANES; 1177 return nd_region_create(nvdimm_bus, ndr_desc, &nd_volatile_device_type, 1178 __func__); 1179} 1180EXPORT_SYMBOL_GPL(nvdimm_volatile_region_create); 1181 1182int nvdimm_flush(struct nd_region *nd_region, struct bio *bio) 1183{ 1184 int rc = 0; 1185 1186 if (!nd_region->flush) 1187 rc = generic_nvdimm_flush(nd_region); 1188 else { 1189 if (nd_region->flush(nd_region, bio)) 1190 rc = -EIO; 1191 } 1192 1193 return rc; 1194} 1195/** 1196 * nvdimm_flush - flush any posted write queues between the cpu and pmem media 1197 * @nd_region: blk or interleaved pmem region 1198 */ 1199int generic_nvdimm_flush(struct nd_region *nd_region) 1200{ 1201 struct nd_region_data *ndrd = dev_get_drvdata(&nd_region->dev); 1202 int i, idx; 1203 1204 /* 1205 * Try to encourage some diversity in flush hint addresses 1206 * across cpus assuming a limited number of flush hints. 1207 */ 1208 idx = this_cpu_read(flush_idx); 1209 idx = this_cpu_add_return(flush_idx, hash_32(current->pid + idx, 8)); 1210 1211 /* 1212 * The pmem_wmb() is needed to 'sfence' all 1213 * previous writes such that they are architecturally visible for 1214 * the platform buffer flush. Note that we've already arranged for pmem 1215 * writes to avoid the cache via memcpy_flushcache(). The final 1216 * wmb() ensures ordering for the NVDIMM flush write. 1217 */ 1218 pmem_wmb(); 1219 for (i = 0; i < nd_region->ndr_mappings; i++) 1220 if (ndrd_get_flush_wpq(ndrd, i, 0)) 1221 writeq(1, ndrd_get_flush_wpq(ndrd, i, idx)); 1222 wmb(); 1223 1224 return 0; 1225} 1226EXPORT_SYMBOL_GPL(nvdimm_flush); 1227 1228/** 1229 * nvdimm_has_flush - determine write flushing requirements 1230 * @nd_region: blk or interleaved pmem region 1231 * 1232 * Returns 1 if writes require flushing 1233 * Returns 0 if writes do not require flushing 1234 * Returns -ENXIO if flushing capability can not be determined 1235 */ 1236int nvdimm_has_flush(struct nd_region *nd_region) 1237{ 1238 int i; 1239 1240 /* no nvdimm or pmem api == flushing capability unknown */ 1241 if (nd_region->ndr_mappings == 0 1242 || !IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API)) 1243 return -ENXIO; 1244 1245 /* Test if an explicit flush function is defined */ 1246 if (test_bit(ND_REGION_ASYNC, &nd_region->flags) && nd_region->flush) 1247 return 1; 1248 1249 /* Test if any flush hints for the region are available */ 1250 for (i = 0; i < nd_region->ndr_mappings; i++) { 1251 struct nd_mapping *nd_mapping = &nd_region->mapping[i]; 1252 struct nvdimm *nvdimm = nd_mapping->nvdimm; 1253 1254 /* flush hints present / available */ 1255 if (nvdimm->num_flush) 1256 return 1; 1257 } 1258 1259 /* 1260 * The platform defines dimm devices without hints nor explicit flush, 1261 * assume platform persistence mechanism like ADR 1262 */ 1263 return 0; 1264} 1265EXPORT_SYMBOL_GPL(nvdimm_has_flush); 1266 1267int nvdimm_has_cache(struct nd_region *nd_region) 1268{ 1269 return is_nd_pmem(&nd_region->dev) && 1270 !test_bit(ND_REGION_PERSIST_CACHE, &nd_region->flags); 1271} 1272EXPORT_SYMBOL_GPL(nvdimm_has_cache); 1273 1274bool is_nvdimm_sync(struct nd_region *nd_region) 1275{ 1276 if (is_nd_volatile(&nd_region->dev)) 1277 return true; 1278 1279 return is_nd_pmem(&nd_region->dev) && 1280 !test_bit(ND_REGION_ASYNC, &nd_region->flags); 1281} 1282EXPORT_SYMBOL_GPL(is_nvdimm_sync); 1283 1284struct conflict_context { 1285 struct nd_region *nd_region; 1286 resource_size_t start, size; 1287}; 1288 1289static int region_conflict(struct device *dev, void *data) 1290{ 1291 struct nd_region *nd_region; 1292 struct conflict_context *ctx = data; 1293 resource_size_t res_end, region_end, region_start; 1294 1295 if (!is_memory(dev)) 1296 return 0; 1297 1298 nd_region = to_nd_region(dev); 1299 if (nd_region == ctx->nd_region) 1300 return 0; 1301 1302 res_end = ctx->start + ctx->size; 1303 region_start = nd_region->ndr_start; 1304 region_end = region_start + nd_region->ndr_size; 1305 if (ctx->start >= region_start && ctx->start < region_end) 1306 return -EBUSY; 1307 if (res_end > region_start && res_end <= region_end) 1308 return -EBUSY; 1309 return 0; 1310} 1311 1312int nd_region_conflict(struct nd_region *nd_region, resource_size_t start, 1313 resource_size_t size) 1314{ 1315 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(&nd_region->dev); 1316 struct conflict_context ctx = { 1317 .nd_region = nd_region, 1318 .start = start, 1319 .size = size, 1320 }; 1321 1322 return device_for_each_child(&nvdimm_bus->dev, &ctx, region_conflict); 1323} 1324