1// SPDX-License-Identifier: GPL-2.0-only 2#include <linux/errno.h> 3#include <linux/numa.h> 4#include <linux/slab.h> 5#include <linux/rculist.h> 6#include <linux/threads.h> 7#include <linux/preempt.h> 8#include <linux/irqflags.h> 9#include <linux/vmalloc.h> 10#include <linux/mm.h> 11#include <linux/module.h> 12#include <linux/device-mapper.h> 13 14#include "dm-core.h" 15#include "dm-stats.h" 16 17#define DM_MSG_PREFIX "stats" 18 19static int dm_stat_need_rcu_barrier; 20 21/* 22 * Using 64-bit values to avoid overflow (which is a 23 * problem that block/genhd.c's IO accounting has). 24 */ 25struct dm_stat_percpu { 26 unsigned long long sectors[2]; 27 unsigned long long ios[2]; 28 unsigned long long merges[2]; 29 unsigned long long ticks[2]; 30 unsigned long long io_ticks[2]; 31 unsigned long long io_ticks_total; 32 unsigned long long time_in_queue; 33 unsigned long long *histogram; 34}; 35 36struct dm_stat_shared { 37 atomic_t in_flight[2]; 38 unsigned long long stamp; 39 struct dm_stat_percpu tmp; 40}; 41 42struct dm_stat { 43 struct list_head list_entry; 44 int id; 45 unsigned int stat_flags; 46 size_t n_entries; 47 sector_t start; 48 sector_t end; 49 sector_t step; 50 unsigned int n_histogram_entries; 51 unsigned long long *histogram_boundaries; 52 const char *program_id; 53 const char *aux_data; 54 struct rcu_head rcu_head; 55 size_t shared_alloc_size; 56 size_t percpu_alloc_size; 57 size_t histogram_alloc_size; 58 struct dm_stat_percpu *stat_percpu[NR_CPUS]; 59 struct dm_stat_shared stat_shared[]; 60}; 61 62#define STAT_PRECISE_TIMESTAMPS 1 63 64struct dm_stats_last_position { 65 sector_t last_sector; 66 unsigned int last_rw; 67}; 68 69/* 70 * A typo on the command line could possibly make the kernel run out of memory 71 * and crash. To prevent the crash we account all used memory. We fail if we 72 * exhaust 1/4 of all memory or 1/2 of vmalloc space. 73 */ 74#define DM_STATS_MEMORY_FACTOR 4 75#define DM_STATS_VMALLOC_FACTOR 2 76 77static DEFINE_SPINLOCK(shared_memory_lock); 78 79static unsigned long shared_memory_amount; 80 81static bool __check_shared_memory(size_t alloc_size) 82{ 83 size_t a; 84 85 a = shared_memory_amount + alloc_size; 86 if (a < shared_memory_amount) 87 return false; 88 if (a >> PAGE_SHIFT > totalram_pages() / DM_STATS_MEMORY_FACTOR) 89 return false; 90#ifdef CONFIG_MMU 91 if (a > (VMALLOC_END - VMALLOC_START) / DM_STATS_VMALLOC_FACTOR) 92 return false; 93#endif 94 return true; 95} 96 97static bool check_shared_memory(size_t alloc_size) 98{ 99 bool ret; 100 101 spin_lock_irq(&shared_memory_lock); 102 103 ret = __check_shared_memory(alloc_size); 104 105 spin_unlock_irq(&shared_memory_lock); 106 107 return ret; 108} 109 110static bool claim_shared_memory(size_t alloc_size) 111{ 112 spin_lock_irq(&shared_memory_lock); 113 114 if (!__check_shared_memory(alloc_size)) { 115 spin_unlock_irq(&shared_memory_lock); 116 return false; 117 } 118 119 shared_memory_amount += alloc_size; 120 121 spin_unlock_irq(&shared_memory_lock); 122 123 return true; 124} 125 126static void free_shared_memory(size_t alloc_size) 127{ 128 unsigned long flags; 129 130 spin_lock_irqsave(&shared_memory_lock, flags); 131 132 if (WARN_ON_ONCE(shared_memory_amount < alloc_size)) { 133 spin_unlock_irqrestore(&shared_memory_lock, flags); 134 DMCRIT("Memory usage accounting bug."); 135 return; 136 } 137 138 shared_memory_amount -= alloc_size; 139 140 spin_unlock_irqrestore(&shared_memory_lock, flags); 141} 142 143static void *dm_kvzalloc(size_t alloc_size, int node) 144{ 145 void *p; 146 147 if (!claim_shared_memory(alloc_size)) 148 return NULL; 149 150 p = kvzalloc_node(alloc_size, GFP_KERNEL | __GFP_NOMEMALLOC, node); 151 if (p) 152 return p; 153 154 free_shared_memory(alloc_size); 155 156 return NULL; 157} 158 159static void dm_kvfree(void *ptr, size_t alloc_size) 160{ 161 if (!ptr) 162 return; 163 164 free_shared_memory(alloc_size); 165 166 kvfree(ptr); 167} 168 169static void dm_stat_free(struct rcu_head *head) 170{ 171 int cpu; 172 struct dm_stat *s = container_of(head, struct dm_stat, rcu_head); 173 174 kfree(s->histogram_boundaries); 175 kfree(s->program_id); 176 kfree(s->aux_data); 177 for_each_possible_cpu(cpu) { 178 dm_kvfree(s->stat_percpu[cpu][0].histogram, s->histogram_alloc_size); 179 dm_kvfree(s->stat_percpu[cpu], s->percpu_alloc_size); 180 } 181 dm_kvfree(s->stat_shared[0].tmp.histogram, s->histogram_alloc_size); 182 dm_kvfree(s, s->shared_alloc_size); 183} 184 185static int dm_stat_in_flight(struct dm_stat_shared *shared) 186{ 187 return atomic_read(&shared->in_flight[READ]) + 188 atomic_read(&shared->in_flight[WRITE]); 189} 190 191int dm_stats_init(struct dm_stats *stats) 192{ 193 int cpu; 194 struct dm_stats_last_position *last; 195 196 mutex_init(&stats->mutex); 197 INIT_LIST_HEAD(&stats->list); 198 stats->precise_timestamps = false; 199 stats->last = alloc_percpu(struct dm_stats_last_position); 200 if (!stats->last) 201 return -ENOMEM; 202 203 for_each_possible_cpu(cpu) { 204 last = per_cpu_ptr(stats->last, cpu); 205 last->last_sector = (sector_t)ULLONG_MAX; 206 last->last_rw = UINT_MAX; 207 } 208 209 return 0; 210} 211 212void dm_stats_cleanup(struct dm_stats *stats) 213{ 214 size_t ni; 215 struct dm_stat *s; 216 struct dm_stat_shared *shared; 217 218 while (!list_empty(&stats->list)) { 219 s = container_of(stats->list.next, struct dm_stat, list_entry); 220 list_del(&s->list_entry); 221 for (ni = 0; ni < s->n_entries; ni++) { 222 shared = &s->stat_shared[ni]; 223 if (WARN_ON(dm_stat_in_flight(shared))) { 224 DMCRIT("leaked in-flight counter at index %lu " 225 "(start %llu, end %llu, step %llu): reads %d, writes %d", 226 (unsigned long)ni, 227 (unsigned long long)s->start, 228 (unsigned long long)s->end, 229 (unsigned long long)s->step, 230 atomic_read(&shared->in_flight[READ]), 231 atomic_read(&shared->in_flight[WRITE])); 232 } 233 cond_resched(); 234 } 235 dm_stat_free(&s->rcu_head); 236 } 237 free_percpu(stats->last); 238 mutex_destroy(&stats->mutex); 239} 240 241static void dm_stats_recalc_precise_timestamps(struct dm_stats *stats) 242{ 243 struct list_head *l; 244 struct dm_stat *tmp_s; 245 bool precise_timestamps = false; 246 247 list_for_each(l, &stats->list) { 248 tmp_s = container_of(l, struct dm_stat, list_entry); 249 if (tmp_s->stat_flags & STAT_PRECISE_TIMESTAMPS) { 250 precise_timestamps = true; 251 break; 252 } 253 } 254 stats->precise_timestamps = precise_timestamps; 255} 256 257static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, 258 sector_t step, unsigned int stat_flags, 259 unsigned int n_histogram_entries, 260 unsigned long long *histogram_boundaries, 261 const char *program_id, const char *aux_data, 262 void (*suspend_callback)(struct mapped_device *), 263 void (*resume_callback)(struct mapped_device *), 264 struct mapped_device *md) 265{ 266 struct list_head *l; 267 struct dm_stat *s, *tmp_s; 268 sector_t n_entries; 269 size_t ni; 270 size_t shared_alloc_size; 271 size_t percpu_alloc_size; 272 size_t histogram_alloc_size; 273 struct dm_stat_percpu *p; 274 int cpu; 275 int ret_id; 276 int r; 277 278 if (end < start || !step) 279 return -EINVAL; 280 281 n_entries = end - start; 282 if (dm_sector_div64(n_entries, step)) 283 n_entries++; 284 285 if (n_entries != (size_t)n_entries || !(size_t)(n_entries + 1)) 286 return -EOVERFLOW; 287 288 shared_alloc_size = struct_size(s, stat_shared, n_entries); 289 if ((shared_alloc_size - sizeof(struct dm_stat)) / sizeof(struct dm_stat_shared) != n_entries) 290 return -EOVERFLOW; 291 292 percpu_alloc_size = (size_t)n_entries * sizeof(struct dm_stat_percpu); 293 if (percpu_alloc_size / sizeof(struct dm_stat_percpu) != n_entries) 294 return -EOVERFLOW; 295 296 histogram_alloc_size = (n_histogram_entries + 1) * (size_t)n_entries * sizeof(unsigned long long); 297 if (histogram_alloc_size / (n_histogram_entries + 1) != (size_t)n_entries * sizeof(unsigned long long)) 298 return -EOVERFLOW; 299 300 if (!check_shared_memory(shared_alloc_size + histogram_alloc_size + 301 num_possible_cpus() * (percpu_alloc_size + histogram_alloc_size))) 302 return -ENOMEM; 303 304 s = dm_kvzalloc(shared_alloc_size, NUMA_NO_NODE); 305 if (!s) 306 return -ENOMEM; 307 308 s->stat_flags = stat_flags; 309 s->n_entries = n_entries; 310 s->start = start; 311 s->end = end; 312 s->step = step; 313 s->shared_alloc_size = shared_alloc_size; 314 s->percpu_alloc_size = percpu_alloc_size; 315 s->histogram_alloc_size = histogram_alloc_size; 316 317 s->n_histogram_entries = n_histogram_entries; 318 s->histogram_boundaries = kmemdup(histogram_boundaries, 319 s->n_histogram_entries * sizeof(unsigned long long), GFP_KERNEL); 320 if (!s->histogram_boundaries) { 321 r = -ENOMEM; 322 goto out; 323 } 324 325 s->program_id = kstrdup(program_id, GFP_KERNEL); 326 if (!s->program_id) { 327 r = -ENOMEM; 328 goto out; 329 } 330 s->aux_data = kstrdup(aux_data, GFP_KERNEL); 331 if (!s->aux_data) { 332 r = -ENOMEM; 333 goto out; 334 } 335 336 for (ni = 0; ni < n_entries; ni++) { 337 atomic_set(&s->stat_shared[ni].in_flight[READ], 0); 338 atomic_set(&s->stat_shared[ni].in_flight[WRITE], 0); 339 cond_resched(); 340 } 341 342 if (s->n_histogram_entries) { 343 unsigned long long *hi; 344 345 hi = dm_kvzalloc(s->histogram_alloc_size, NUMA_NO_NODE); 346 if (!hi) { 347 r = -ENOMEM; 348 goto out; 349 } 350 for (ni = 0; ni < n_entries; ni++) { 351 s->stat_shared[ni].tmp.histogram = hi; 352 hi += s->n_histogram_entries + 1; 353 cond_resched(); 354 } 355 } 356 357 for_each_possible_cpu(cpu) { 358 p = dm_kvzalloc(percpu_alloc_size, cpu_to_node(cpu)); 359 if (!p) { 360 r = -ENOMEM; 361 goto out; 362 } 363 s->stat_percpu[cpu] = p; 364 if (s->n_histogram_entries) { 365 unsigned long long *hi; 366 367 hi = dm_kvzalloc(s->histogram_alloc_size, cpu_to_node(cpu)); 368 if (!hi) { 369 r = -ENOMEM; 370 goto out; 371 } 372 for (ni = 0; ni < n_entries; ni++) { 373 p[ni].histogram = hi; 374 hi += s->n_histogram_entries + 1; 375 cond_resched(); 376 } 377 } 378 } 379 380 /* 381 * Suspend/resume to make sure there is no i/o in flight, 382 * so that newly created statistics will be exact. 383 * 384 * (note: we couldn't suspend earlier because we must not 385 * allocate memory while suspended) 386 */ 387 suspend_callback(md); 388 389 mutex_lock(&stats->mutex); 390 s->id = 0; 391 list_for_each(l, &stats->list) { 392 tmp_s = container_of(l, struct dm_stat, list_entry); 393 if (WARN_ON(tmp_s->id < s->id)) { 394 r = -EINVAL; 395 goto out_unlock_resume; 396 } 397 if (tmp_s->id > s->id) 398 break; 399 if (unlikely(s->id == INT_MAX)) { 400 r = -ENFILE; 401 goto out_unlock_resume; 402 } 403 s->id++; 404 } 405 ret_id = s->id; 406 list_add_tail_rcu(&s->list_entry, l); 407 408 dm_stats_recalc_precise_timestamps(stats); 409 410 if (!static_key_enabled(&stats_enabled.key)) 411 static_branch_enable(&stats_enabled); 412 413 mutex_unlock(&stats->mutex); 414 415 resume_callback(md); 416 417 return ret_id; 418 419out_unlock_resume: 420 mutex_unlock(&stats->mutex); 421 resume_callback(md); 422out: 423 dm_stat_free(&s->rcu_head); 424 return r; 425} 426 427static struct dm_stat *__dm_stats_find(struct dm_stats *stats, int id) 428{ 429 struct dm_stat *s; 430 431 list_for_each_entry(s, &stats->list, list_entry) { 432 if (s->id > id) 433 break; 434 if (s->id == id) 435 return s; 436 } 437 438 return NULL; 439} 440 441static int dm_stats_delete(struct dm_stats *stats, int id) 442{ 443 struct dm_stat *s; 444 int cpu; 445 446 mutex_lock(&stats->mutex); 447 448 s = __dm_stats_find(stats, id); 449 if (!s) { 450 mutex_unlock(&stats->mutex); 451 return -ENOENT; 452 } 453 454 list_del_rcu(&s->list_entry); 455 456 dm_stats_recalc_precise_timestamps(stats); 457 458 mutex_unlock(&stats->mutex); 459 460 /* 461 * vfree can't be called from RCU callback 462 */ 463 for_each_possible_cpu(cpu) 464 if (is_vmalloc_addr(s->stat_percpu) || 465 is_vmalloc_addr(s->stat_percpu[cpu][0].histogram)) 466 goto do_sync_free; 467 if (is_vmalloc_addr(s) || 468 is_vmalloc_addr(s->stat_shared[0].tmp.histogram)) { 469do_sync_free: 470 synchronize_rcu_expedited(); 471 dm_stat_free(&s->rcu_head); 472 } else { 473 WRITE_ONCE(dm_stat_need_rcu_barrier, 1); 474 call_rcu(&s->rcu_head, dm_stat_free); 475 } 476 return 0; 477} 478 479static int dm_stats_list(struct dm_stats *stats, const char *program, 480 char *result, unsigned int maxlen) 481{ 482 struct dm_stat *s; 483 sector_t len; 484 unsigned int sz = 0; 485 486 /* 487 * Output format: 488 * <region_id>: <start_sector>+<length> <step> <program_id> <aux_data> 489 */ 490 491 mutex_lock(&stats->mutex); 492 list_for_each_entry(s, &stats->list, list_entry) { 493 if (!program || !strcmp(program, s->program_id)) { 494 len = s->end - s->start; 495 DMEMIT("%d: %llu+%llu %llu %s %s", s->id, 496 (unsigned long long)s->start, 497 (unsigned long long)len, 498 (unsigned long long)s->step, 499 s->program_id, 500 s->aux_data); 501 if (s->stat_flags & STAT_PRECISE_TIMESTAMPS) 502 DMEMIT(" precise_timestamps"); 503 if (s->n_histogram_entries) { 504 unsigned int i; 505 506 DMEMIT(" histogram:"); 507 for (i = 0; i < s->n_histogram_entries; i++) { 508 if (i) 509 DMEMIT(","); 510 DMEMIT("%llu", s->histogram_boundaries[i]); 511 } 512 } 513 DMEMIT("\n"); 514 } 515 cond_resched(); 516 } 517 mutex_unlock(&stats->mutex); 518 519 return 1; 520} 521 522static void dm_stat_round(struct dm_stat *s, struct dm_stat_shared *shared, 523 struct dm_stat_percpu *p) 524{ 525 /* 526 * This is racy, but so is part_round_stats_single. 527 */ 528 unsigned long long now, difference; 529 unsigned int in_flight_read, in_flight_write; 530 531 if (likely(!(s->stat_flags & STAT_PRECISE_TIMESTAMPS))) 532 now = jiffies; 533 else 534 now = ktime_to_ns(ktime_get()); 535 536 difference = now - shared->stamp; 537 if (!difference) 538 return; 539 540 in_flight_read = (unsigned int)atomic_read(&shared->in_flight[READ]); 541 in_flight_write = (unsigned int)atomic_read(&shared->in_flight[WRITE]); 542 if (in_flight_read) 543 p->io_ticks[READ] += difference; 544 if (in_flight_write) 545 p->io_ticks[WRITE] += difference; 546 if (in_flight_read + in_flight_write) { 547 p->io_ticks_total += difference; 548 p->time_in_queue += (in_flight_read + in_flight_write) * difference; 549 } 550 shared->stamp = now; 551} 552 553static void dm_stat_for_entry(struct dm_stat *s, size_t entry, 554 int idx, sector_t len, 555 struct dm_stats_aux *stats_aux, bool end, 556 unsigned long duration_jiffies) 557{ 558 struct dm_stat_shared *shared = &s->stat_shared[entry]; 559 struct dm_stat_percpu *p; 560 561 /* 562 * For strict correctness we should use local_irq_save/restore 563 * instead of preempt_disable/enable. 564 * 565 * preempt_disable/enable is racy if the driver finishes bios 566 * from non-interrupt context as well as from interrupt context 567 * or from more different interrupts. 568 * 569 * On 64-bit architectures the race only results in not counting some 570 * events, so it is acceptable. On 32-bit architectures the race could 571 * cause the counter going off by 2^32, so we need to do proper locking 572 * there. 573 * 574 * part_stat_lock()/part_stat_unlock() have this race too. 575 */ 576#if BITS_PER_LONG == 32 577 unsigned long flags; 578 579 local_irq_save(flags); 580#else 581 preempt_disable(); 582#endif 583 p = &s->stat_percpu[smp_processor_id()][entry]; 584 585 if (!end) { 586 dm_stat_round(s, shared, p); 587 atomic_inc(&shared->in_flight[idx]); 588 } else { 589 unsigned long long duration; 590 591 dm_stat_round(s, shared, p); 592 atomic_dec(&shared->in_flight[idx]); 593 p->sectors[idx] += len; 594 p->ios[idx] += 1; 595 p->merges[idx] += stats_aux->merged; 596 if (!(s->stat_flags & STAT_PRECISE_TIMESTAMPS)) { 597 p->ticks[idx] += duration_jiffies; 598 duration = jiffies_to_msecs(duration_jiffies); 599 } else { 600 p->ticks[idx] += stats_aux->duration_ns; 601 duration = stats_aux->duration_ns; 602 } 603 if (s->n_histogram_entries) { 604 unsigned int lo = 0, hi = s->n_histogram_entries + 1; 605 606 while (lo + 1 < hi) { 607 unsigned int mid = (lo + hi) / 2; 608 609 if (s->histogram_boundaries[mid - 1] > duration) 610 hi = mid; 611 else 612 lo = mid; 613 } 614 p->histogram[lo]++; 615 } 616 } 617 618#if BITS_PER_LONG == 32 619 local_irq_restore(flags); 620#else 621 preempt_enable(); 622#endif 623} 624 625static void __dm_stat_bio(struct dm_stat *s, int bi_rw, 626 sector_t bi_sector, sector_t end_sector, 627 bool end, unsigned long duration_jiffies, 628 struct dm_stats_aux *stats_aux) 629{ 630 sector_t rel_sector, offset, todo, fragment_len; 631 size_t entry; 632 633 if (end_sector <= s->start || bi_sector >= s->end) 634 return; 635 if (unlikely(bi_sector < s->start)) { 636 rel_sector = 0; 637 todo = end_sector - s->start; 638 } else { 639 rel_sector = bi_sector - s->start; 640 todo = end_sector - bi_sector; 641 } 642 if (unlikely(end_sector > s->end)) 643 todo -= (end_sector - s->end); 644 645 offset = dm_sector_div64(rel_sector, s->step); 646 entry = rel_sector; 647 do { 648 if (WARN_ON_ONCE(entry >= s->n_entries)) { 649 DMCRIT("Invalid area access in region id %d", s->id); 650 return; 651 } 652 fragment_len = todo; 653 if (fragment_len > s->step - offset) 654 fragment_len = s->step - offset; 655 dm_stat_for_entry(s, entry, bi_rw, fragment_len, 656 stats_aux, end, duration_jiffies); 657 todo -= fragment_len; 658 entry++; 659 offset = 0; 660 } while (unlikely(todo != 0)); 661} 662 663void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw, 664 sector_t bi_sector, unsigned int bi_sectors, bool end, 665 unsigned long start_time, 666 struct dm_stats_aux *stats_aux) 667{ 668 struct dm_stat *s; 669 sector_t end_sector; 670 struct dm_stats_last_position *last; 671 bool got_precise_time; 672 unsigned long duration_jiffies = 0; 673 674 if (unlikely(!bi_sectors)) 675 return; 676 677 end_sector = bi_sector + bi_sectors; 678 679 if (!end) { 680 /* 681 * A race condition can at worst result in the merged flag being 682 * misrepresented, so we don't have to disable preemption here. 683 */ 684 last = raw_cpu_ptr(stats->last); 685 stats_aux->merged = 686 (bi_sector == (READ_ONCE(last->last_sector) && 687 ((bi_rw == WRITE) == 688 (READ_ONCE(last->last_rw) == WRITE)) 689 )); 690 WRITE_ONCE(last->last_sector, end_sector); 691 WRITE_ONCE(last->last_rw, bi_rw); 692 } else 693 duration_jiffies = jiffies - start_time; 694 695 rcu_read_lock(); 696 697 got_precise_time = false; 698 list_for_each_entry_rcu(s, &stats->list, list_entry) { 699 if (s->stat_flags & STAT_PRECISE_TIMESTAMPS && !got_precise_time) { 700 /* start (!end) duration_ns is set by DM core's alloc_io() */ 701 if (end) 702 stats_aux->duration_ns = ktime_to_ns(ktime_get()) - stats_aux->duration_ns; 703 got_precise_time = true; 704 } 705 __dm_stat_bio(s, bi_rw, bi_sector, end_sector, end, duration_jiffies, stats_aux); 706 } 707 708 rcu_read_unlock(); 709} 710 711static void __dm_stat_init_temporary_percpu_totals(struct dm_stat_shared *shared, 712 struct dm_stat *s, size_t x) 713{ 714 int cpu; 715 struct dm_stat_percpu *p; 716 717 local_irq_disable(); 718 p = &s->stat_percpu[smp_processor_id()][x]; 719 dm_stat_round(s, shared, p); 720 local_irq_enable(); 721 722 shared->tmp.sectors[READ] = 0; 723 shared->tmp.sectors[WRITE] = 0; 724 shared->tmp.ios[READ] = 0; 725 shared->tmp.ios[WRITE] = 0; 726 shared->tmp.merges[READ] = 0; 727 shared->tmp.merges[WRITE] = 0; 728 shared->tmp.ticks[READ] = 0; 729 shared->tmp.ticks[WRITE] = 0; 730 shared->tmp.io_ticks[READ] = 0; 731 shared->tmp.io_ticks[WRITE] = 0; 732 shared->tmp.io_ticks_total = 0; 733 shared->tmp.time_in_queue = 0; 734 735 if (s->n_histogram_entries) 736 memset(shared->tmp.histogram, 0, (s->n_histogram_entries + 1) * sizeof(unsigned long long)); 737 738 for_each_possible_cpu(cpu) { 739 p = &s->stat_percpu[cpu][x]; 740 shared->tmp.sectors[READ] += READ_ONCE(p->sectors[READ]); 741 shared->tmp.sectors[WRITE] += READ_ONCE(p->sectors[WRITE]); 742 shared->tmp.ios[READ] += READ_ONCE(p->ios[READ]); 743 shared->tmp.ios[WRITE] += READ_ONCE(p->ios[WRITE]); 744 shared->tmp.merges[READ] += READ_ONCE(p->merges[READ]); 745 shared->tmp.merges[WRITE] += READ_ONCE(p->merges[WRITE]); 746 shared->tmp.ticks[READ] += READ_ONCE(p->ticks[READ]); 747 shared->tmp.ticks[WRITE] += READ_ONCE(p->ticks[WRITE]); 748 shared->tmp.io_ticks[READ] += READ_ONCE(p->io_ticks[READ]); 749 shared->tmp.io_ticks[WRITE] += READ_ONCE(p->io_ticks[WRITE]); 750 shared->tmp.io_ticks_total += READ_ONCE(p->io_ticks_total); 751 shared->tmp.time_in_queue += READ_ONCE(p->time_in_queue); 752 if (s->n_histogram_entries) { 753 unsigned int i; 754 755 for (i = 0; i < s->n_histogram_entries + 1; i++) 756 shared->tmp.histogram[i] += READ_ONCE(p->histogram[i]); 757 } 758 } 759} 760 761static void __dm_stat_clear(struct dm_stat *s, size_t idx_start, size_t idx_end, 762 bool init_tmp_percpu_totals) 763{ 764 size_t x; 765 struct dm_stat_shared *shared; 766 struct dm_stat_percpu *p; 767 768 for (x = idx_start; x < idx_end; x++) { 769 shared = &s->stat_shared[x]; 770 if (init_tmp_percpu_totals) 771 __dm_stat_init_temporary_percpu_totals(shared, s, x); 772 local_irq_disable(); 773 p = &s->stat_percpu[smp_processor_id()][x]; 774 p->sectors[READ] -= shared->tmp.sectors[READ]; 775 p->sectors[WRITE] -= shared->tmp.sectors[WRITE]; 776 p->ios[READ] -= shared->tmp.ios[READ]; 777 p->ios[WRITE] -= shared->tmp.ios[WRITE]; 778 p->merges[READ] -= shared->tmp.merges[READ]; 779 p->merges[WRITE] -= shared->tmp.merges[WRITE]; 780 p->ticks[READ] -= shared->tmp.ticks[READ]; 781 p->ticks[WRITE] -= shared->tmp.ticks[WRITE]; 782 p->io_ticks[READ] -= shared->tmp.io_ticks[READ]; 783 p->io_ticks[WRITE] -= shared->tmp.io_ticks[WRITE]; 784 p->io_ticks_total -= shared->tmp.io_ticks_total; 785 p->time_in_queue -= shared->tmp.time_in_queue; 786 local_irq_enable(); 787 if (s->n_histogram_entries) { 788 unsigned int i; 789 790 for (i = 0; i < s->n_histogram_entries + 1; i++) { 791 local_irq_disable(); 792 p = &s->stat_percpu[smp_processor_id()][x]; 793 p->histogram[i] -= shared->tmp.histogram[i]; 794 local_irq_enable(); 795 } 796 } 797 cond_resched(); 798 } 799} 800 801static int dm_stats_clear(struct dm_stats *stats, int id) 802{ 803 struct dm_stat *s; 804 805 mutex_lock(&stats->mutex); 806 807 s = __dm_stats_find(stats, id); 808 if (!s) { 809 mutex_unlock(&stats->mutex); 810 return -ENOENT; 811 } 812 813 __dm_stat_clear(s, 0, s->n_entries, true); 814 815 mutex_unlock(&stats->mutex); 816 817 return 1; 818} 819 820/* 821 * This is like jiffies_to_msec, but works for 64-bit values. 822 */ 823static unsigned long long dm_jiffies_to_msec64(struct dm_stat *s, unsigned long long j) 824{ 825 unsigned long long result; 826 unsigned int mult; 827 828 if (s->stat_flags & STAT_PRECISE_TIMESTAMPS) 829 return j; 830 831 result = 0; 832 if (j) 833 result = jiffies_to_msecs(j & 0x3fffff); 834 if (j >= 1 << 22) { 835 mult = jiffies_to_msecs(1 << 22); 836 result += (unsigned long long)mult * (unsigned long long)jiffies_to_msecs((j >> 22) & 0x3fffff); 837 } 838 if (j >= 1ULL << 44) 839 result += (unsigned long long)mult * (unsigned long long)mult * (unsigned long long)jiffies_to_msecs(j >> 44); 840 841 return result; 842} 843 844static int dm_stats_print(struct dm_stats *stats, int id, 845 size_t idx_start, size_t idx_len, 846 bool clear, char *result, unsigned int maxlen) 847{ 848 unsigned int sz = 0; 849 struct dm_stat *s; 850 size_t x; 851 sector_t start, end, step; 852 size_t idx_end; 853 struct dm_stat_shared *shared; 854 855 /* 856 * Output format: 857 * <start_sector>+<length> counters 858 */ 859 860 mutex_lock(&stats->mutex); 861 862 s = __dm_stats_find(stats, id); 863 if (!s) { 864 mutex_unlock(&stats->mutex); 865 return -ENOENT; 866 } 867 868 idx_end = idx_start + idx_len; 869 if (idx_end < idx_start || 870 idx_end > s->n_entries) 871 idx_end = s->n_entries; 872 873 if (idx_start > idx_end) 874 idx_start = idx_end; 875 876 step = s->step; 877 start = s->start + (step * idx_start); 878 879 for (x = idx_start; x < idx_end; x++, start = end) { 880 shared = &s->stat_shared[x]; 881 end = start + step; 882 if (unlikely(end > s->end)) 883 end = s->end; 884 885 __dm_stat_init_temporary_percpu_totals(shared, s, x); 886 887 DMEMIT("%llu+%llu %llu %llu %llu %llu %llu %llu %llu %llu %d %llu %llu %llu %llu", 888 (unsigned long long)start, 889 (unsigned long long)step, 890 shared->tmp.ios[READ], 891 shared->tmp.merges[READ], 892 shared->tmp.sectors[READ], 893 dm_jiffies_to_msec64(s, shared->tmp.ticks[READ]), 894 shared->tmp.ios[WRITE], 895 shared->tmp.merges[WRITE], 896 shared->tmp.sectors[WRITE], 897 dm_jiffies_to_msec64(s, shared->tmp.ticks[WRITE]), 898 dm_stat_in_flight(shared), 899 dm_jiffies_to_msec64(s, shared->tmp.io_ticks_total), 900 dm_jiffies_to_msec64(s, shared->tmp.time_in_queue), 901 dm_jiffies_to_msec64(s, shared->tmp.io_ticks[READ]), 902 dm_jiffies_to_msec64(s, shared->tmp.io_ticks[WRITE])); 903 if (s->n_histogram_entries) { 904 unsigned int i; 905 906 for (i = 0; i < s->n_histogram_entries + 1; i++) 907 DMEMIT("%s%llu", !i ? " " : ":", shared->tmp.histogram[i]); 908 } 909 DMEMIT("\n"); 910 911 if (unlikely(sz + 1 >= maxlen)) 912 goto buffer_overflow; 913 914 cond_resched(); 915 } 916 917 if (clear) 918 __dm_stat_clear(s, idx_start, idx_end, false); 919 920buffer_overflow: 921 mutex_unlock(&stats->mutex); 922 923 return 1; 924} 925 926static int dm_stats_set_aux(struct dm_stats *stats, int id, const char *aux_data) 927{ 928 struct dm_stat *s; 929 const char *new_aux_data; 930 931 mutex_lock(&stats->mutex); 932 933 s = __dm_stats_find(stats, id); 934 if (!s) { 935 mutex_unlock(&stats->mutex); 936 return -ENOENT; 937 } 938 939 new_aux_data = kstrdup(aux_data, GFP_KERNEL); 940 if (!new_aux_data) { 941 mutex_unlock(&stats->mutex); 942 return -ENOMEM; 943 } 944 945 kfree(s->aux_data); 946 s->aux_data = new_aux_data; 947 948 mutex_unlock(&stats->mutex); 949 950 return 0; 951} 952 953static int parse_histogram(const char *h, unsigned int *n_histogram_entries, 954 unsigned long long **histogram_boundaries) 955{ 956 const char *q; 957 unsigned int n; 958 unsigned long long last; 959 960 *n_histogram_entries = 1; 961 for (q = h; *q; q++) 962 if (*q == ',') 963 (*n_histogram_entries)++; 964 965 *histogram_boundaries = kmalloc_array(*n_histogram_entries, 966 sizeof(unsigned long long), 967 GFP_KERNEL); 968 if (!*histogram_boundaries) 969 return -ENOMEM; 970 971 n = 0; 972 last = 0; 973 while (1) { 974 unsigned long long hi; 975 int s; 976 char ch; 977 978 s = sscanf(h, "%llu%c", &hi, &ch); 979 if (!s || (s == 2 && ch != ',')) 980 return -EINVAL; 981 if (hi <= last) 982 return -EINVAL; 983 last = hi; 984 (*histogram_boundaries)[n] = hi; 985 if (s == 1) 986 return 0; 987 h = strchr(h, ',') + 1; 988 n++; 989 } 990} 991 992static int message_stats_create(struct mapped_device *md, 993 unsigned int argc, char **argv, 994 char *result, unsigned int maxlen) 995{ 996 int r; 997 int id; 998 char dummy; 999 unsigned long long start, end, len, step; 1000 unsigned int divisor; 1001 const char *program_id, *aux_data; 1002 unsigned int stat_flags = 0; 1003 unsigned int n_histogram_entries = 0; 1004 unsigned long long *histogram_boundaries = NULL; 1005 struct dm_arg_set as, as_backup; 1006 const char *a; 1007 unsigned int feature_args; 1008 1009 /* 1010 * Input format: 1011 * <range> <step> [<extra_parameters> <parameters>] [<program_id> [<aux_data>]] 1012 */ 1013 1014 if (argc < 3) 1015 goto ret_einval; 1016 1017 as.argc = argc; 1018 as.argv = argv; 1019 dm_consume_args(&as, 1); 1020 1021 a = dm_shift_arg(&as); 1022 if (!strcmp(a, "-")) { 1023 start = 0; 1024 len = dm_get_size(md); 1025 if (!len) 1026 len = 1; 1027 } else if (sscanf(a, "%llu+%llu%c", &start, &len, &dummy) != 2 || 1028 start != (sector_t)start || len != (sector_t)len) 1029 goto ret_einval; 1030 1031 end = start + len; 1032 if (start >= end) 1033 goto ret_einval; 1034 1035 a = dm_shift_arg(&as); 1036 if (sscanf(a, "/%u%c", &divisor, &dummy) == 1) { 1037 if (!divisor) 1038 return -EINVAL; 1039 step = end - start; 1040 if (do_div(step, divisor)) 1041 step++; 1042 if (!step) 1043 step = 1; 1044 } else if (sscanf(a, "%llu%c", &step, &dummy) != 1 || 1045 step != (sector_t)step || !step) 1046 goto ret_einval; 1047 1048 as_backup = as; 1049 a = dm_shift_arg(&as); 1050 if (a && sscanf(a, "%u%c", &feature_args, &dummy) == 1) { 1051 while (feature_args--) { 1052 a = dm_shift_arg(&as); 1053 if (!a) 1054 goto ret_einval; 1055 if (!strcasecmp(a, "precise_timestamps")) 1056 stat_flags |= STAT_PRECISE_TIMESTAMPS; 1057 else if (!strncasecmp(a, "histogram:", 10)) { 1058 if (n_histogram_entries) 1059 goto ret_einval; 1060 r = parse_histogram(a + 10, &n_histogram_entries, &histogram_boundaries); 1061 if (r) 1062 goto ret; 1063 } else 1064 goto ret_einval; 1065 } 1066 } else { 1067 as = as_backup; 1068 } 1069 1070 program_id = "-"; 1071 aux_data = "-"; 1072 1073 a = dm_shift_arg(&as); 1074 if (a) 1075 program_id = a; 1076 1077 a = dm_shift_arg(&as); 1078 if (a) 1079 aux_data = a; 1080 1081 if (as.argc) 1082 goto ret_einval; 1083 1084 /* 1085 * If a buffer overflow happens after we created the region, 1086 * it's too late (the userspace would retry with a larger 1087 * buffer, but the region id that caused the overflow is already 1088 * leaked). So we must detect buffer overflow in advance. 1089 */ 1090 snprintf(result, maxlen, "%d", INT_MAX); 1091 if (dm_message_test_buffer_overflow(result, maxlen)) { 1092 r = 1; 1093 goto ret; 1094 } 1095 1096 id = dm_stats_create(dm_get_stats(md), start, end, step, stat_flags, 1097 n_histogram_entries, histogram_boundaries, program_id, aux_data, 1098 dm_internal_suspend_fast, dm_internal_resume_fast, md); 1099 if (id < 0) { 1100 r = id; 1101 goto ret; 1102 } 1103 1104 snprintf(result, maxlen, "%d", id); 1105 1106 r = 1; 1107 goto ret; 1108 1109ret_einval: 1110 r = -EINVAL; 1111ret: 1112 kfree(histogram_boundaries); 1113 return r; 1114} 1115 1116static int message_stats_delete(struct mapped_device *md, 1117 unsigned int argc, char **argv) 1118{ 1119 int id; 1120 char dummy; 1121 1122 if (argc != 2) 1123 return -EINVAL; 1124 1125 if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0) 1126 return -EINVAL; 1127 1128 return dm_stats_delete(dm_get_stats(md), id); 1129} 1130 1131static int message_stats_clear(struct mapped_device *md, 1132 unsigned int argc, char **argv) 1133{ 1134 int id; 1135 char dummy; 1136 1137 if (argc != 2) 1138 return -EINVAL; 1139 1140 if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0) 1141 return -EINVAL; 1142 1143 return dm_stats_clear(dm_get_stats(md), id); 1144} 1145 1146static int message_stats_list(struct mapped_device *md, 1147 unsigned int argc, char **argv, 1148 char *result, unsigned int maxlen) 1149{ 1150 int r; 1151 const char *program = NULL; 1152 1153 if (argc < 1 || argc > 2) 1154 return -EINVAL; 1155 1156 if (argc > 1) { 1157 program = kstrdup(argv[1], GFP_KERNEL); 1158 if (!program) 1159 return -ENOMEM; 1160 } 1161 1162 r = dm_stats_list(dm_get_stats(md), program, result, maxlen); 1163 1164 kfree(program); 1165 1166 return r; 1167} 1168 1169static int message_stats_print(struct mapped_device *md, 1170 unsigned int argc, char **argv, bool clear, 1171 char *result, unsigned int maxlen) 1172{ 1173 int id; 1174 char dummy; 1175 unsigned long idx_start = 0, idx_len = ULONG_MAX; 1176 1177 if (argc != 2 && argc != 4) 1178 return -EINVAL; 1179 1180 if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0) 1181 return -EINVAL; 1182 1183 if (argc > 3) { 1184 if (strcmp(argv[2], "-") && 1185 sscanf(argv[2], "%lu%c", &idx_start, &dummy) != 1) 1186 return -EINVAL; 1187 if (strcmp(argv[3], "-") && 1188 sscanf(argv[3], "%lu%c", &idx_len, &dummy) != 1) 1189 return -EINVAL; 1190 } 1191 1192 return dm_stats_print(dm_get_stats(md), id, idx_start, idx_len, clear, 1193 result, maxlen); 1194} 1195 1196static int message_stats_set_aux(struct mapped_device *md, 1197 unsigned int argc, char **argv) 1198{ 1199 int id; 1200 char dummy; 1201 1202 if (argc != 3) 1203 return -EINVAL; 1204 1205 if (sscanf(argv[1], "%d%c", &id, &dummy) != 1 || id < 0) 1206 return -EINVAL; 1207 1208 return dm_stats_set_aux(dm_get_stats(md), id, argv[2]); 1209} 1210 1211int dm_stats_message(struct mapped_device *md, unsigned int argc, char **argv, 1212 char *result, unsigned int maxlen) 1213{ 1214 int r; 1215 1216 /* All messages here must start with '@' */ 1217 if (!strcasecmp(argv[0], "@stats_create")) 1218 r = message_stats_create(md, argc, argv, result, maxlen); 1219 else if (!strcasecmp(argv[0], "@stats_delete")) 1220 r = message_stats_delete(md, argc, argv); 1221 else if (!strcasecmp(argv[0], "@stats_clear")) 1222 r = message_stats_clear(md, argc, argv); 1223 else if (!strcasecmp(argv[0], "@stats_list")) 1224 r = message_stats_list(md, argc, argv, result, maxlen); 1225 else if (!strcasecmp(argv[0], "@stats_print")) 1226 r = message_stats_print(md, argc, argv, false, result, maxlen); 1227 else if (!strcasecmp(argv[0], "@stats_print_clear")) 1228 r = message_stats_print(md, argc, argv, true, result, maxlen); 1229 else if (!strcasecmp(argv[0], "@stats_set_aux")) 1230 r = message_stats_set_aux(md, argc, argv); 1231 else 1232 return 2; /* this wasn't a stats message */ 1233 1234 if (r == -EINVAL) 1235 DMCRIT("Invalid parameters for message %s", argv[0]); 1236 1237 return r; 1238} 1239 1240int __init dm_statistics_init(void) 1241{ 1242 shared_memory_amount = 0; 1243 dm_stat_need_rcu_barrier = 0; 1244 return 0; 1245} 1246 1247void dm_statistics_exit(void) 1248{ 1249 if (dm_stat_need_rcu_barrier) 1250 rcu_barrier(); 1251 if (WARN_ON(shared_memory_amount)) 1252 DMCRIT("shared_memory_amount leaked: %lu", shared_memory_amount); 1253} 1254 1255module_param_named(stats_current_allocated_bytes, shared_memory_amount, ulong, 0444); 1256MODULE_PARM_DESC(stats_current_allocated_bytes, "Memory currently used by statistics"); 1257