1// SPDX-License-Identifier: GPL-2.0 2/* 3 * Copyright (C) 2016 CNEX Labs 4 * Initial release: Javier Gonzalez <javier@cnexlabs.com> 5 * 6 * Based upon the circular ringbuffer. 7 * 8 * This program is free software; you can redistribute it and/or 9 * modify it under the terms of the GNU General Public License version 10 * 2 as published by the Free Software Foundation. 11 * 12 * This program is distributed in the hope that it will be useful, but 13 * WITHOUT ANY WARRANTY; without even the implied warranty of 14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15 * General Public License for more details. 16 * 17 * pblk-rb.c - pblk's write buffer 18 */ 19 20#include <linux/circ_buf.h> 21 22#include "pblk.h" 23 24static DECLARE_RWSEM(pblk_rb_lock); 25 26static void pblk_rb_data_free(struct pblk_rb *rb) 27{ 28 struct pblk_rb_pages *p, *t; 29 30 down_write(&pblk_rb_lock); 31 list_for_each_entry_safe(p, t, &rb->pages, list) { 32 free_pages((unsigned long)page_address(p->pages), p->order); 33 list_del(&p->list); 34 kfree(p); 35 } 36 up_write(&pblk_rb_lock); 37} 38 39void pblk_rb_free(struct pblk_rb *rb) 40{ 41 pblk_rb_data_free(rb); 42 vfree(rb->entries); 43} 44 45/* 46 * pblk_rb_calculate_size -- calculate the size of the write buffer 47 */ 48static unsigned int pblk_rb_calculate_size(unsigned int nr_entries, 49 unsigned int threshold) 50{ 51 unsigned int thr_sz = 1 << (get_count_order(threshold + NVM_MAX_VLBA)); 52 unsigned int max_sz = max(thr_sz, nr_entries); 53 unsigned int max_io; 54 55 /* Alloc a write buffer that can (i) fit at least two split bios 56 * (considering max I/O size NVM_MAX_VLBA, and (ii) guarantee that the 57 * threshold will be respected 58 */ 59 max_io = (1 << max((int)(get_count_order(max_sz)), 60 (int)(get_count_order(NVM_MAX_VLBA << 1)))); 61 if ((threshold + NVM_MAX_VLBA) >= max_io) 62 max_io <<= 1; 63 64 return max_io; 65} 66 67/* 68 * Initialize ring buffer. The data and metadata buffers must be previously 69 * allocated and their size must be a power of two 70 * (Documentation/core-api/circular-buffers.rst) 71 */ 72int pblk_rb_init(struct pblk_rb *rb, unsigned int size, unsigned int threshold, 73 unsigned int seg_size) 74{ 75 struct pblk *pblk = container_of(rb, struct pblk, rwb); 76 struct pblk_rb_entry *entries; 77 unsigned int init_entry = 0; 78 unsigned int max_order = MAX_ORDER - 1; 79 unsigned int power_size, power_seg_sz; 80 unsigned int alloc_order, order, iter; 81 unsigned int nr_entries; 82 83 nr_entries = pblk_rb_calculate_size(size, threshold); 84 entries = vzalloc(array_size(nr_entries, sizeof(struct pblk_rb_entry))); 85 if (!entries) 86 return -ENOMEM; 87 88 power_size = get_count_order(nr_entries); 89 power_seg_sz = get_count_order(seg_size); 90 91 down_write(&pblk_rb_lock); 92 rb->entries = entries; 93 rb->seg_size = (1 << power_seg_sz); 94 rb->nr_entries = (1 << power_size); 95 rb->mem = rb->subm = rb->sync = rb->l2p_update = 0; 96 rb->back_thres = threshold; 97 rb->flush_point = EMPTY_ENTRY; 98 99 spin_lock_init(&rb->w_lock); 100 spin_lock_init(&rb->s_lock); 101 102 INIT_LIST_HEAD(&rb->pages); 103 104 alloc_order = power_size; 105 if (alloc_order >= max_order) { 106 order = max_order; 107 iter = (1 << (alloc_order - max_order)); 108 } else { 109 order = alloc_order; 110 iter = 1; 111 } 112 113 do { 114 struct pblk_rb_entry *entry; 115 struct pblk_rb_pages *page_set; 116 void *kaddr; 117 unsigned long set_size; 118 int i; 119 120 page_set = kmalloc(sizeof(struct pblk_rb_pages), GFP_KERNEL); 121 if (!page_set) { 122 up_write(&pblk_rb_lock); 123 vfree(entries); 124 return -ENOMEM; 125 } 126 127 page_set->order = order; 128 page_set->pages = alloc_pages(GFP_KERNEL, order); 129 if (!page_set->pages) { 130 kfree(page_set); 131 pblk_rb_data_free(rb); 132 up_write(&pblk_rb_lock); 133 vfree(entries); 134 return -ENOMEM; 135 } 136 kaddr = page_address(page_set->pages); 137 138 entry = &rb->entries[init_entry]; 139 entry->data = kaddr; 140 entry->cacheline = pblk_cacheline_to_addr(init_entry++); 141 entry->w_ctx.flags = PBLK_WRITABLE_ENTRY; 142 143 set_size = (1 << order); 144 for (i = 1; i < set_size; i++) { 145 entry = &rb->entries[init_entry]; 146 entry->cacheline = pblk_cacheline_to_addr(init_entry++); 147 entry->data = kaddr + (i * rb->seg_size); 148 entry->w_ctx.flags = PBLK_WRITABLE_ENTRY; 149 bio_list_init(&entry->w_ctx.bios); 150 } 151 152 list_add_tail(&page_set->list, &rb->pages); 153 iter--; 154 } while (iter > 0); 155 up_write(&pblk_rb_lock); 156 157#ifdef CONFIG_NVM_PBLK_DEBUG 158 atomic_set(&rb->inflight_flush_point, 0); 159#endif 160 161 /* 162 * Initialize rate-limiter, which controls access to the write buffer 163 * by user and GC I/O 164 */ 165 pblk_rl_init(&pblk->rl, rb->nr_entries, threshold); 166 167 return 0; 168} 169 170static void clean_wctx(struct pblk_w_ctx *w_ctx) 171{ 172 int flags; 173 174 flags = READ_ONCE(w_ctx->flags); 175 WARN_ONCE(!(flags & PBLK_SUBMITTED_ENTRY), 176 "pblk: overwriting unsubmitted data\n"); 177 178 /* Release flags on context. Protect from writes and reads */ 179 smp_store_release(&w_ctx->flags, PBLK_WRITABLE_ENTRY); 180 pblk_ppa_set_empty(&w_ctx->ppa); 181 w_ctx->lba = ADDR_EMPTY; 182} 183 184#define pblk_rb_ring_count(head, tail, size) CIRC_CNT(head, tail, size) 185#define pblk_rb_ring_space(rb, head, tail, size) \ 186 (CIRC_SPACE(head, tail, size)) 187 188/* 189 * Buffer space is calculated with respect to the back pointer signaling 190 * synchronized entries to the media. 191 */ 192static unsigned int pblk_rb_space(struct pblk_rb *rb) 193{ 194 unsigned int mem = READ_ONCE(rb->mem); 195 unsigned int sync = READ_ONCE(rb->sync); 196 197 return pblk_rb_ring_space(rb, mem, sync, rb->nr_entries); 198} 199 200unsigned int pblk_rb_ptr_wrap(struct pblk_rb *rb, unsigned int p, 201 unsigned int nr_entries) 202{ 203 return (p + nr_entries) & (rb->nr_entries - 1); 204} 205 206/* 207 * Buffer count is calculated with respect to the submission entry signaling the 208 * entries that are available to send to the media 209 */ 210unsigned int pblk_rb_read_count(struct pblk_rb *rb) 211{ 212 unsigned int mem = READ_ONCE(rb->mem); 213 unsigned int subm = READ_ONCE(rb->subm); 214 215 return pblk_rb_ring_count(mem, subm, rb->nr_entries); 216} 217 218unsigned int pblk_rb_sync_count(struct pblk_rb *rb) 219{ 220 unsigned int mem = READ_ONCE(rb->mem); 221 unsigned int sync = READ_ONCE(rb->sync); 222 223 return pblk_rb_ring_count(mem, sync, rb->nr_entries); 224} 225 226unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int nr_entries) 227{ 228 unsigned int subm; 229 230 subm = READ_ONCE(rb->subm); 231 /* Commit read means updating submission pointer */ 232 smp_store_release(&rb->subm, pblk_rb_ptr_wrap(rb, subm, nr_entries)); 233 234 return subm; 235} 236 237static int __pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int to_update) 238{ 239 struct pblk *pblk = container_of(rb, struct pblk, rwb); 240 struct pblk_line *line; 241 struct pblk_rb_entry *entry; 242 struct pblk_w_ctx *w_ctx; 243 unsigned int user_io = 0, gc_io = 0; 244 unsigned int i; 245 int flags; 246 247 for (i = 0; i < to_update; i++) { 248 entry = &rb->entries[rb->l2p_update]; 249 w_ctx = &entry->w_ctx; 250 251 flags = READ_ONCE(entry->w_ctx.flags); 252 if (flags & PBLK_IOTYPE_USER) 253 user_io++; 254 else if (flags & PBLK_IOTYPE_GC) 255 gc_io++; 256 else 257 WARN(1, "pblk: unknown IO type\n"); 258 259 pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa, 260 entry->cacheline); 261 262 line = pblk_ppa_to_line(pblk, w_ctx->ppa); 263 atomic_dec(&line->sec_to_update); 264 kref_put(&line->ref, pblk_line_put); 265 clean_wctx(w_ctx); 266 rb->l2p_update = pblk_rb_ptr_wrap(rb, rb->l2p_update, 1); 267 } 268 269 pblk_rl_out(&pblk->rl, user_io, gc_io); 270 271 return 0; 272} 273 274/* 275 * When we move the l2p_update pointer, we update the l2p table - lookups will 276 * point to the physical address instead of to the cacheline in the write buffer 277 * from this moment on. 278 */ 279static int pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int nr_entries, 280 unsigned int mem, unsigned int sync) 281{ 282 unsigned int space, count; 283 int ret = 0; 284 285 lockdep_assert_held(&rb->w_lock); 286 287 /* Update l2p only as buffer entries are being overwritten */ 288 space = pblk_rb_ring_space(rb, mem, rb->l2p_update, rb->nr_entries); 289 if (space > nr_entries) 290 goto out; 291 292 count = nr_entries - space; 293 /* l2p_update used exclusively under rb->w_lock */ 294 ret = __pblk_rb_update_l2p(rb, count); 295 296out: 297 return ret; 298} 299 300/* 301 * Update the l2p entry for all sectors stored on the write buffer. This means 302 * that all future lookups to the l2p table will point to a device address, not 303 * to the cacheline in the write buffer. 304 */ 305void pblk_rb_sync_l2p(struct pblk_rb *rb) 306{ 307 unsigned int sync; 308 unsigned int to_update; 309 310 spin_lock(&rb->w_lock); 311 312 /* Protect from reads and writes */ 313 sync = smp_load_acquire(&rb->sync); 314 315 to_update = pblk_rb_ring_count(sync, rb->l2p_update, rb->nr_entries); 316 __pblk_rb_update_l2p(rb, to_update); 317 318 spin_unlock(&rb->w_lock); 319} 320 321/* 322 * Write @nr_entries to ring buffer from @data buffer if there is enough space. 323 * Typically, 4KB data chunks coming from a bio will be copied to the ring 324 * buffer, thus the write will fail if not all incoming data can be copied. 325 * 326 */ 327static void __pblk_rb_write_entry(struct pblk_rb *rb, void *data, 328 struct pblk_w_ctx w_ctx, 329 struct pblk_rb_entry *entry) 330{ 331 memcpy(entry->data, data, rb->seg_size); 332 333 entry->w_ctx.lba = w_ctx.lba; 334 entry->w_ctx.ppa = w_ctx.ppa; 335} 336 337void pblk_rb_write_entry_user(struct pblk_rb *rb, void *data, 338 struct pblk_w_ctx w_ctx, unsigned int ring_pos) 339{ 340 struct pblk *pblk = container_of(rb, struct pblk, rwb); 341 struct pblk_rb_entry *entry; 342 int flags; 343 344 entry = &rb->entries[ring_pos]; 345 flags = READ_ONCE(entry->w_ctx.flags); 346#ifdef CONFIG_NVM_PBLK_DEBUG 347 /* Caller must guarantee that the entry is free */ 348 BUG_ON(!(flags & PBLK_WRITABLE_ENTRY)); 349#endif 350 351 __pblk_rb_write_entry(rb, data, w_ctx, entry); 352 353 pblk_update_map_cache(pblk, w_ctx.lba, entry->cacheline); 354 flags = w_ctx.flags | PBLK_WRITTEN_DATA; 355 356 /* Release flags on write context. Protect from writes */ 357 smp_store_release(&entry->w_ctx.flags, flags); 358} 359 360void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data, 361 struct pblk_w_ctx w_ctx, struct pblk_line *line, 362 u64 paddr, unsigned int ring_pos) 363{ 364 struct pblk *pblk = container_of(rb, struct pblk, rwb); 365 struct pblk_rb_entry *entry; 366 int flags; 367 368 entry = &rb->entries[ring_pos]; 369 flags = READ_ONCE(entry->w_ctx.flags); 370#ifdef CONFIG_NVM_PBLK_DEBUG 371 /* Caller must guarantee that the entry is free */ 372 BUG_ON(!(flags & PBLK_WRITABLE_ENTRY)); 373#endif 374 375 __pblk_rb_write_entry(rb, data, w_ctx, entry); 376 377 if (!pblk_update_map_gc(pblk, w_ctx.lba, entry->cacheline, line, paddr)) 378 entry->w_ctx.lba = ADDR_EMPTY; 379 380 flags = w_ctx.flags | PBLK_WRITTEN_DATA; 381 382 /* Release flags on write context. Protect from writes */ 383 smp_store_release(&entry->w_ctx.flags, flags); 384} 385 386static int pblk_rb_flush_point_set(struct pblk_rb *rb, struct bio *bio, 387 unsigned int pos) 388{ 389 struct pblk_rb_entry *entry; 390 unsigned int sync, flush_point; 391 392 pblk_rb_sync_init(rb, NULL); 393 sync = READ_ONCE(rb->sync); 394 395 if (pos == sync) { 396 pblk_rb_sync_end(rb, NULL); 397 return 0; 398 } 399 400#ifdef CONFIG_NVM_PBLK_DEBUG 401 atomic_inc(&rb->inflight_flush_point); 402#endif 403 404 flush_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1); 405 entry = &rb->entries[flush_point]; 406 407 /* Protect flush points */ 408 smp_store_release(&rb->flush_point, flush_point); 409 410 if (bio) 411 bio_list_add(&entry->w_ctx.bios, bio); 412 413 pblk_rb_sync_end(rb, NULL); 414 415 return bio ? 1 : 0; 416} 417 418static int __pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries, 419 unsigned int *pos) 420{ 421 unsigned int mem; 422 unsigned int sync; 423 unsigned int threshold; 424 425 sync = READ_ONCE(rb->sync); 426 mem = READ_ONCE(rb->mem); 427 428 threshold = nr_entries + rb->back_thres; 429 430 if (pblk_rb_ring_space(rb, mem, sync, rb->nr_entries) < threshold) 431 return 0; 432 433 if (pblk_rb_update_l2p(rb, nr_entries, mem, sync)) 434 return 0; 435 436 *pos = mem; 437 438 return 1; 439} 440 441static int pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries, 442 unsigned int *pos) 443{ 444 if (!__pblk_rb_may_write(rb, nr_entries, pos)) 445 return 0; 446 447 /* Protect from read count */ 448 smp_store_release(&rb->mem, pblk_rb_ptr_wrap(rb, *pos, nr_entries)); 449 return 1; 450} 451 452void pblk_rb_flush(struct pblk_rb *rb) 453{ 454 struct pblk *pblk = container_of(rb, struct pblk, rwb); 455 unsigned int mem = READ_ONCE(rb->mem); 456 457 if (pblk_rb_flush_point_set(rb, NULL, mem)) 458 return; 459 460 pblk_write_kick(pblk); 461} 462 463static int pblk_rb_may_write_flush(struct pblk_rb *rb, unsigned int nr_entries, 464 unsigned int *pos, struct bio *bio, 465 int *io_ret) 466{ 467 unsigned int mem; 468 469 if (!__pblk_rb_may_write(rb, nr_entries, pos)) 470 return 0; 471 472 mem = pblk_rb_ptr_wrap(rb, *pos, nr_entries); 473 *io_ret = NVM_IO_DONE; 474 475 if (bio->bi_opf & REQ_PREFLUSH) { 476 struct pblk *pblk = container_of(rb, struct pblk, rwb); 477 478 atomic64_inc(&pblk->nr_flush); 479 if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem)) 480 *io_ret = NVM_IO_OK; 481 } 482 483 /* Protect from read count */ 484 smp_store_release(&rb->mem, mem); 485 486 return 1; 487} 488 489/* 490 * Atomically check that (i) there is space on the write buffer for the 491 * incoming I/O, and (ii) the current I/O type has enough budget in the write 492 * buffer (rate-limiter). 493 */ 494int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio, 495 unsigned int nr_entries, unsigned int *pos) 496{ 497 struct pblk *pblk = container_of(rb, struct pblk, rwb); 498 int io_ret; 499 500 spin_lock(&rb->w_lock); 501 io_ret = pblk_rl_user_may_insert(&pblk->rl, nr_entries); 502 if (io_ret) { 503 spin_unlock(&rb->w_lock); 504 return io_ret; 505 } 506 507 if (!pblk_rb_may_write_flush(rb, nr_entries, pos, bio, &io_ret)) { 508 spin_unlock(&rb->w_lock); 509 return NVM_IO_REQUEUE; 510 } 511 512 pblk_rl_user_in(&pblk->rl, nr_entries); 513 spin_unlock(&rb->w_lock); 514 515 return io_ret; 516} 517 518/* 519 * Look at pblk_rb_may_write_user comment 520 */ 521int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries, 522 unsigned int *pos) 523{ 524 struct pblk *pblk = container_of(rb, struct pblk, rwb); 525 526 spin_lock(&rb->w_lock); 527 if (!pblk_rl_gc_may_insert(&pblk->rl, nr_entries)) { 528 spin_unlock(&rb->w_lock); 529 return 0; 530 } 531 532 if (!pblk_rb_may_write(rb, nr_entries, pos)) { 533 spin_unlock(&rb->w_lock); 534 return 0; 535 } 536 537 pblk_rl_gc_in(&pblk->rl, nr_entries); 538 spin_unlock(&rb->w_lock); 539 540 return 1; 541} 542 543/* 544 * Read available entries on rb and add them to the given bio. To avoid a memory 545 * copy, a page reference to the write buffer is used to be added to the bio. 546 * 547 * This function is used by the write thread to form the write bio that will 548 * persist data on the write buffer to the media. 549 */ 550unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd, 551 unsigned int pos, unsigned int nr_entries, 552 unsigned int count) 553{ 554 struct pblk *pblk = container_of(rb, struct pblk, rwb); 555 struct request_queue *q = pblk->dev->q; 556 struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd); 557 struct bio *bio = rqd->bio; 558 struct pblk_rb_entry *entry; 559 struct page *page; 560 unsigned int pad = 0, to_read = nr_entries; 561 unsigned int i; 562 int flags; 563 564 if (count < nr_entries) { 565 pad = nr_entries - count; 566 to_read = count; 567 } 568 569 /* Add space for packed metadata if in use*/ 570 pad += (pblk->min_write_pgs - pblk->min_write_pgs_data); 571 572 c_ctx->sentry = pos; 573 c_ctx->nr_valid = to_read; 574 c_ctx->nr_padded = pad; 575 576 for (i = 0; i < to_read; i++) { 577 entry = &rb->entries[pos]; 578 579 /* A write has been allowed into the buffer, but data is still 580 * being copied to it. It is ok to busy wait. 581 */ 582try: 583 flags = READ_ONCE(entry->w_ctx.flags); 584 if (!(flags & PBLK_WRITTEN_DATA)) { 585 io_schedule(); 586 goto try; 587 } 588 589 page = virt_to_page(entry->data); 590 if (!page) { 591 pblk_err(pblk, "could not allocate write bio page\n"); 592 flags &= ~PBLK_WRITTEN_DATA; 593 flags |= PBLK_SUBMITTED_ENTRY; 594 /* Release flags on context. Protect from writes */ 595 smp_store_release(&entry->w_ctx.flags, flags); 596 return NVM_IO_ERR; 597 } 598 599 if (bio_add_pc_page(q, bio, page, rb->seg_size, 0) != 600 rb->seg_size) { 601 pblk_err(pblk, "could not add page to write bio\n"); 602 flags &= ~PBLK_WRITTEN_DATA; 603 flags |= PBLK_SUBMITTED_ENTRY; 604 /* Release flags on context. Protect from writes */ 605 smp_store_release(&entry->w_ctx.flags, flags); 606 return NVM_IO_ERR; 607 } 608 609 flags &= ~PBLK_WRITTEN_DATA; 610 flags |= PBLK_SUBMITTED_ENTRY; 611 612 /* Release flags on context. Protect from writes */ 613 smp_store_release(&entry->w_ctx.flags, flags); 614 615 pos = pblk_rb_ptr_wrap(rb, pos, 1); 616 } 617 618 if (pad) { 619 if (pblk_bio_add_pages(pblk, bio, GFP_KERNEL, pad)) { 620 pblk_err(pblk, "could not pad page in write bio\n"); 621 return NVM_IO_ERR; 622 } 623 624 if (pad < pblk->min_write_pgs) 625 atomic64_inc(&pblk->pad_dist[pad - 1]); 626 else 627 pblk_warn(pblk, "padding more than min. sectors\n"); 628 629 atomic64_add(pad, &pblk->pad_wa); 630 } 631 632#ifdef CONFIG_NVM_PBLK_DEBUG 633 atomic_long_add(pad, &pblk->padded_writes); 634#endif 635 636 return NVM_IO_OK; 637} 638 639/* 640 * Copy to bio only if the lba matches the one on the given cache entry. 641 * Otherwise, it means that the entry has been overwritten, and the bio should 642 * be directed to disk. 643 */ 644int pblk_rb_copy_to_bio(struct pblk_rb *rb, struct bio *bio, sector_t lba, 645 struct ppa_addr ppa) 646{ 647 struct pblk *pblk = container_of(rb, struct pblk, rwb); 648 struct pblk_rb_entry *entry; 649 struct pblk_w_ctx *w_ctx; 650 struct ppa_addr l2p_ppa; 651 u64 pos = pblk_addr_to_cacheline(ppa); 652 void *data; 653 int flags; 654 int ret = 1; 655 656 657#ifdef CONFIG_NVM_PBLK_DEBUG 658 /* Caller must ensure that the access will not cause an overflow */ 659 BUG_ON(pos >= rb->nr_entries); 660#endif 661 entry = &rb->entries[pos]; 662 w_ctx = &entry->w_ctx; 663 flags = READ_ONCE(w_ctx->flags); 664 665 spin_lock(&rb->w_lock); 666 spin_lock(&pblk->trans_lock); 667 l2p_ppa = pblk_trans_map_get(pblk, lba); 668 spin_unlock(&pblk->trans_lock); 669 670 /* Check if the entry has been overwritten or is scheduled to be */ 671 if (!pblk_ppa_comp(l2p_ppa, ppa) || w_ctx->lba != lba || 672 flags & PBLK_WRITABLE_ENTRY) { 673 ret = 0; 674 goto out; 675 } 676 data = bio_data(bio); 677 memcpy(data, entry->data, rb->seg_size); 678 679out: 680 spin_unlock(&rb->w_lock); 681 return ret; 682} 683 684struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos) 685{ 686 unsigned int entry = pblk_rb_ptr_wrap(rb, pos, 0); 687 688 return &rb->entries[entry].w_ctx; 689} 690 691unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags) 692 __acquires(&rb->s_lock) 693{ 694 if (flags) 695 spin_lock_irqsave(&rb->s_lock, *flags); 696 else 697 spin_lock_irq(&rb->s_lock); 698 699 return rb->sync; 700} 701 702void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags) 703 __releases(&rb->s_lock) 704{ 705 lockdep_assert_held(&rb->s_lock); 706 707 if (flags) 708 spin_unlock_irqrestore(&rb->s_lock, *flags); 709 else 710 spin_unlock_irq(&rb->s_lock); 711} 712 713unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries) 714{ 715 unsigned int sync, flush_point; 716 lockdep_assert_held(&rb->s_lock); 717 718 sync = READ_ONCE(rb->sync); 719 flush_point = READ_ONCE(rb->flush_point); 720 721 if (flush_point != EMPTY_ENTRY) { 722 unsigned int secs_to_flush; 723 724 secs_to_flush = pblk_rb_ring_count(flush_point, sync, 725 rb->nr_entries); 726 if (secs_to_flush < nr_entries) { 727 /* Protect flush points */ 728 smp_store_release(&rb->flush_point, EMPTY_ENTRY); 729 } 730 } 731 732 sync = pblk_rb_ptr_wrap(rb, sync, nr_entries); 733 734 /* Protect from counts */ 735 smp_store_release(&rb->sync, sync); 736 737 return sync; 738} 739 740/* Calculate how many sectors to submit up to the current flush point. */ 741unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb) 742{ 743 unsigned int subm, sync, flush_point; 744 unsigned int submitted, to_flush; 745 746 /* Protect flush points */ 747 flush_point = smp_load_acquire(&rb->flush_point); 748 if (flush_point == EMPTY_ENTRY) 749 return 0; 750 751 /* Protect syncs */ 752 sync = smp_load_acquire(&rb->sync); 753 754 subm = READ_ONCE(rb->subm); 755 submitted = pblk_rb_ring_count(subm, sync, rb->nr_entries); 756 757 /* The sync point itself counts as a sector to sync */ 758 to_flush = pblk_rb_ring_count(flush_point, sync, rb->nr_entries) + 1; 759 760 return (submitted < to_flush) ? (to_flush - submitted) : 0; 761} 762 763int pblk_rb_tear_down_check(struct pblk_rb *rb) 764{ 765 struct pblk_rb_entry *entry; 766 int i; 767 int ret = 0; 768 769 spin_lock(&rb->w_lock); 770 spin_lock_irq(&rb->s_lock); 771 772 if ((rb->mem == rb->subm) && (rb->subm == rb->sync) && 773 (rb->sync == rb->l2p_update) && 774 (rb->flush_point == EMPTY_ENTRY)) { 775 goto out; 776 } 777 778 if (!rb->entries) { 779 ret = 1; 780 goto out; 781 } 782 783 for (i = 0; i < rb->nr_entries; i++) { 784 entry = &rb->entries[i]; 785 786 if (!entry->data) { 787 ret = 1; 788 goto out; 789 } 790 } 791 792out: 793 spin_unlock_irq(&rb->s_lock); 794 spin_unlock(&rb->w_lock); 795 796 return ret; 797} 798 799unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos) 800{ 801 return (pos & (rb->nr_entries - 1)); 802} 803 804int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos) 805{ 806 return (pos >= rb->nr_entries); 807} 808 809ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf) 810{ 811 struct pblk *pblk = container_of(rb, struct pblk, rwb); 812 struct pblk_c_ctx *c; 813 ssize_t offset; 814 int queued_entries = 0; 815 816 spin_lock_irq(&rb->s_lock); 817 list_for_each_entry(c, &pblk->compl_list, list) 818 queued_entries++; 819 spin_unlock_irq(&rb->s_lock); 820 821 if (rb->flush_point != EMPTY_ENTRY) 822 offset = scnprintf(buf, PAGE_SIZE, 823 "%u\t%u\t%u\t%u\t%u\t%u\t%u - %u/%u/%u - %d\n", 824 rb->nr_entries, 825 rb->mem, 826 rb->subm, 827 rb->sync, 828 rb->l2p_update, 829#ifdef CONFIG_NVM_PBLK_DEBUG 830 atomic_read(&rb->inflight_flush_point), 831#else 832 0, 833#endif 834 rb->flush_point, 835 pblk_rb_read_count(rb), 836 pblk_rb_space(rb), 837 pblk_rb_flush_point_count(rb), 838 queued_entries); 839 else 840 offset = scnprintf(buf, PAGE_SIZE, 841 "%u\t%u\t%u\t%u\t%u\t%u\tNULL - %u/%u/%u - %d\n", 842 rb->nr_entries, 843 rb->mem, 844 rb->subm, 845 rb->sync, 846 rb->l2p_update, 847#ifdef CONFIG_NVM_PBLK_DEBUG 848 atomic_read(&rb->inflight_flush_point), 849#else 850 0, 851#endif 852 pblk_rb_read_count(rb), 853 pblk_rb_space(rb), 854 pblk_rb_flush_point_count(rb), 855 queued_entries); 856 857 return offset; 858} 859