Lines Matching refs:pblk
19 * pblk-init.c - pblk's initialization.
22 #include "pblk.h"
23 #include "pblk-trace.h"
52 struct pblk *pblk = bio->bi_disk->queue->queuedata;
55 pblk_discard(pblk, bio);
67 pblk_submit_read(pblk, bio);
73 if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl))
76 pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER);
88 static size_t pblk_trans_map_size(struct pblk *pblk)
92 if (pblk->addrf_len < 32)
95 return entry_size * pblk->capacity;
99 static u32 pblk_l2p_crc(struct pblk *pblk)
104 map_size = pblk_trans_map_size(pblk);
105 crc = crc32_le(crc, pblk->trans_map, map_size);
110 static void pblk_l2p_free(struct pblk *pblk)
112 vfree(pblk->trans_map);
115 static int pblk_l2p_recover(struct pblk *pblk, bool factory_init)
120 guid_gen(&pblk->instance_uuid);
122 line = pblk_recov_l2p(pblk);
124 pblk_err(pblk, "could not recover l2p table\n");
130 pblk_info(pblk, "init: L2P CRC: %x\n", pblk_l2p_crc(pblk));
134 pblk_gc_free_full_lines(pblk);
138 line = pblk_line_get_first_data(pblk);
146 static int pblk_l2p_init(struct pblk *pblk, bool factory_init)
153 map_size = pblk_trans_map_size(pblk);
154 pblk->trans_map = __vmalloc(map_size, GFP_KERNEL | __GFP_NOWARN |
156 if (!pblk->trans_map) {
157 pblk_err(pblk, "failed to allocate L2P (need %zu of memory)\n",
164 for (i = 0; i < pblk->capacity; i++)
165 pblk_trans_map_set(pblk, i, ppa);
167 ret = pblk_l2p_recover(pblk, factory_init);
169 vfree(pblk->trans_map);
174 static void pblk_rwb_free(struct pblk *pblk)
176 if (pblk_rb_tear_down_check(&pblk->rwb))
177 pblk_err(pblk, "write buffer error on tear down\n");
179 pblk_rb_free(&pblk->rwb);
182 static int pblk_rwb_init(struct pblk *pblk)
184 struct nvm_tgt_dev *dev = pblk->dev;
198 return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs);
201 static int pblk_set_addrf_12(struct pblk *pblk, struct nvm_geo *geo,
210 pblk_err(pblk, "supports only power-of-two channel config.\n");
217 pblk_err(pblk, "supports only power-of-two LUN config.\n");
274 static int pblk_set_addrf(struct pblk *pblk)
276 struct nvm_tgt_dev *dev = pblk->dev;
282 div_u64_rem(geo->clba, pblk->min_write_pgs, &mod);
284 pblk_err(pblk, "bad configuration of sectors/pages\n");
288 pblk->addrf_len = pblk_set_addrf_12(pblk, geo,
289 (void *)&pblk->addrf);
292 pblk->addrf_len = pblk_set_addrf_20(geo, (void *)&pblk->addrf,
293 &pblk->uaddrf);
296 pblk_err(pblk, "OCSSD revision not supported (%d)\n",
376 static int pblk_core_init(struct pblk *pblk)
378 struct nvm_tgt_dev *dev = pblk->dev;
382 atomic64_set(&pblk->user_wa, 0);
383 atomic64_set(&pblk->pad_wa, 0);
384 atomic64_set(&pblk->gc_wa, 0);
385 pblk->user_rst_wa = 0;
386 pblk->pad_rst_wa = 0;
387 pblk->gc_rst_wa = 0;
389 atomic64_set(&pblk->nr_flush, 0);
390 pblk->nr_flush_rst = 0;
392 pblk->min_write_pgs = geo->ws_opt;
393 pblk->min_write_pgs_data = pblk->min_write_pgs;
394 max_write_ppas = pblk->min_write_pgs * geo->all_luns;
395 pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA);
396 pblk->max_write_pgs = min_t(int, pblk->max_write_pgs,
398 pblk_set_sec_per_write(pblk, pblk->min_write_pgs);
400 pblk->oob_meta_size = geo->sos;
401 if (!pblk_is_oob_meta_supported(pblk)) {
409 if (pblk->min_write_pgs
419 pblk_err(pblk, "Not supported min write size\n");
428 pblk->max_write_pgs = pblk->min_write_pgs;
429 pblk->min_write_pgs_data = pblk->min_write_pgs - 1;
432 pblk->pad_dist = kcalloc(pblk->min_write_pgs - 1, sizeof(atomic64_t),
434 if (!pblk->pad_dist)
441 ret = mempool_init_page_pool(&pblk->page_bio_pool, NVM_MAX_VLBA, 0);
445 ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE,
450 ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns,
455 ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns,
460 ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns,
465 ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns,
470 pblk->close_wq = alloc_workqueue("pblk-close-wq",
472 if (!pblk->close_wq)
475 pblk->bb_wq = alloc_workqueue("pblk-bb-wq",
477 if (!pblk->bb_wq)
480 pblk->r_end_wq = alloc_workqueue("pblk-read-end-wq",
482 if (!pblk->r_end_wq)
485 if (pblk_set_addrf(pblk))
488 INIT_LIST_HEAD(&pblk->compl_list);
489 INIT_LIST_HEAD(&pblk->resubmit_list);
494 destroy_workqueue(pblk->r_end_wq);
496 destroy_workqueue(pblk->bb_wq);
498 destroy_workqueue(pblk->close_wq);
500 mempool_exit(&pblk->w_rq_pool);
502 mempool_exit(&pblk->e_rq_pool);
504 mempool_exit(&pblk->r_rq_pool);
506 mempool_exit(&pblk->rec_pool);
508 mempool_exit(&pblk->gen_ws_pool);
510 mempool_exit(&pblk->page_bio_pool);
514 kfree(pblk->pad_dist);
518 static void pblk_core_free(struct pblk *pblk)
520 if (pblk->close_wq)
521 destroy_workqueue(pblk->close_wq);
523 if (pblk->r_end_wq)
524 destroy_workqueue(pblk->r_end_wq);
526 if (pblk->bb_wq)
527 destroy_workqueue(pblk->bb_wq);
529 mempool_exit(&pblk->page_bio_pool);
530 mempool_exit(&pblk->gen_ws_pool);
531 mempool_exit(&pblk->rec_pool);
532 mempool_exit(&pblk->r_rq_pool);
533 mempool_exit(&pblk->e_rq_pool);
534 mempool_exit(&pblk->w_rq_pool);
537 kfree(pblk->pad_dist);
540 static void pblk_line_mg_free(struct pblk *pblk)
542 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
572 static void pblk_lines_free(struct pblk *pblk)
574 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
579 line = &pblk->lines[i];
585 pblk_line_mg_free(pblk);
587 kfree(pblk->luns);
588 kfree(pblk->lines);
591 static int pblk_luns_init(struct pblk *pblk)
593 struct nvm_tgt_dev *dev = pblk->dev;
600 pblk_err(pblk, "unbalanced LUN config.\n");
604 pblk->luns = kcalloc(geo->all_luns, sizeof(struct pblk_lun),
606 if (!pblk->luns)
615 rlun = &pblk->luns[i];
625 static unsigned int calc_emeta_len(struct pblk *pblk)
627 struct pblk_line_meta *lm = &pblk->lm;
628 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
629 struct nvm_tgt_dev *dev = pblk->dev;
653 static int pblk_set_provision(struct pblk *pblk, int nr_free_chks)
655 struct nvm_tgt_dev *dev = pblk->dev;
656 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
657 struct pblk_line_meta *lm = &pblk->lm;
664 pblk->op = PBLK_DEFAULT_OP;
666 pblk->op = geo->op;
668 minimum = pblk_get_min_chks(pblk);
670 provisioned *= (100 - pblk->op);
675 pblk_err(pblk, "OP too small to create a sane instance\n");
684 pblk->op = (100 * minimum) / nr_free_chks;
685 pblk_info(pblk, "Default OP insufficient, adjusting OP to %d\n",
686 pblk->op);
689 pblk->op_blks = nr_free_chks - provisioned;
691 /* Internally pblk manages all free blocks, but all calculations based
694 pblk->rl.total_blocks = nr_free_chks;
700 clba = (geo->clba / pblk->min_write_pgs) * pblk->min_write_pgs_data;
701 pblk->capacity = (provisioned - blk_meta) * clba;
703 atomic_set(&pblk->rl.free_blocks, nr_free_chks);
704 atomic_set(&pblk->rl.free_user_blocks, nr_free_chks);
709 static int pblk_setup_line_meta_chk(struct pblk *pblk, struct pblk_line *line,
712 struct nvm_tgt_dev *dev = pblk->dev;
714 struct pblk_line_meta *lm = &pblk->lm;
718 struct pblk_lun *rlun = &pblk->luns[i];
729 chunk_meta = pblk_chunk_get_off(pblk, meta, ppa);
738 trace_pblk_chunk_state(pblk_disk_name(pblk), &ppa,
742 WARN_ONCE(1, "pblk: custom-sized chunks unsupported\n");
756 static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line,
759 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
760 struct pblk_line_meta *lm = &pblk->lm;
763 line->pblk = pblk;
771 nr_bad_chks = pblk_setup_line_meta_chk(pblk, line, chunk_meta);
788 static int pblk_alloc_line_meta(struct pblk *pblk, struct pblk_line *line)
790 struct pblk_line_meta *lm = &pblk->lm;
821 static int pblk_line_mg_init(struct pblk *pblk)
823 struct nvm_tgt_dev *dev = pblk->dev;
825 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
826 struct pblk_line_meta *lm = &pblk->lm;
869 * emeta depends on the number of LUNs allocated to the pblk instance
938 static int pblk_line_meta_init(struct pblk *pblk)
940 struct nvm_tgt_dev *dev = pblk->dev;
942 struct pblk_line_meta *lm = &pblk->lm;
953 lm->meta_distance = (geo->all_luns / 2) * pblk->min_write_pgs;
977 emeta_len = calc_emeta_len(pblk);
991 pblk_err(pblk, "config. not supported. Min. LUN in line:%d\n",
999 static int pblk_lines_init(struct pblk *pblk)
1001 struct pblk_line_mgmt *l_mg = &pblk->l_mg;
1007 ret = pblk_line_meta_init(pblk);
1011 ret = pblk_line_mg_init(pblk);
1015 ret = pblk_luns_init(pblk);
1019 chunk_meta = pblk_get_chunk_meta(pblk);
1025 pblk->lines = kcalloc(l_mg->nr_lines, sizeof(struct pblk_line),
1027 if (!pblk->lines) {
1033 line = &pblk->lines[i];
1035 ret = pblk_alloc_line_meta(pblk, line);
1039 nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i);
1041 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
1046 pblk_err(pblk, "too many bad blocks prevent for sane instance\n");
1051 ret = pblk_set_provision(pblk, nr_free_chks);
1060 pblk_line_meta_free(l_mg, &pblk->lines[i]);
1061 kfree(pblk->lines);
1065 kfree(pblk->luns);
1067 pblk_line_mg_free(pblk);
1072 static int pblk_writer_init(struct pblk *pblk)
1074 pblk->writer_ts = kthread_create(pblk_write_ts, pblk, "pblk-writer-t");
1075 if (IS_ERR(pblk->writer_ts)) {
1076 int err = PTR_ERR(pblk->writer_ts);
1079 pblk_err(pblk, "could not allocate writer kthread (%d)\n",
1084 timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0);
1085 mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100));
1090 static void pblk_writer_stop(struct pblk *pblk)
1095 WARN(pblk_rb_read_count(&pblk->rwb),
1098 WARN(pblk_rb_sync_count(&pblk->rwb),
1101 del_timer_sync(&pblk->wtimer);
1102 if (pblk->writer_ts)
1103 kthread_stop(pblk->writer_ts);
1106 static void pblk_free(struct pblk *pblk)
1108 pblk_lines_free(pblk);
1109 pblk_l2p_free(pblk);
1110 pblk_rwb_free(pblk);
1111 pblk_core_free(pblk);
1113 kfree(pblk);
1116 static void pblk_tear_down(struct pblk *pblk, bool graceful)
1119 __pblk_pipeline_flush(pblk);
1120 __pblk_pipeline_stop(pblk);
1121 pblk_writer_stop(pblk);
1122 pblk_rb_sync_l2p(&pblk->rwb);
1123 pblk_rl_free(&pblk->rl);
1125 pblk_debug(pblk, "consistent tear down (graceful:%d)\n", graceful);
1130 struct pblk *pblk = private;
1132 pblk_gc_exit(pblk, graceful);
1133 pblk_tear_down(pblk, graceful);
1136 pblk_info(pblk, "exit: L2P CRC: %x\n", pblk_l2p_crc(pblk));
1139 pblk_free(pblk);
1144 struct pblk *pblk = private;
1146 return pblk->capacity * NR_PHY_IN_LOG;
1155 struct pblk *pblk;
1158 pblk = kzalloc(sizeof(struct pblk), GFP_KERNEL);
1159 if (!pblk)
1162 pblk->dev = dev;
1163 pblk->disk = tdisk;
1164 pblk->state = PBLK_STATE_RUNNING;
1165 trace_pblk_state(pblk_disk_name(pblk), pblk->state);
1166 pblk->gc.gc_enabled = 0;
1170 pblk_err(pblk, "OCSSD version not supported (%u)\n",
1172 kfree(pblk);
1177 pblk_err(pblk, "extended metadata not supported\n");
1178 kfree(pblk);
1182 spin_lock_init(&pblk->resubmit_lock);
1183 spin_lock_init(&pblk->trans_lock);
1184 spin_lock_init(&pblk->lock);
1187 atomic_long_set(&pblk->inflight_writes, 0);
1188 atomic_long_set(&pblk->padded_writes, 0);
1189 atomic_long_set(&pblk->padded_wb, 0);
1190 atomic_long_set(&pblk->req_writes, 0);
1191 atomic_long_set(&pblk->sub_writes, 0);
1192 atomic_long_set(&pblk->sync_writes, 0);
1193 atomic_long_set(&pblk->inflight_reads, 0);
1194 atomic_long_set(&pblk->cache_reads, 0);
1195 atomic_long_set(&pblk->sync_reads, 0);
1196 atomic_long_set(&pblk->recov_writes, 0);
1197 atomic_long_set(&pblk->recov_writes, 0);
1198 atomic_long_set(&pblk->recov_gc_writes, 0);
1199 atomic_long_set(&pblk->recov_gc_reads, 0);
1202 atomic_long_set(&pblk->read_failed, 0);
1203 atomic_long_set(&pblk->read_empty, 0);
1204 atomic_long_set(&pblk->read_high_ecc, 0);
1205 atomic_long_set(&pblk->read_failed_gc, 0);
1206 atomic_long_set(&pblk->write_failed, 0);
1207 atomic_long_set(&pblk->erase_failed, 0);
1209 ret = pblk_core_init(pblk);
1211 pblk_err(pblk, "could not initialize core\n");
1215 ret = pblk_lines_init(pblk);
1217 pblk_err(pblk, "could not initialize lines\n");
1221 ret = pblk_rwb_init(pblk);
1223 pblk_err(pblk, "could not initialize write buffer\n");
1227 ret = pblk_l2p_init(pblk, flags & NVM_TARGET_FACTORY);
1229 pblk_err(pblk, "could not initialize maps\n");
1233 ret = pblk_writer_init(pblk);
1236 pblk_err(pblk, "could not initialize write thread\n");
1240 ret = pblk_gc_init(pblk);
1242 pblk_err(pblk, "could not initialize gc\n");
1257 pblk_info(pblk, "luns:%u, lines:%d, secs:%llu, buf entries:%u\n",
1258 geo->all_luns, pblk->l_mg.nr_lines,
1259 (unsigned long long)pblk->capacity,
1260 pblk->rwb.nr_entries);
1262 wake_up_process(pblk->writer_ts);
1265 pblk_gc_should_kick(pblk);
1267 return pblk;
1270 pblk_writer_stop(pblk);
1272 pblk_l2p_free(pblk);
1274 pblk_rwb_free(pblk);
1276 pblk_lines_free(pblk);
1278 pblk_core_free(pblk);
1280 kfree(pblk);
1286 .name = "pblk",