Lines Matching refs:line
36 struct pblk_line *line;
39 line = pblk_ppa_to_line(pblk, *ppa);
42 pblk_err(pblk, "failed to mark bb, line:%d, pos:%d\n",
43 line->id, pos);
50 static void pblk_mark_bb(struct pblk *pblk, struct pblk_line *line,
58 pblk_debug(pblk, "erase failed: line:%d, pos:%d\n", line->id, pos);
61 atomic_dec(&line->blk_in_line);
62 if (test_and_set_bit(pos, line->blk_bitmap))
63 pblk_err(pblk, "attempted to erase bb: line:%d, pos:%d\n",
64 line->id, pos);
84 struct pblk_line *line;
87 line = pblk_ppa_to_line(pblk, rqd->ppa_addr);
89 chunk = &line->chks[pos];
91 atomic_dec(&line->left_seblks);
98 pblk_mark_bb(pblk, line, rqd->ppa_addr);
164 void __pblk_map_invalidate(struct pblk *pblk, struct pblk_line *line,
174 spin_lock(&line->lock);
175 WARN_ON(line->state == PBLK_LINESTATE_FREE);
177 if (test_and_set_bit(paddr, line->invalid_bitmap)) {
179 spin_unlock(&line->lock);
182 le32_add_cpu(line->vsc, -1);
184 if (line->state == PBLK_LINESTATE_CLOSED)
185 move_list = pblk_line_gc_list(pblk, line);
186 spin_unlock(&line->lock);
190 spin_lock(&line->lock);
191 /* Prevent moving a line that has just been chosen for GC */
192 if (line->state == PBLK_LINESTATE_GC) {
193 spin_unlock(&line->lock);
197 spin_unlock(&line->lock);
199 list_move_tail(&line->list, move_list);
206 struct pblk_line *line;
215 line = pblk_ppa_to_line(pblk, ppa);
218 __pblk_map_invalidate(pblk, line, paddr);
407 struct list_head *pblk_line_gc_list(struct pblk *pblk, struct pblk_line *line)
412 int packed_meta = (le32_to_cpu(*line->vsc) / pblk->min_write_pgs_data)
414 int vsc = le32_to_cpu(*line->vsc) + packed_meta;
416 lockdep_assert_held(&line->lock);
418 if (line->w_err_gc->has_write_err) {
419 if (line->gc_group != PBLK_LINEGC_WERR) {
420 line->gc_group = PBLK_LINEGC_WERR;
425 if (line->gc_group != PBLK_LINEGC_FULL) {
426 line->gc_group = PBLK_LINEGC_FULL;
430 if (line->gc_group != PBLK_LINEGC_HIGH) {
431 line->gc_group = PBLK_LINEGC_HIGH;
435 if (line->gc_group != PBLK_LINEGC_MID) {
436 line->gc_group = PBLK_LINEGC_MID;
439 } else if (vsc < line->sec_in_line) {
440 if (line->gc_group != PBLK_LINEGC_LOW) {
441 line->gc_group = PBLK_LINEGC_LOW;
444 } else if (vsc == line->sec_in_line) {
445 if (line->gc_group != PBLK_LINEGC_EMPTY) {
446 line->gc_group = PBLK_LINEGC_EMPTY;
450 line->state = PBLK_LINESTATE_CORRUPT;
451 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
452 line->state);
454 line->gc_group = PBLK_LINEGC_NONE;
456 pblk_err(pblk, "corrupted vsc for line %d, vsc:%d (%d/%d/%d)\n",
457 line->id, vsc,
458 line->sec_in_line,
598 void pblk_dealloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs)
603 spin_lock(&line->lock);
604 addr = find_next_zero_bit(line->map_bitmap,
605 pblk->lm.sec_per_line, line->cur_sec);
606 line->cur_sec = addr - nr_secs;
608 for (i = 0; i < nr_secs; i++, line->cur_sec--)
609 WARN_ON(!test_and_clear_bit(line->cur_sec, line->map_bitmap));
610 spin_unlock(&line->lock);
613 u64 __pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs)
618 lockdep_assert_held(&line->lock);
621 if (line->cur_sec + nr_secs > pblk->lm.sec_per_line) {
623 nr_secs = pblk->lm.sec_per_line - line->cur_sec;
626 line->cur_sec = addr = find_next_zero_bit(line->map_bitmap,
627 pblk->lm.sec_per_line, line->cur_sec);
628 for (i = 0; i < nr_secs; i++, line->cur_sec++)
629 WARN_ON(test_and_set_bit(line->cur_sec, line->map_bitmap));
634 u64 pblk_alloc_page(struct pblk *pblk, struct pblk_line *line, int nr_secs)
641 spin_lock(&line->lock);
642 addr = __pblk_alloc_page(pblk, line, nr_secs);
643 line->left_msecs -= nr_secs;
644 WARN(line->left_msecs < 0, "pblk: page allocation out of bounds\n");
645 spin_unlock(&line->lock);
650 u64 pblk_lookup_page(struct pblk *pblk, struct pblk_line *line)
654 spin_lock(&line->lock);
655 paddr = find_next_zero_bit(line->map_bitmap,
656 pblk->lm.sec_per_line, line->cur_sec);
657 spin_unlock(&line->lock);
662 u64 pblk_line_smeta_start(struct pblk *pblk, struct pblk_line *line)
670 bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line);
677 int pblk_line_smeta_read(struct pblk *pblk, struct pblk_line *line)
682 u64 paddr = pblk_line_smeta_start(pblk, line);
697 ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id);
699 ret = pblk_submit_io_sync(pblk, &rqd, line->smeta);
717 static int pblk_line_smeta_write(struct pblk *pblk, struct pblk_line *line,
723 __le64 *lba_list = emeta_to_lbas(pblk, line->emeta->buf);
742 ppa_list[i] = addr_to_gen_ppa(pblk, paddr, line->id);
746 ret = pblk_submit_io_sync_sem(pblk, &rqd, line->smeta);
764 int pblk_line_emeta_read(struct pblk *pblk, struct pblk_line *line,
773 u64 paddr = line->emeta_ssec;
777 int line_id = line->id;
811 while (test_bit(pos, line->blk_bitmap)) {
885 int pblk_line_erase(struct pblk *pblk, struct pblk_line *line)
893 spin_lock(&line->lock);
894 bit = find_next_zero_bit(line->erase_bitmap, lm->blk_per_line,
897 spin_unlock(&line->lock);
902 ppa.a.blk = line->id;
904 atomic_dec(&line->left_eblks);
905 WARN_ON(test_and_set_bit(bit, line->erase_bitmap));
906 spin_unlock(&line->lock);
910 pblk_err(pblk, "failed to erase line %d\n", line->id);
918 static void pblk_line_setup_metadata(struct pblk_line *line,
936 line->meta_line = meta_line;
938 line->smeta = l_mg->sline_meta[meta_line];
939 line->emeta = l_mg->eline_meta[meta_line];
941 memset(line->smeta, 0, lm->smeta_len);
942 memset(line->emeta->buf, 0, lm->emeta_len[0]);
944 line->emeta->mem = 0;
945 atomic_set(&line->emeta->sync, 0);
951 static int pblk_line_init_metadata(struct pblk *pblk, struct pblk_line *line,
958 struct pblk_emeta *emeta = line->emeta;
960 struct line_smeta *smeta_buf = (struct line_smeta *)line->smeta;
963 /* After erasing the line, new bad blocks might appear and we risk
964 * having an invalid line
967 bitmap_weight(line->blk_bitmap, lm->blk_per_line);
970 spin_lock(&line->lock);
971 line->state = PBLK_LINESTATE_BAD;
972 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
973 line->state);
974 spin_unlock(&line->lock);
976 list_add_tail(&line->list, &l_mg->bad_list);
979 pblk_debug(pblk, "line %d is bad\n", line->id);
985 line->lun_bitmap = ((void *)(smeta_buf)) + sizeof(struct line_smeta);
987 /* Mark LUNs allocated in this line (all for now) */
988 bitmap_set(line->lun_bitmap, 0, lm->lun_bitmap_len);
992 smeta_buf->header.id = cpu_to_le32(line->id);
993 smeta_buf->header.type = cpu_to_le16(line->type);
998 smeta_buf->seq_nr = cpu_to_le64(line->seq_nr);
1003 memcpy(line->lun_bitmap, cur->lun_bitmap, lm->lun_bitmap_len);
1005 cur->emeta->buf->next_id = cpu_to_le32(line->id);
1024 emeta_buf->seq_nr = cpu_to_le64(line->seq_nr);
1025 emeta_buf->nr_lbas = cpu_to_le64(line->sec_in_line);
1034 static int pblk_line_alloc_bitmaps(struct pblk *pblk, struct pblk_line *line)
1039 line->map_bitmap = mempool_alloc(l_mg->bitmap_pool, GFP_KERNEL);
1040 if (!line->map_bitmap)
1043 memset(line->map_bitmap, 0, lm->sec_bitmap_len);
1046 line->invalid_bitmap = mempool_alloc(l_mg->bitmap_pool, GFP_KERNEL);
1047 if (!line->invalid_bitmap) {
1048 mempool_free(line->map_bitmap, l_mg->bitmap_pool);
1049 line->map_bitmap = NULL;
1059 static int pblk_line_init_bb(struct pblk *pblk, struct pblk_line *line,
1070 line->sec_in_line = lm->sec_per_line;
1072 /* Capture bad block information on line mapping bitmaps */
1073 while ((bit = find_next_bit(line->blk_bitmap, lm->blk_per_line,
1078 bitmap_or(line->map_bitmap, line->map_bitmap, l_mg->bb_aux,
1080 line->sec_in_line -= geo->clba;
1084 bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line);
1086 bitmap_set(line->map_bitmap, off, lm->smeta_sec);
1087 line->sec_in_line -= lm->smeta_sec;
1088 line->cur_sec = off + lm->smeta_sec;
1090 if (init && pblk_line_smeta_write(pblk, line, off)) {
1091 pblk_debug(pblk, "line smeta I/O failed. Retry\n");
1095 bitmap_copy(line->invalid_bitmap, line->map_bitmap, lm->sec_per_line);
1104 if (!test_bit(off, line->invalid_bitmap)) {
1105 bitmap_set(line->invalid_bitmap, off, geo->ws_opt);
1110 line->emeta_ssec = off;
1111 line->sec_in_line -= lm->emeta_sec[0];
1112 line->nr_valid_lbas = 0;
1113 line->left_msecs = line->sec_in_line;
1114 *line->vsc = cpu_to_le32(line->sec_in_line);
1116 if (lm->sec_per_line - line->sec_in_line !=
1117 bitmap_weight(line->invalid_bitmap, lm->sec_per_line)) {
1118 spin_lock(&line->lock);
1119 line->state = PBLK_LINESTATE_BAD;
1120 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
1121 line->state);
1122 spin_unlock(&line->lock);
1124 list_add_tail(&line->list, &l_mg->bad_list);
1125 pblk_err(pblk, "unexpected line %d is bad\n", line->id);
1133 static int pblk_prepare_new_line(struct pblk *pblk, struct pblk_line *line)
1138 int blk_to_erase = atomic_read(&line->blk_in_line);
1144 int state = line->chks[pos].state;
1149 line->erase_bitmap);
1157 static int pblk_line_prepare(struct pblk *pblk, struct pblk_line *line)
1160 int blk_in_line = atomic_read(&line->blk_in_line);
1164 bitmap_copy(line->erase_bitmap, line->blk_bitmap, lm->blk_per_line);
1166 spin_lock(&line->lock);
1168 /* If we have not written to this line, we need to mark up free chunks
1171 if (line->state == PBLK_LINESTATE_NEW) {
1172 blk_to_erase = pblk_prepare_new_line(pblk, line);
1173 line->state = PBLK_LINESTATE_FREE;
1174 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
1175 line->state);
1181 spin_unlock(&line->lock);
1185 if (line->state != PBLK_LINESTATE_FREE) {
1186 WARN(1, "pblk: corrupted line %d, state %d\n",
1187 line->id, line->state);
1188 spin_unlock(&line->lock);
1192 line->state = PBLK_LINESTATE_OPEN;
1193 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
1194 line->state);
1196 atomic_set(&line->left_eblks, blk_to_erase);
1197 atomic_set(&line->left_seblks, blk_to_erase);
1199 line->meta_distance = lm->meta_distance;
1200 spin_unlock(&line->lock);
1202 kref_init(&line->ref);
1203 atomic_set(&line->sec_to_update, 0);
1209 int pblk_line_recov_alloc(struct pblk *pblk, struct pblk_line *line)
1215 l_mg->data_line = line;
1216 list_del(&line->list);
1218 ret = pblk_line_prepare(pblk, line);
1220 list_add(&line->list, &l_mg->free_list);
1226 ret = pblk_line_alloc_bitmaps(pblk, line);
1230 if (!pblk_line_init_bb(pblk, line, 0)) {
1235 pblk_rl_free_lines_dec(&pblk->rl, line, true);
1240 list_add(&line->list, &l_mg->free_list);
1246 void pblk_line_recov_close(struct pblk *pblk, struct pblk_line *line)
1250 mempool_free(line->map_bitmap, l_mg->bitmap_pool);
1251 line->map_bitmap = NULL;
1252 line->smeta = NULL;
1253 line->emeta = NULL;
1256 static void pblk_line_reinit(struct pblk_line *line)
1258 *line->vsc = cpu_to_le32(EMPTY_ENTRY);
1260 line->map_bitmap = NULL;
1261 line->invalid_bitmap = NULL;
1262 line->smeta = NULL;
1263 line->emeta = NULL;
1266 void pblk_line_free(struct pblk_line *line)
1268 struct pblk *pblk = line->pblk;
1271 mempool_free(line->map_bitmap, l_mg->bitmap_pool);
1272 mempool_free(line->invalid_bitmap, l_mg->bitmap_pool);
1274 pblk_line_reinit(line);
1281 struct pblk_line *line;
1292 line = list_first_entry(&l_mg->free_list, struct pblk_line, list);
1293 list_del(&line->list);
1296 bit = find_first_zero_bit(line->blk_bitmap, lm->blk_per_line);
1298 spin_lock(&line->lock);
1299 line->state = PBLK_LINESTATE_BAD;
1300 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
1301 line->state);
1302 spin_unlock(&line->lock);
1304 list_add_tail(&line->list, &l_mg->bad_list);
1306 pblk_debug(pblk, "line %d is bad\n", line->id);
1310 ret = pblk_line_prepare(pblk, line);
1314 list_add(&line->list, &l_mg->bad_list);
1317 list_add(&line->list, &l_mg->corrupt_list);
1320 pblk_err(pblk, "failed to prepare line %d\n", line->id);
1321 list_add(&line->list, &l_mg->free_list);
1327 return line;
1331 struct pblk_line *line)
1345 retry_line->map_bitmap = line->map_bitmap;
1346 retry_line->invalid_bitmap = line->invalid_bitmap;
1347 retry_line->smeta = line->smeta;
1348 retry_line->emeta = line->emeta;
1349 retry_line->meta_line = line->meta_line;
1351 pblk_line_reinit(line);
1356 pblk_rl_free_lines_dec(&pblk->rl, line, false);
1374 struct pblk_line *line;
1377 line = pblk_line_get(pblk);
1378 if (!line) {
1383 line->seq_nr = l_mg->d_seq_nr++;
1384 line->type = PBLK_LINETYPE_DATA;
1385 l_mg->data_line = line;
1387 pblk_line_setup_metadata(line, l_mg, &pblk->lm);
1389 /* Allocate next line for preparation */
1392 /* If we cannot get a new line, we need to stop the pipeline.
1405 if (pblk_line_alloc_bitmaps(pblk, line))
1408 if (pblk_line_erase(pblk, line)) {
1409 line = pblk_line_retry(pblk, line);
1410 if (!line)
1415 if (!pblk_line_init_metadata(pblk, line, NULL)) {
1416 line = pblk_line_retry(pblk, line);
1417 if (!line)
1423 if (!pblk_line_init_bb(pblk, line, 1)) {
1424 line = pblk_line_retry(pblk, line);
1425 if (!line)
1431 pblk_rl_free_lines_dec(&pblk->rl, line, true);
1433 return line;
1438 struct pblk_line *line;
1440 line = pblk_ppa_to_line(pblk, ppa);
1441 kref_put(&line->ref, pblk_line_put_wq);
1453 static void pblk_stop_writes(struct pblk *pblk, struct pblk_line *line)
1466 struct pblk_line *line, *tline;
1478 list_for_each_entry_safe(line, tline, &list, list) {
1479 struct pblk_emeta *emeta = line->emeta;
1484 ret = pblk_submit_meta_io(pblk, line);
1486 pblk_err(pblk, "sync meta line %d failed (%d)\n",
1487 line->id, ret);
1563 /* If line is not fully erased, erase it */
1595 /* Allocate next line for preparation */
1599 /* If we cannot get a new line, we need to stop the pipeline.
1615 static void __pblk_line_put(struct pblk *pblk, struct pblk_line *line)
1620 spin_lock(&line->lock);
1621 WARN_ON(line->state != PBLK_LINESTATE_GC);
1622 if (line->w_err_gc->has_gc_err) {
1623 spin_unlock(&line->lock);
1624 pblk_err(pblk, "line %d had errors during GC\n", line->id);
1625 pblk_put_line_back(pblk, line);
1626 line->w_err_gc->has_gc_err = 0;
1630 line->state = PBLK_LINESTATE_FREE;
1631 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
1632 line->state);
1633 line->gc_group = PBLK_LINEGC_NONE;
1634 pblk_line_free(line);
1636 if (line->w_err_gc->has_write_err) {
1638 line->w_err_gc->has_write_err = 0;
1641 spin_unlock(&line->lock);
1645 list_add_tail(&line->list, &l_mg->free_list);
1649 pblk_rl_free_lines_inc(&pblk->rl, line);
1657 struct pblk_line *line = line_put_ws->line;
1659 __pblk_line_put(pblk, line);
1665 struct pblk_line *line = container_of(ref, struct pblk_line, ref);
1666 struct pblk *pblk = line->pblk;
1668 __pblk_line_put(pblk, line);
1673 struct pblk_line *line = container_of(ref, struct pblk_line, ref);
1674 struct pblk *pblk = line->pblk;
1682 line_put_ws->line = line;
1712 pblk_err(pblk, "could not async erase line:%d,blk:%d\n",
1725 /* For now, always erase next line */
1731 int pblk_line_is_full(struct pblk_line *line)
1733 return (line->left_msecs == 0);
1742 void pblk_line_close(struct pblk *pblk, struct pblk_line *line)
1752 WARN(!bitmap_full(line->map_bitmap, lm->sec_per_line),
1753 "pblk: corrupt closed line %d\n", line->id);
1757 WARN_ON(!test_and_clear_bit(line->meta_line, &l_mg->meta_bitmap));
1761 spin_lock(&line->lock);
1762 WARN_ON(line->state != PBLK_LINESTATE_OPEN);
1763 line->state = PBLK_LINESTATE_CLOSED;
1764 move_list = pblk_line_gc_list(pblk, line);
1765 list_add_tail(&line->list, move_list);
1767 mempool_free(line->map_bitmap, l_mg->bitmap_pool);
1768 line->map_bitmap = NULL;
1769 line->smeta = NULL;
1770 line->emeta = NULL;
1775 int state = line->chks[pos].state;
1781 spin_unlock(&line->lock);
1784 trace_pblk_line_state(pblk_disk_name(pblk), line->id,
1785 line->state);
1788 void pblk_line_close_meta(struct pblk *pblk, struct pblk_line *line)
1792 struct pblk_emeta *emeta = line->emeta;
1796 /* No need for exact vsc value; avoid a big line lock and take aprox. */
1798 memcpy(emeta_to_bb(emeta_buf), line->blk_bitmap, lm->blk_bitmap_len);
1808 emeta_buf->header.id = cpu_to_le32(line->id);
1809 emeta_buf->header.type = cpu_to_le16(line->type);
1816 emeta_buf->nr_valid_lbas = cpu_to_le64(line->nr_valid_lbas);
1820 spin_lock(&line->lock);
1825 if (line->emeta_ssec != line->cur_sec)
1826 line->emeta_ssec = line->cur_sec;
1828 list_add_tail(&line->list, &l_mg->emeta_list);
1829 spin_unlock(&line->lock);
1835 static void pblk_save_lba_list(struct pblk *pblk, struct pblk_line *line)
1839 struct pblk_w_err_gc *w_err_gc = line->w_err_gc;
1840 struct pblk_emeta *emeta = line->emeta;
1852 struct pblk_line *line = line_ws->line;
1853 struct pblk_w_err_gc *w_err_gc = line->w_err_gc;
1856 * so keep a copy of the lba list until we've gc'd the line
1859 pblk_save_lba_list(pblk, line);
1861 pblk_line_close(pblk, line);
1865 void pblk_gen_run_ws(struct pblk *pblk, struct pblk_line *line, void *priv,
1874 line_ws->line = line;
2075 /* If the L2P entry maps to a line, the reference is valid */
2077 struct pblk_line *line = pblk_ppa_to_line(pblk, ppa);
2083 kref_get(&line->ref);