Lines Matching refs:stripe

96 	 * Set for data stripes if it's triggered from P/Q stripe.
125 * How many data/meta extents are in this stripe. Only for scrub status
136 * Indicate the states of the stripe. Bits are defined in
145 * The errors hit during the initial read of the stripe.
177 * Checksum for the whole stripe if this stripe is inside a data block
232 static void release_scrub_stripe(struct scrub_stripe *stripe)
234 if (!stripe)
238 if (stripe->pages[i])
239 __free_page(stripe->pages[i]);
240 stripe->pages[i] = NULL;
242 kfree(stripe->sectors);
243 kfree(stripe->csums);
244 stripe->sectors = NULL;
245 stripe->csums = NULL;
246 stripe->sctx = NULL;
247 stripe->state = 0;
251 struct scrub_stripe *stripe)
255 memset(stripe, 0, sizeof(*stripe));
257 stripe->nr_sectors = BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits;
258 stripe->state = 0;
260 init_waitqueue_head(&stripe->io_wait);
261 init_waitqueue_head(&stripe->repair_wait);
262 atomic_set(&stripe->pending_io, 0);
263 spin_lock_init(&stripe->write_error_lock);
265 ret = btrfs_alloc_page_array(SCRUB_STRIPE_PAGES, stripe->pages);
269 stripe->sectors = kcalloc(stripe->nr_sectors,
272 if (!stripe->sectors)
275 stripe->csums = kcalloc(BTRFS_STRIPE_LEN >> fs_info->sectorsize_bits,
277 if (!stripe->csums)
281 release_scrub_stripe(stripe);
285 static void wait_scrub_stripe_io(struct scrub_stripe *stripe)
287 wait_event(stripe->io_wait, atomic_read(&stripe->pending_io) == 0);
583 static struct page *scrub_stripe_get_page(struct scrub_stripe *stripe, int sector_nr)
585 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
588 return stripe->pages[page_index];
591 static unsigned int scrub_stripe_get_page_offset(struct scrub_stripe *stripe,
594 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
599 static void scrub_verify_one_metadata(struct scrub_stripe *stripe, int sector_nr)
601 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
603 const u64 logical = stripe->logical + (sector_nr << fs_info->sectorsize_bits);
604 const struct page *first_page = scrub_stripe_get_page(stripe, sector_nr);
605 const unsigned int first_off = scrub_stripe_get_page_offset(stripe, sector_nr);
620 bitmap_set(&stripe->csum_error_bitmap, sector_nr, sectors_per_tree);
621 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);
624 logical, stripe->mirror_num,
630 bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);
631 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);
634 logical, stripe->mirror_num,
640 bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);
641 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);
644 logical, stripe->mirror_num,
656 struct page *page = scrub_stripe_get_page(stripe, i);
657 unsigned int page_off = scrub_stripe_get_page_offset(stripe, i);
665 bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);
666 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);
669 logical, stripe->mirror_num,
674 if (stripe->sectors[sector_nr].generation !=
676 bitmap_set(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);
677 bitmap_set(&stripe->error_bitmap, sector_nr, sectors_per_tree);
680 logical, stripe->mirror_num,
682 stripe->sectors[sector_nr].generation);
685 bitmap_clear(&stripe->error_bitmap, sector_nr, sectors_per_tree);
686 bitmap_clear(&stripe->csum_error_bitmap, sector_nr, sectors_per_tree);
687 bitmap_clear(&stripe->meta_error_bitmap, sector_nr, sectors_per_tree);
690 static void scrub_verify_one_sector(struct scrub_stripe *stripe, int sector_nr)
692 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
693 struct scrub_sector_verification *sector = &stripe->sectors[sector_nr];
695 struct page *page = scrub_stripe_get_page(stripe, sector_nr);
696 unsigned int pgoff = scrub_stripe_get_page_offset(stripe, sector_nr);
700 ASSERT(sector_nr >= 0 && sector_nr < stripe->nr_sectors);
703 if (!test_bit(sector_nr, &stripe->extent_sector_bitmap))
707 if (test_bit(sector_nr, &stripe->io_error_bitmap))
713 * Check if the tree block crosses the stripe boudary. If
718 * are not ensured to be stripe aligned.
720 if (unlikely(sector_nr + sectors_per_tree > stripe->nr_sectors)) {
722 "tree block at %llu crosses stripe boundary %llu",
723 stripe->logical +
725 stripe->logical);
728 scrub_verify_one_metadata(stripe, sector_nr);
737 clear_bit(sector_nr, &stripe->error_bitmap);
743 set_bit(sector_nr, &stripe->csum_error_bitmap);
744 set_bit(sector_nr, &stripe->error_bitmap);
746 clear_bit(sector_nr, &stripe->csum_error_bitmap);
747 clear_bit(sector_nr, &stripe->error_bitmap);
751 /* Verify specified sectors of a stripe. */
752 static void scrub_verify_one_stripe(struct scrub_stripe *stripe, unsigned long bitmap)
754 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
758 for_each_set_bit(sector_nr, &bitmap, stripe->nr_sectors) {
759 scrub_verify_one_sector(stripe, sector_nr);
760 if (stripe->sectors[sector_nr].is_metadata)
765 static int calc_sector_number(struct scrub_stripe *stripe, struct bio_vec *first_bvec)
769 for (i = 0; i < stripe->nr_sectors; i++) {
770 if (scrub_stripe_get_page(stripe, i) == first_bvec->bv_page &&
771 scrub_stripe_get_page_offset(stripe, i) == first_bvec->bv_offset)
774 ASSERT(i < stripe->nr_sectors);
786 struct scrub_stripe *stripe = bbio->private;
787 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
789 int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));
793 ASSERT(sector_nr < stripe->nr_sectors);
799 bitmap_set(&stripe->io_error_bitmap, sector_nr,
801 bitmap_set(&stripe->error_bitmap, sector_nr,
804 bitmap_clear(&stripe->io_error_bitmap, sector_nr,
808 if (atomic_dec_and_test(&stripe->pending_io))
809 wake_up(&stripe->io_wait);
818 static void scrub_stripe_submit_repair_read(struct scrub_stripe *stripe,
821 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
823 const unsigned long old_error_bitmap = stripe->error_bitmap;
826 ASSERT(stripe->mirror_num >= 1);
827 ASSERT(atomic_read(&stripe->pending_io) == 0);
829 for_each_set_bit(i, &old_error_bitmap, stripe->nr_sectors) {
834 page = scrub_stripe_get_page(stripe, i);
835 pgoff = scrub_stripe_get_page_offset(stripe, i);
838 if (bbio && ((i > 0 && !test_bit(i - 1, &stripe->error_bitmap)) ||
841 atomic_inc(&stripe->pending_io);
844 wait_scrub_stripe_io(stripe);
849 bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_READ,
850 fs_info, scrub_repair_read_endio, stripe);
851 bbio->bio.bi_iter.bi_sector = (stripe->logical +
860 atomic_inc(&stripe->pending_io);
863 wait_scrub_stripe_io(stripe);
868 struct scrub_stripe *stripe)
881 if (test_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state))
890 if (!bitmap_empty(&stripe->init_error_bitmap, stripe->nr_sectors)) {
893 int stripe_index = stripe->mirror_num - 1;
897 ASSERT(stripe->mirror_num >= 1);
899 stripe->logical, &mapped_len, &bioc,
913 for_each_set_bit(sector_nr, &stripe->extent_sector_bitmap, stripe->nr_sectors) {
916 if (stripe->sectors[sector_nr].is_metadata) {
920 if (!stripe->sectors[sector_nr].csum)
924 if (test_bit(sector_nr, &stripe->init_error_bitmap) &&
925 !test_bit(sector_nr, &stripe->error_bitmap)) {
931 if (!test_bit(sector_nr, &stripe->init_error_bitmap))
942 stripe->logical, btrfs_dev_name(dev),
947 stripe->logical, stripe->mirror_num);
956 stripe->logical, btrfs_dev_name(dev),
961 stripe->logical, stripe->mirror_num);
964 if (test_bit(sector_nr, &stripe->io_error_bitmap))
967 stripe->logical, physical);
968 if (test_bit(sector_nr, &stripe->csum_error_bitmap))
971 stripe->logical, physical);
972 if (test_bit(sector_nr, &stripe->meta_error_bitmap))
975 stripe->logical, physical);
979 sctx->stat.data_extents_scrubbed += stripe->nr_data_extents;
980 sctx->stat.tree_extents_scrubbed += stripe->nr_meta_extents;
984 sctx->stat.read_errors += stripe->init_nr_io_errors;
985 sctx->stat.csum_errors += stripe->init_nr_csum_errors;
986 sctx->stat.verify_errors += stripe->init_nr_meta_errors;
988 bitmap_weight(&stripe->error_bitmap, stripe->nr_sectors);
993 static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe,
1011 struct scrub_stripe *stripe = container_of(work, struct scrub_stripe, work);
1012 struct scrub_ctx *sctx = stripe->sctx;
1014 int num_copies = btrfs_num_copies(fs_info, stripe->bg->start,
1015 stripe->bg->length);
1019 ASSERT(stripe->mirror_num > 0);
1021 wait_scrub_stripe_io(stripe);
1022 scrub_verify_one_stripe(stripe, stripe->extent_sector_bitmap);
1024 stripe->init_error_bitmap = stripe->error_bitmap;
1025 stripe->init_nr_io_errors = bitmap_weight(&stripe->io_error_bitmap,
1026 stripe->nr_sectors);
1027 stripe->init_nr_csum_errors = bitmap_weight(&stripe->csum_error_bitmap,
1028 stripe->nr_sectors);
1029 stripe->init_nr_meta_errors = bitmap_weight(&stripe->meta_error_bitmap,
1030 stripe->nr_sectors);
1032 if (bitmap_empty(&stripe->init_error_bitmap, stripe->nr_sectors))
1041 for (mirror = calc_next_mirror(stripe->mirror_num, num_copies);
1042 mirror != stripe->mirror_num;
1044 const unsigned long old_error_bitmap = stripe->error_bitmap;
1046 scrub_stripe_submit_repair_read(stripe, mirror,
1048 wait_scrub_stripe_io(stripe);
1049 scrub_verify_one_stripe(stripe, old_error_bitmap);
1050 if (bitmap_empty(&stripe->error_bitmap, stripe->nr_sectors))
1065 for (i = 0, mirror = stripe->mirror_num;
1068 const unsigned long old_error_bitmap = stripe->error_bitmap;
1070 scrub_stripe_submit_repair_read(stripe, mirror,
1072 wait_scrub_stripe_io(stripe);
1073 scrub_verify_one_stripe(stripe, old_error_bitmap);
1074 if (bitmap_empty(&stripe->error_bitmap, stripe->nr_sectors))
1083 if (!bitmap_empty(&stripe->error_bitmap, stripe->nr_sectors))
1088 bitmap_andnot(&repaired, &stripe->init_error_bitmap,
1089 &stripe->error_bitmap, stripe->nr_sectors);
1090 scrub_write_sectors(sctx, stripe, repaired, false);
1091 wait_scrub_stripe_io(stripe);
1094 scrub_stripe_report_errors(sctx, stripe);
1095 set_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state);
1096 wake_up(&stripe->repair_wait);
1101 struct scrub_stripe *stripe = bbio->private;
1103 int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));
1108 ASSERT(sector_nr < stripe->nr_sectors);
1111 num_sectors = bio_size >> stripe->bg->fs_info->sectorsize_bits;
1114 bitmap_set(&stripe->io_error_bitmap, sector_nr, num_sectors);
1115 bitmap_set(&stripe->error_bitmap, sector_nr, num_sectors);
1117 bitmap_clear(&stripe->io_error_bitmap, sector_nr, num_sectors);
1120 if (atomic_dec_and_test(&stripe->pending_io)) {
1121 wake_up(&stripe->io_wait);
1122 INIT_WORK(&stripe->work, scrub_stripe_read_repair_worker);
1123 queue_work(stripe->bg->fs_info->scrub_workers, &stripe->work);
1129 struct scrub_stripe *stripe = bbio->private;
1130 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
1132 int sector_nr = calc_sector_number(stripe, bio_first_bvec_all(&bbio->bio));
1142 spin_lock_irqsave(&stripe->write_error_lock, flags);
1143 bitmap_set(&stripe->write_error_bitmap, sector_nr,
1145 spin_unlock_irqrestore(&stripe->write_error_lock, flags);
1149 if (atomic_dec_and_test(&stripe->pending_io))
1150 wake_up(&stripe->io_wait);
1154 struct scrub_stripe *stripe,
1160 stripe->logical;
1162 fill_writer_pointer_gap(sctx, stripe->physical + bio_off);
1163 atomic_inc(&stripe->pending_io);
1164 btrfs_submit_repair_write(bbio, stripe->mirror_num, dev_replace);
1171 wait_scrub_stripe_io(stripe);
1178 &stripe->write_error_bitmap))
1195 static void scrub_write_sectors(struct scrub_ctx *sctx, struct scrub_stripe *stripe,
1198 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
1202 for_each_set_bit(sector_nr, &write_bitmap, stripe->nr_sectors) {
1203 struct page *page = scrub_stripe_get_page(stripe, sector_nr);
1204 unsigned int pgoff = scrub_stripe_get_page_offset(stripe, sector_nr);
1208 ASSERT(test_bit(sector_nr, &stripe->extent_sector_bitmap));
1212 scrub_submit_write_bio(sctx, stripe, bbio, dev_replace);
1216 bbio = btrfs_bio_alloc(stripe->nr_sectors, REQ_OP_WRITE,
1217 fs_info, scrub_write_endio, stripe);
1218 bbio->bio.bi_iter.bi_sector = (stripe->logical +
1226 scrub_submit_write_bio(sctx, stripe, bbio, dev_replace);
1287 * logical offset. if this is a parity stripe, it will return
1288 * the most left data stripe's logical offset.
1290 * return 0 if it is a data stripe, 1 means parity stripe.
1315 /* Work out the disk rotation on this stripe-set */
1317 /* calculate which stripe this data locates */
1366 * return the extent item. This is for data extent crossing stripe boundary.
1480 struct scrub_stripe *stripe,
1484 for (u64 cur_logical = max(stripe->logical, extent_start);
1485 cur_logical < min(stripe->logical + BTRFS_STRIPE_LEN,
1488 const int nr_sector = (cur_logical - stripe->logical) >>
1491 &stripe->sectors[nr_sector];
1493 set_bit(nr_sector, &stripe->extent_sector_bitmap);
1501 static void scrub_stripe_reset_bitmaps(struct scrub_stripe *stripe)
1503 stripe->extent_sector_bitmap = 0;
1504 stripe->init_error_bitmap = 0;
1505 stripe->init_nr_io_errors = 0;
1506 stripe->init_nr_csum_errors = 0;
1507 stripe->init_nr_meta_errors = 0;
1508 stripe->error_bitmap = 0;
1509 stripe->io_error_bitmap = 0;
1510 stripe->csum_error_bitmap = 0;
1511 stripe->meta_error_bitmap = 0;
1515 * Locate one stripe which has at least one extent in its range.
1517 * Return 0 if found such stripe, and store its info into @stripe.
1518 * Return >0 if there is no such stripe in the specified range.
1527 struct scrub_stripe *stripe)
1541 memset(stripe->sectors, 0, sizeof(struct scrub_sector_verification) *
1542 stripe->nr_sectors);
1543 scrub_stripe_reset_bitmaps(stripe);
1556 stripe->nr_meta_extents++;
1558 stripe->nr_data_extents++;
1562 * Round down to stripe boundary.
1567 stripe->logical = round_down(cur_logical - bg->start, BTRFS_STRIPE_LEN) +
1569 stripe->physical = physical + stripe->logical - logical_start;
1570 stripe->dev = dev;
1571 stripe->bg = bg;
1572 stripe->mirror_num = mirror_num;
1573 stripe_end = stripe->logical + BTRFS_STRIPE_LEN - 1;
1575 /* Fill the first extent info into stripe->sectors[] array. */
1576 fill_one_extent_info(fs_info, stripe, extent_start, extent_len,
1593 stripe->nr_meta_extents++;
1595 stripe->nr_data_extents++;
1596 fill_one_extent_info(fs_info, stripe, extent_start, extent_len,
1607 ASSERT(stripe->csums);
1616 stripe->logical, stripe_end,
1617 stripe->csums, &csum_bitmap);
1623 for_each_set_bit(sector_nr, &csum_bitmap, stripe->nr_sectors) {
1624 stripe->sectors[sector_nr].csum = stripe->csums +
1628 set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state);
1633 static void scrub_reset_stripe(struct scrub_stripe *stripe)
1635 scrub_stripe_reset_bitmaps(stripe);
1637 stripe->nr_meta_extents = 0;
1638 stripe->nr_data_extents = 0;
1639 stripe->state = 0;
1641 for (int i = 0; i < stripe->nr_sectors; i++) {
1642 stripe->sectors[i].is_metadata = false;
1643 stripe->sectors[i].csum = NULL;
1644 stripe->sectors[i].generation = 0;
1649 struct scrub_stripe *stripe)
1653 unsigned int nr_sectors = min_t(u64, BTRFS_STRIPE_LEN, stripe->bg->start +
1654 stripe->bg->length - stripe->logical) >>
1656 int mirror = stripe->mirror_num;
1658 ASSERT(stripe->bg);
1659 ASSERT(stripe->mirror_num > 0);
1660 ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state));
1663 scrub_read_endio, stripe);
1665 bbio->bio.bi_iter.bi_sector = stripe->logical >> SECTOR_SHIFT;
1668 struct page *page = scrub_stripe_get_page(stripe, cur);
1669 unsigned int pgoff = scrub_stripe_get_page_offset(stripe, cur);
1676 atomic_inc(&stripe->pending_io);
1685 !stripe->dev->bdev)) {
1686 int num_copies = btrfs_num_copies(fs_info, stripe->bg->start,
1687 stripe->bg->length);
1694 static bool stripe_has_metadata_error(struct scrub_stripe *stripe)
1698 for_each_set_bit(i, &stripe->error_bitmap, stripe->nr_sectors) {
1699 if (stripe->sectors[i].is_metadata) {
1700 struct btrfs_fs_info *fs_info = stripe->bg->fs_info;
1703 "stripe %llu has unrepaired metadata sector at %llu",
1704 stripe->logical,
1705 stripe->logical + (i << fs_info->sectorsize_bits));
1725 struct scrub_stripe *stripe = &sctx->stripes[first_slot + i];
1728 ASSERT(test_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state));
1729 scrub_submit_initial_read(sctx, stripe);
1737 struct scrub_stripe *stripe;
1754 stripe = &sctx->stripes[i];
1756 wait_event(stripe->repair_wait,
1757 test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state));
1775 stripe = &sctx->stripes[i];
1777 ASSERT(stripe->dev == fs_info->dev_replace.srcdev);
1779 bitmap_andnot(&good, &stripe->extent_sector_bitmap,
1780 &stripe->error_bitmap, stripe->nr_sectors);
1781 scrub_write_sectors(sctx, stripe, good, true);
1787 stripe = &sctx->stripes[i];
1789 wait_scrub_stripe_io(stripe);
1790 scrub_reset_stripe(stripe);
1807 struct scrub_stripe *stripe;
1819 stripe = &sctx->stripes[sctx->cur_stripe];
1820 scrub_reset_stripe(stripe);
1823 mirror_num, logical, length, stripe);
1827 *found_logical_ret = stripe->logical;
1856 struct scrub_stripe *stripe;
1866 * For data stripe search, we cannot re-use the same extent/csum paths,
1867 * as the data stripe bytenr may be smaller than previous extent. Thus
1880 stripe = &sctx->raid56_data_stripes[i];
1887 scrub_reset_stripe(stripe);
1888 set_bit(SCRUB_STRIPE_FLAG_NO_REPORT, &stripe->state);
1892 BTRFS_STRIPE_LEN, stripe);
1896 * No extent in this data stripe, need to manually mark them
1900 stripe->logical = full_stripe_start +
1902 stripe->dev = map->stripes[stripe_index].dev;
1903 stripe->mirror_num = 1;
1904 set_bit(SCRUB_STRIPE_FLAG_INITIALIZED, &stripe->state);
1910 stripe = &sctx->raid56_data_stripes[i];
1911 if (!bitmap_empty(&stripe->extent_sector_bitmap, stripe->nr_sectors)) {
1922 stripe = &sctx->raid56_data_stripes[i];
1923 scrub_submit_initial_read(sctx, stripe);
1926 stripe = &sctx->raid56_data_stripes[i];
1928 wait_event(stripe->repair_wait,
1929 test_bit(SCRUB_STRIPE_FLAG_REPAIR_DONE, &stripe->state));
1944 stripe = &sctx->raid56_data_stripes[i];
1948 * As we may hit an empty data stripe while it's missing.
1950 bitmap_and(&error, &stripe->error_bitmap,
1951 &stripe->extent_sector_bitmap, stripe->nr_sectors);
1952 if (!bitmap_empty(&error, stripe->nr_sectors)) {
1954 "unrepaired sectors detected, full stripe %llu data stripe %u errors %*pbl",
1955 full_stripe_start, i, stripe->nr_sectors,
1961 &stripe->extent_sector_bitmap, stripe->nr_sectors);
1964 /* Now we can check and regenerate the P/Q stripe. */
1988 stripe = &sctx->raid56_data_stripes[i];
1990 raid56_parity_cache_data_pages(rbio, stripe->pages,
2007 * (Including all range in SINGLE/DUP/RAID1/RAID1C*, and each stripe in
2075 /* Calculate the full stripe length for simple stripe based profiles */
2084 /* Get the logical bytenr for the stripe */
2101 /* Get the mirror number for the stripe */
2128 * Inside each stripe, RAID0 is just SINGLE, and RAID10 is
2130 * this stripe.
2137 /* Skip to next stripe which belongs to the target device */
2139 /* For physical offset, we just go to next stripe */
2162 /* The logical increment after finishing one stripe */
2213 * mirrored duplication without stripe.
2234 /* Calculate the logical end of the stripe */
2244 * Due to the rotation, for RAID56 it's better to iterate each stripe
2262 * Now we're at a data stripe, scrub each extents in the range.
2265 * stripe it is no different than SINGLE profile.