Lines Matching refs:rbio
22 /* set when additional merges to this rbio are not allowed */
26 * set when this rbio is sitting in the hash, but it is just a cache
136 * helps us decide if the rbio maps to a full
150 * rbio big enough to hold them both and setup their
151 * locations when the rbio is allocated
177 static int __raid56_parity_recover(struct btrfs_raid_bio *rbio);
178 static noinline void finish_rmw(struct btrfs_raid_bio *rbio);
181 static int fail_bio_stripe(struct btrfs_raid_bio *rbio, struct bio *bio);
182 static int fail_rbio_index(struct btrfs_raid_bio *rbio, int failed);
183 static void __free_raid_bio(struct btrfs_raid_bio *rbio);
184 static void index_rbio_pages(struct btrfs_raid_bio *rbio);
185 static int alloc_rbio_pages(struct btrfs_raid_bio *rbio);
187 static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio,
191 static void start_async_work(struct btrfs_raid_bio *rbio, btrfs_func_t work_func)
193 btrfs_init_work(&rbio->work, work_func, NULL, NULL);
194 btrfs_queue_work(rbio->fs_info->rmw_workers, &rbio->work);
242 * caching an rbio means to copy anything from the
250 static void cache_rbio_pages(struct btrfs_raid_bio *rbio)
257 ret = alloc_rbio_pages(rbio);
261 for (i = 0; i < rbio->nr_pages; i++) {
262 if (!rbio->bio_pages[i])
265 s = kmap(rbio->bio_pages[i]);
266 d = kmap(rbio->stripe_pages[i]);
270 kunmap(rbio->bio_pages[i]);
271 kunmap(rbio->stripe_pages[i]);
272 SetPageUptodate(rbio->stripe_pages[i]);
274 set_bit(RBIO_CACHE_READY_BIT, &rbio->flags);
280 static int rbio_bucket(struct btrfs_raid_bio *rbio)
282 u64 num = rbio->bbio->raid_map[0];
296 * stealing an rbio means taking all the uptodate pages from the stripe
297 * array in the source rbio and putting them into the destination rbio
346 static void __remove_rbio_from_cache(struct btrfs_raid_bio *rbio)
348 int bucket = rbio_bucket(rbio);
356 if (!test_bit(RBIO_CACHE_BIT, &rbio->flags))
359 table = rbio->fs_info->stripe_hash_table;
371 spin_lock(&rbio->bio_list_lock);
373 if (test_and_clear_bit(RBIO_CACHE_BIT, &rbio->flags)) {
374 list_del_init(&rbio->stripe_cache);
378 /* if the bio list isn't empty, this rbio is
384 * the rbio from the hash_table, and drop
387 if (bio_list_empty(&rbio->bio_list)) {
388 if (!list_empty(&rbio->hash_list)) {
389 list_del_init(&rbio->hash_list);
390 refcount_dec(&rbio->refs);
391 BUG_ON(!list_empty(&rbio->plug_list));
396 spin_unlock(&rbio->bio_list_lock);
400 __free_raid_bio(rbio);
404 * prune a given rbio from the cache
406 static void remove_rbio_from_cache(struct btrfs_raid_bio *rbio)
411 if (!test_bit(RBIO_CACHE_BIT, &rbio->flags))
414 table = rbio->fs_info->stripe_hash_table;
417 __remove_rbio_from_cache(rbio);
428 struct btrfs_raid_bio *rbio;
434 rbio = list_entry(table->stripe_cache.next,
437 __remove_rbio_from_cache(rbio);
456 * insert an rbio into the stripe cache. It
460 * If this rbio was already cached, it gets
463 * If the size of the rbio cache is too big, we
466 static void cache_rbio(struct btrfs_raid_bio *rbio)
471 if (!test_bit(RBIO_CACHE_READY_BIT, &rbio->flags))
474 table = rbio->fs_info->stripe_hash_table;
477 spin_lock(&rbio->bio_list_lock);
480 if (!test_and_set_bit(RBIO_CACHE_BIT, &rbio->flags))
481 refcount_inc(&rbio->refs);
483 if (!list_empty(&rbio->stripe_cache)){
484 list_move(&rbio->stripe_cache, &table->stripe_cache);
486 list_add(&rbio->stripe_cache, &table->stripe_cache);
490 spin_unlock(&rbio->bio_list_lock);
499 if (found != rbio)
527 * Returns true if the bio list inside this rbio covers an entire stripe (no
530 static int rbio_is_full(struct btrfs_raid_bio *rbio)
533 unsigned long size = rbio->bio_list_bytes;
536 spin_lock_irqsave(&rbio->bio_list_lock, flags);
537 if (size != rbio->nr_data * rbio->stripe_len)
539 BUG_ON(size > rbio->nr_data * rbio->stripe_len);
540 spin_unlock_irqrestore(&rbio->bio_list_lock, flags);
565 * rbio is going to run our IO for us. We can
616 static int rbio_stripe_page_index(struct btrfs_raid_bio *rbio, int stripe,
619 return stripe * rbio->stripe_npages + index;
623 * these are just the pages from the rbio array, not from anything
626 static struct page *rbio_stripe_page(struct btrfs_raid_bio *rbio, int stripe,
629 return rbio->stripe_pages[rbio_stripe_page_index(rbio, stripe, index)];
635 static struct page *rbio_pstripe_page(struct btrfs_raid_bio *rbio, int index)
637 return rbio_stripe_page(rbio, rbio->nr_data, index);
644 static struct page *rbio_qstripe_page(struct btrfs_raid_bio *rbio, int index)
646 if (rbio->nr_data + 1 == rbio->real_stripes)
648 return rbio_stripe_page(rbio, rbio->nr_data + 1, index);
655 * 1) Nobody has the stripe locked yet. The rbio is given
660 * with the lock owner. The rbio is freed and the IO will
661 * start automatically along with the existing rbio. 1 is returned.
664 * The rbio is added to the lock owner's plug list, or merged into
665 * an rbio already on the plug list. When the lock owner unlocks,
666 * the next rbio on the list is run and the IO is started automatically.
669 * If we return 0, the caller still owns the rbio and must continue with
670 * IO submission. If we return 1, the caller must assume the rbio has
673 static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio)
683 h = rbio->fs_info->stripe_hash_table->table + rbio_bucket(rbio);
687 if (cur->bbio->raid_map[0] != rbio->bbio->raid_map[0])
692 /* Can we steal this cached rbio's pages? */
700 steal_rbio(cur, rbio);
708 if (rbio_can_merge(cur, rbio)) {
709 merge_rbio(cur, rbio);
711 freeit = rbio;
718 * We couldn't merge with the running rbio, see if we can merge
723 if (rbio_can_merge(pending, rbio)) {
724 merge_rbio(pending, rbio);
726 freeit = rbio;
733 * No merging, put us on the tail of the plug list, our rbio
734 * will be started with the currently running rbio unlocks
736 list_add_tail(&rbio->plug_list, &cur->plug_list);
742 refcount_inc(&rbio->refs);
743 list_add(&rbio->hash_list, &h->hash_list);
757 static noinline void unlock_stripe(struct btrfs_raid_bio *rbio)
764 bucket = rbio_bucket(rbio);
765 h = rbio->fs_info->stripe_hash_table->table + bucket;
767 if (list_empty(&rbio->plug_list))
768 cache_rbio(rbio);
771 spin_lock(&rbio->bio_list_lock);
773 if (!list_empty(&rbio->hash_list)) {
776 * to perform, just leave this rbio here for others
779 if (list_empty(&rbio->plug_list) &&
780 test_bit(RBIO_CACHE_BIT, &rbio->flags)) {
782 clear_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
783 BUG_ON(!bio_list_empty(&rbio->bio_list));
787 list_del_init(&rbio->hash_list);
788 refcount_dec(&rbio->refs);
795 if (!list_empty(&rbio->plug_list)) {
797 struct list_head *head = rbio->plug_list.next;
802 list_del_init(&rbio->plug_list);
806 spin_unlock(&rbio->bio_list_lock);
812 steal_rbio(rbio, next);
815 steal_rbio(rbio, next);
818 steal_rbio(rbio, next);
826 spin_unlock(&rbio->bio_list_lock);
831 remove_rbio_from_cache(rbio);
834 static void __free_raid_bio(struct btrfs_raid_bio *rbio)
838 if (!refcount_dec_and_test(&rbio->refs))
841 WARN_ON(!list_empty(&rbio->stripe_cache));
842 WARN_ON(!list_empty(&rbio->hash_list));
843 WARN_ON(!bio_list_empty(&rbio->bio_list));
845 for (i = 0; i < rbio->nr_pages; i++) {
846 if (rbio->stripe_pages[i]) {
847 __free_page(rbio->stripe_pages[i]);
848 rbio->stripe_pages[i] = NULL;
852 btrfs_put_bbio(rbio->bbio);
853 kfree(rbio);
870 * this frees the rbio and runs through all the bios in the
873 static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, blk_status_t err)
875 struct bio *cur = bio_list_get(&rbio->bio_list);
878 if (rbio->generic_bio_cnt)
879 btrfs_bio_counter_sub(rbio->fs_info, rbio->generic_bio_cnt);
881 * Clear the data bitmap, as the rbio may be cached for later usage.
885 bitmap_clear(rbio->dbitmap, 0, rbio->stripe_npages);
888 * At this moment, rbio->bio_list is empty, however since rbio does not
889 * always have RBIO_RMW_LOCKED_BIT set and rbio is still linked on the
890 * hash list, rbio may be merged with others so that rbio->bio_list
892 * Once unlock_stripe() is done, rbio->bio_list will not be updated any
895 unlock_stripe(rbio);
896 extra = bio_list_get(&rbio->bio_list);
897 __free_raid_bio(rbio);
910 struct btrfs_raid_bio *rbio = bio->bi_private;
915 fail_bio_stripe(rbio, bio);
919 if (!atomic_dec_and_test(&rbio->stripes_pending))
925 max_errors = (rbio->operation == BTRFS_RBIO_PARITY_SCRUB) ?
926 0 : rbio->bbio->max_errors;
927 if (atomic_read(&rbio->error) > max_errors)
930 rbio_orig_end_io(rbio, err);
935 * any pages it included, and then use the rbio for everything
938 * or the rbio.
949 static struct page *page_in_rbio(struct btrfs_raid_bio *rbio,
955 chunk_page = index * (rbio->stripe_len >> PAGE_SHIFT) + pagenr;
957 spin_lock_irq(&rbio->bio_list_lock);
958 p = rbio->bio_pages[chunk_page];
959 spin_unlock_irq(&rbio->bio_list_lock);
964 return rbio->stripe_pages[chunk_page];
978 * this does not allocate any pages for rbio->pages.
984 struct btrfs_raid_bio *rbio;
991 rbio = kzalloc(sizeof(*rbio) +
992 sizeof(*rbio->stripe_pages) * num_pages +
993 sizeof(*rbio->bio_pages) * num_pages +
994 sizeof(*rbio->finish_pointers) * real_stripes +
995 sizeof(*rbio->dbitmap) * BITS_TO_LONGS(stripe_npages) +
996 sizeof(*rbio->finish_pbitmap) *
999 if (!rbio)
1002 bio_list_init(&rbio->bio_list);
1003 INIT_LIST_HEAD(&rbio->plug_list);
1004 spin_lock_init(&rbio->bio_list_lock);
1005 INIT_LIST_HEAD(&rbio->stripe_cache);
1006 INIT_LIST_HEAD(&rbio->hash_list);
1007 rbio->bbio = bbio;
1008 rbio->fs_info = fs_info;
1009 rbio->stripe_len = stripe_len;
1010 rbio->nr_pages = num_pages;
1011 rbio->real_stripes = real_stripes;
1012 rbio->stripe_npages = stripe_npages;
1013 rbio->faila = -1;
1014 rbio->failb = -1;
1015 refcount_set(&rbio->refs, 1);
1016 atomic_set(&rbio->error, 0);
1017 atomic_set(&rbio->stripes_pending, 0);
1021 * memory we allocated past the end of the rbio
1023 p = rbio + 1;
1028 CONSUME_ALLOC(rbio->stripe_pages, num_pages);
1029 CONSUME_ALLOC(rbio->bio_pages, num_pages);
1030 CONSUME_ALLOC(rbio->finish_pointers, real_stripes);
1031 CONSUME_ALLOC(rbio->dbitmap, BITS_TO_LONGS(stripe_npages));
1032 CONSUME_ALLOC(rbio->finish_pbitmap, BITS_TO_LONGS(stripe_npages));
1042 rbio->nr_data = nr_data;
1043 return rbio;
1047 static int alloc_rbio_pages(struct btrfs_raid_bio *rbio)
1052 for (i = 0; i < rbio->nr_pages; i++) {
1053 if (rbio->stripe_pages[i])
1058 rbio->stripe_pages[i] = page;
1064 static int alloc_rbio_parity_pages(struct btrfs_raid_bio *rbio)
1069 i = rbio_stripe_page_index(rbio, rbio->nr_data, 0);
1071 for (; i < rbio->nr_pages; i++) {
1072 if (rbio->stripe_pages[i])
1077 rbio->stripe_pages[i] = page;
1087 static int rbio_add_io_page(struct btrfs_raid_bio *rbio,
1100 stripe = &rbio->bbio->stripes[stripe_nr];
1105 return fail_rbio_index(rbio, stripe_nr);
1144 static void validate_rbio_for_rmw(struct btrfs_raid_bio *rbio)
1146 if (rbio->faila >= 0 || rbio->failb >= 0) {
1147 BUG_ON(rbio->faila == rbio->real_stripes - 1);
1148 __raid56_parity_recover(rbio);
1150 finish_rmw(rbio);
1162 static void index_rbio_pages(struct btrfs_raid_bio *rbio)
1169 spin_lock_irq(&rbio->bio_list_lock);
1170 bio_list_for_each(bio, &rbio->bio_list) {
1176 stripe_offset = start - rbio->bbio->raid_map[0];
1183 rbio->bio_pages[page_index + i] = bvec.bv_page;
1187 spin_unlock_irq(&rbio->bio_list_lock);
1198 static noinline void finish_rmw(struct btrfs_raid_bio *rbio)
1200 struct btrfs_bio *bbio = rbio->bbio;
1201 void **pointers = rbio->finish_pointers;
1202 int nr_data = rbio->nr_data;
1212 if (rbio->real_stripes - rbio->nr_data == 1)
1214 else if (rbio->real_stripes - rbio->nr_data == 2)
1220 ASSERT(bitmap_weight(rbio->dbitmap, rbio->stripe_npages));
1230 spin_lock_irq(&rbio->bio_list_lock);
1231 set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
1232 spin_unlock_irq(&rbio->bio_list_lock);
1234 atomic_set(&rbio->error, 0);
1245 index_rbio_pages(rbio);
1246 if (!rbio_is_full(rbio))
1247 cache_rbio_pages(rbio);
1249 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags);
1251 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) {
1255 p = page_in_rbio(rbio, stripe, pagenr, 0);
1260 p = rbio_pstripe_page(rbio, pagenr);
1270 p = rbio_qstripe_page(rbio, pagenr);
1274 raid6_call.gen_syndrome(rbio->real_stripes, PAGE_SIZE,
1283 for (stripe = 0; stripe < rbio->real_stripes; stripe++)
1284 kunmap(page_in_rbio(rbio, stripe, pagenr, 0));
1289 * higher layers (the bio_list in our rbio) and our p/q. Ignore
1292 for (stripe = 0; stripe < rbio->real_stripes; stripe++) {
1293 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) {
1297 if (!test_bit(pagenr, rbio->dbitmap))
1300 if (stripe < rbio->nr_data) {
1301 page = page_in_rbio(rbio, stripe, pagenr, 1);
1305 page = rbio_stripe_page(rbio, stripe, pagenr);
1308 ret = rbio_add_io_page(rbio, &bio_list,
1309 page, stripe, pagenr, rbio->stripe_len);
1318 for (stripe = 0; stripe < rbio->real_stripes; stripe++) {
1322 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) {
1326 if (!test_bit(pagenr, rbio->dbitmap))
1329 if (stripe < rbio->nr_data) {
1330 page = page_in_rbio(rbio, stripe, pagenr, 1);
1334 page = rbio_stripe_page(rbio, stripe, pagenr);
1337 ret = rbio_add_io_page(rbio, &bio_list, page,
1338 rbio->bbio->tgtdev_map[stripe],
1339 pagenr, rbio->stripe_len);
1346 atomic_set(&rbio->stripes_pending, bio_list_size(&bio_list));
1347 BUG_ON(atomic_read(&rbio->stripes_pending) == 0);
1350 bio->bi_private = rbio;
1359 rbio_orig_end_io(rbio, BLK_STS_IOERR);
1370 static int find_bio_stripe(struct btrfs_raid_bio *rbio,
1379 for (i = 0; i < rbio->bbio->num_stripes; i++) {
1380 stripe = &rbio->bbio->stripes[i];
1381 if (in_range(physical, stripe->physical, rbio->stripe_len) &&
1396 static int find_logical_bio_stripe(struct btrfs_raid_bio *rbio,
1402 for (i = 0; i < rbio->nr_data; i++) {
1403 u64 stripe_start = rbio->bbio->raid_map[i];
1405 if (in_range(logical, stripe_start, rbio->stripe_len))
1414 static int fail_rbio_index(struct btrfs_raid_bio *rbio, int failed)
1419 spin_lock_irqsave(&rbio->bio_list_lock, flags);
1422 if (rbio->faila == failed || rbio->failb == failed)
1425 if (rbio->faila == -1) {
1426 /* first failure on this rbio */
1427 rbio->faila = failed;
1428 atomic_inc(&rbio->error);
1429 } else if (rbio->failb == -1) {
1430 /* second failure on this rbio */
1431 rbio->failb = failed;
1432 atomic_inc(&rbio->error);
1437 spin_unlock_irqrestore(&rbio->bio_list_lock, flags);
1446 static int fail_bio_stripe(struct btrfs_raid_bio *rbio,
1449 int failed = find_bio_stripe(rbio, bio);
1454 return fail_rbio_index(rbio, failed);
1459 * rbio pages, nothing that comes in from the higher layers
1482 struct btrfs_raid_bio *rbio = bio->bi_private;
1485 fail_bio_stripe(rbio, bio);
1491 if (!atomic_dec_and_test(&rbio->stripes_pending))
1494 if (atomic_read(&rbio->error) > rbio->bbio->max_errors)
1502 validate_rbio_for_rmw(rbio);
1507 rbio_orig_end_io(rbio, BLK_STS_IOERR);
1514 static int raid56_rmw_stripe(struct btrfs_raid_bio *rbio)
1525 ret = alloc_rbio_pages(rbio);
1529 index_rbio_pages(rbio);
1531 atomic_set(&rbio->error, 0);
1536 for (stripe = 0; stripe < rbio->nr_data; stripe++) {
1537 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) {
1541 * the rbio and read them from the disk. If
1545 page = page_in_rbio(rbio, stripe, pagenr, 1);
1549 page = rbio_stripe_page(rbio, stripe, pagenr);
1557 ret = rbio_add_io_page(rbio, &bio_list, page,
1558 stripe, pagenr, rbio->stripe_len);
1579 atomic_set(&rbio->stripes_pending, bios_to_read);
1581 bio->bi_private = rbio;
1585 btrfs_bio_wq_end_io(rbio->fs_info, bio, BTRFS_WQ_ENDIO_RAID56);
1593 rbio_orig_end_io(rbio, BLK_STS_IOERR);
1601 validate_rbio_for_rmw(rbio);
1609 static int full_stripe_write(struct btrfs_raid_bio *rbio)
1613 ret = alloc_rbio_parity_pages(rbio);
1615 __free_raid_bio(rbio);
1619 ret = lock_stripe_add(rbio);
1621 finish_rmw(rbio);
1628 * rbio before calculating new parity
1630 static int partial_stripe_write(struct btrfs_raid_bio *rbio)
1634 ret = lock_stripe_add(rbio);
1636 start_async_work(rbio, rmw_work);
1646 static int __raid56_parity_write(struct btrfs_raid_bio *rbio)
1649 if (!rbio_is_full(rbio))
1650 return partial_stripe_write(rbio);
1651 return full_stripe_write(rbio);
1754 /* Add the original bio into rbio->bio_list, and update rbio::dbitmap. */
1755 static void rbio_add_bio(struct btrfs_raid_bio *rbio, struct bio *orig_bio)
1757 const struct btrfs_fs_info *fs_info = rbio->fs_info;
1759 const u64 full_stripe_start = rbio->bbio->raid_map[0];
1766 rbio->nr_data * rbio->stripe_len);
1768 bio_list_add(&rbio->bio_list, orig_bio);
1769 rbio->bio_list_bytes += orig_bio->bi_iter.bi_size;
1775 PAGE_SHIFT) % rbio->stripe_npages;
1777 set_bit(bit, rbio->dbitmap);
1787 struct btrfs_raid_bio *rbio;
1792 rbio = alloc_rbio(fs_info, bbio, stripe_len);
1793 if (IS_ERR(rbio)) {
1795 return PTR_ERR(rbio);
1797 rbio->operation = BTRFS_RBIO_WRITE;
1798 rbio_add_bio(rbio, bio);
1801 rbio->generic_bio_cnt = 1;
1807 if (rbio_is_full(rbio)) {
1808 ret = full_stripe_write(rbio);
1821 list_add_tail(&rbio->plug_list, &plug->rbio_list);
1824 ret = __raid56_parity_write(rbio);
1836 static void __raid_recover_end_io(struct btrfs_raid_bio *rbio)
1845 pointers = kcalloc(rbio->real_stripes, sizeof(void *), GFP_NOFS);
1851 faila = rbio->faila;
1852 failb = rbio->failb;
1854 if (rbio->operation == BTRFS_RBIO_READ_REBUILD ||
1855 rbio->operation == BTRFS_RBIO_REBUILD_MISSING) {
1856 spin_lock_irq(&rbio->bio_list_lock);
1857 set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
1858 spin_unlock_irq(&rbio->bio_list_lock);
1861 index_rbio_pages(rbio);
1863 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) {
1868 if (rbio->operation == BTRFS_RBIO_PARITY_SCRUB &&
1869 !test_bit(pagenr, rbio->dbitmap))
1875 for (stripe = 0; stripe < rbio->real_stripes; stripe++) {
1880 if ((rbio->operation == BTRFS_RBIO_READ_REBUILD ||
1881 rbio->operation == BTRFS_RBIO_REBUILD_MISSING) &&
1883 page = page_in_rbio(rbio, stripe, pagenr, 0);
1885 page = rbio_stripe_page(rbio, stripe, pagenr);
1891 if (rbio->bbio->map_type & BTRFS_BLOCK_GROUP_RAID6) {
1897 if (faila == rbio->nr_data) {
1923 if (rbio->bbio->raid_map[failb] == RAID6_Q_STRIPE) {
1924 if (rbio->bbio->raid_map[faila] ==
1936 if (rbio->bbio->raid_map[failb] == RAID5_P_STRIPE) {
1937 raid6_datap_recov(rbio->real_stripes,
1940 raid6_2data_recov(rbio->real_stripes,
1951 copy_page(pointers[faila], pointers[rbio->nr_data]);
1955 for (stripe = faila; stripe < rbio->nr_data - 1; stripe++)
1957 pointers[rbio->nr_data - 1] = p;
1960 run_xor(pointers, rbio->nr_data - 1, PAGE_SIZE);
1963 * and set all of our private rbio pages in the
1968 if (rbio->operation == BTRFS_RBIO_WRITE) {
1969 for (i = 0; i < rbio->stripe_npages; i++) {
1971 page = rbio_stripe_page(rbio, faila, i);
1975 page = rbio_stripe_page(rbio, failb, i);
1980 for (stripe = 0; stripe < rbio->real_stripes; stripe++) {
1985 if ((rbio->operation == BTRFS_RBIO_READ_REBUILD ||
1986 rbio->operation == BTRFS_RBIO_REBUILD_MISSING) &&
1988 page = page_in_rbio(rbio, stripe, pagenr, 0);
1990 page = rbio_stripe_page(rbio, stripe, pagenr);
2003 * valid rbio which is consistent with ondisk content, thus such a
2004 * valid rbio can be cached to avoid further disk reads.
2006 if (rbio->operation == BTRFS_RBIO_READ_REBUILD ||
2007 rbio->operation == BTRFS_RBIO_REBUILD_MISSING) {
2009 * - In case of two failures, where rbio->failb != -1:
2011 * Do not cache this rbio since the above read reconstruction
2015 * may steal stripe_pages from this rbio and end up with
2018 * - In case of single failure, where rbio->failb == -1:
2020 * Cache this rbio iff the above read reconstruction is
2023 if (err == BLK_STS_OK && rbio->failb < 0)
2024 cache_rbio_pages(rbio);
2026 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags);
2028 rbio_orig_end_io(rbio, err);
2030 rbio->faila = -1;
2031 rbio->failb = -1;
2033 if (rbio->operation == BTRFS_RBIO_WRITE)
2034 finish_rmw(rbio);
2035 else if (rbio->operation == BTRFS_RBIO_PARITY_SCRUB)
2036 finish_parity_scrub(rbio, 0);
2040 rbio_orig_end_io(rbio, err);
2050 struct btrfs_raid_bio *rbio = bio->bi_private;
2057 fail_bio_stripe(rbio, bio);
2062 if (!atomic_dec_and_test(&rbio->stripes_pending))
2065 if (atomic_read(&rbio->error) > rbio->bbio->max_errors)
2066 rbio_orig_end_io(rbio, BLK_STS_IOERR);
2068 __raid_recover_end_io(rbio);
2079 static int __raid56_parity_recover(struct btrfs_raid_bio *rbio)
2090 ret = alloc_rbio_pages(rbio);
2094 atomic_set(&rbio->error, 0);
2104 for (stripe = 0; stripe < rbio->real_stripes; stripe++) {
2105 if (rbio->faila == stripe || rbio->failb == stripe) {
2106 atomic_inc(&rbio->error);
2110 for (pagenr = 0; pagenr < rbio->stripe_npages; pagenr++) {
2111 ret = rbio_add_io_page(rbio, &bio_list,
2112 rbio_stripe_page(rbio, stripe, pagenr),
2113 stripe, pagenr, rbio->stripe_len);
2126 if (atomic_read(&rbio->error) <= rbio->bbio->max_errors) {
2127 __raid_recover_end_io(rbio);
2138 atomic_set(&rbio->stripes_pending, bios_to_read);
2140 bio->bi_private = rbio;
2144 btrfs_bio_wq_end_io(rbio->fs_info, bio, BTRFS_WQ_ENDIO_RAID56);
2152 if (rbio->operation == BTRFS_RBIO_READ_REBUILD ||
2153 rbio->operation == BTRFS_RBIO_REBUILD_MISSING)
2154 rbio_orig_end_io(rbio, BLK_STS_IOERR);
2172 struct btrfs_raid_bio *rbio;
2180 rbio = alloc_rbio(fs_info, bbio, stripe_len);
2181 if (IS_ERR(rbio)) {
2184 return PTR_ERR(rbio);
2187 rbio->operation = BTRFS_RBIO_READ_REBUILD;
2188 rbio_add_bio(rbio, bio);
2190 rbio->faila = find_logical_bio_stripe(rbio, bio);
2191 if (rbio->faila == -1) {
2198 kfree(rbio);
2204 rbio->generic_bio_cnt = 1;
2220 rbio->failb = rbio->real_stripes - (mirror_num - 1);
2221 ASSERT(rbio->failb > 0);
2222 if (rbio->failb <= rbio->faila)
2223 rbio->failb--;
2226 ret = lock_stripe_add(rbio);
2236 __raid56_parity_recover(rbio);
2238 * our rbio has been added to the list of
2248 struct btrfs_raid_bio *rbio;
2250 rbio = container_of(work, struct btrfs_raid_bio, work);
2251 raid56_rmw_stripe(rbio);
2256 struct btrfs_raid_bio *rbio;
2258 rbio = container_of(work, struct btrfs_raid_bio, work);
2259 __raid56_parity_recover(rbio);
2278 struct btrfs_raid_bio *rbio;
2281 rbio = alloc_rbio(fs_info, bbio, stripe_len);
2282 if (IS_ERR(rbio))
2284 bio_list_add(&rbio->bio_list, bio);
2287 * and make the scrub rbio is similar to the other types
2290 rbio->operation = BTRFS_RBIO_PARITY_SCRUB;
2297 for (i = rbio->nr_data; i < rbio->real_stripes; i++) {
2299 rbio->scrubp = i;
2303 ASSERT(i < rbio->real_stripes);
2307 ASSERT(rbio->stripe_npages == stripe_nsectors);
2308 bitmap_copy(rbio->dbitmap, dbitmap, stripe_nsectors);
2314 rbio->generic_bio_cnt = 1;
2316 return rbio;
2320 void raid56_add_scrub_pages(struct btrfs_raid_bio *rbio, struct page *page,
2326 ASSERT(logical >= rbio->bbio->raid_map[0]);
2327 ASSERT(logical + PAGE_SIZE <= rbio->bbio->raid_map[0] +
2328 rbio->stripe_len * rbio->nr_data);
2329 stripe_offset = (int)(logical - rbio->bbio->raid_map[0]);
2331 rbio->bio_pages[index] = page;
2338 static int alloc_rbio_essential_pages(struct btrfs_raid_bio *rbio)
2345 for_each_set_bit(bit, rbio->dbitmap, rbio->stripe_npages) {
2346 for (i = 0; i < rbio->real_stripes; i++) {
2347 index = i * rbio->stripe_npages + bit;
2348 if (rbio->stripe_pages[index])
2354 rbio->stripe_pages[index] = page;
2360 static noinline void finish_parity_scrub(struct btrfs_raid_bio *rbio,
2363 struct btrfs_bio *bbio = rbio->bbio;
2364 void **pointers = rbio->finish_pointers;
2365 unsigned long *pbitmap = rbio->finish_pbitmap;
2366 int nr_data = rbio->nr_data;
2379 if (rbio->real_stripes - rbio->nr_data == 1)
2381 else if (rbio->real_stripes - rbio->nr_data == 2)
2386 if (bbio->num_tgtdevs && bbio->tgtdev_map[rbio->scrubp]) {
2388 bitmap_copy(pbitmap, rbio->dbitmap, rbio->stripe_npages);
2396 clear_bit(RBIO_CACHE_READY_BIT, &rbio->flags);
2414 pointers[rbio->real_stripes - 1] = kmap(q_page);
2417 atomic_set(&rbio->error, 0);
2422 for_each_set_bit(pagenr, rbio->dbitmap, rbio->stripe_npages) {
2427 p = page_in_rbio(rbio, stripe, pagenr, 0);
2433 raid6_call.gen_syndrome(rbio->real_stripes, PAGE_SIZE,
2442 p = rbio_stripe_page(rbio, rbio->scrubp, pagenr);
2444 if (memcmp(parity, pointers[rbio->scrubp], PAGE_SIZE))
2445 copy_page(parity, pointers[rbio->scrubp]);
2448 bitmap_clear(rbio->dbitmap, pagenr, 1);
2452 kunmap(page_in_rbio(rbio, stripe, pagenr, 0));
2465 * higher layers (the bio_list in our rbio) and our p/q. Ignore
2468 for_each_set_bit(pagenr, rbio->dbitmap, rbio->stripe_npages) {
2471 page = rbio_stripe_page(rbio, rbio->scrubp, pagenr);
2472 ret = rbio_add_io_page(rbio, &bio_list,
2473 page, rbio->scrubp, pagenr, rbio->stripe_len);
2481 for_each_set_bit(pagenr, pbitmap, rbio->stripe_npages) {
2484 page = rbio_stripe_page(rbio, rbio->scrubp, pagenr);
2485 ret = rbio_add_io_page(rbio, &bio_list, page,
2486 bbio->tgtdev_map[rbio->scrubp],
2487 pagenr, rbio->stripe_len);
2496 rbio_orig_end_io(rbio, BLK_STS_OK);
2500 atomic_set(&rbio->stripes_pending, nr_data);
2503 bio->bi_private = rbio;
2512 rbio_orig_end_io(rbio, BLK_STS_IOERR);
2518 static inline int is_data_stripe(struct btrfs_raid_bio *rbio, int stripe)
2520 if (stripe >= 0 && stripe < rbio->nr_data)
2532 static void validate_rbio_for_parity_scrub(struct btrfs_raid_bio *rbio)
2534 if (atomic_read(&rbio->error) > rbio->bbio->max_errors)
2537 if (rbio->faila >= 0 || rbio->failb >= 0) {
2540 if (is_data_stripe(rbio, rbio->faila))
2542 else if (is_parity_stripe(rbio->faila))
2543 failp = rbio->faila;
2545 if (is_data_stripe(rbio, rbio->failb))
2547 else if (is_parity_stripe(rbio->failb))
2548 failp = rbio->failb;
2555 if (dfail > rbio->bbio->max_errors - 1)
2563 finish_parity_scrub(rbio, 0);
2573 if (failp != rbio->scrubp)
2576 __raid_recover_end_io(rbio);
2578 finish_parity_scrub(rbio, 1);
2583 rbio_orig_end_io(rbio, BLK_STS_IOERR);
2596 struct btrfs_raid_bio *rbio = bio->bi_private;
2599 fail_bio_stripe(rbio, bio);
2605 if (!atomic_dec_and_test(&rbio->stripes_pending))
2613 validate_rbio_for_parity_scrub(rbio);
2616 static void raid56_parity_scrub_stripe(struct btrfs_raid_bio *rbio)
2627 ret = alloc_rbio_essential_pages(rbio);
2631 atomic_set(&rbio->error, 0);
2636 for (stripe = 0; stripe < rbio->real_stripes; stripe++) {
2637 for_each_set_bit(pagenr, rbio->dbitmap, rbio->stripe_npages) {
2641 * the rbio and read them from the disk. If
2645 page = page_in_rbio(rbio, stripe, pagenr, 1);
2649 page = rbio_stripe_page(rbio, stripe, pagenr);
2657 ret = rbio_add_io_page(rbio, &bio_list, page,
2658 stripe, pagenr, rbio->stripe_len);
2679 atomic_set(&rbio->stripes_pending, bios_to_read);
2681 bio->bi_private = rbio;
2685 btrfs_bio_wq_end_io(rbio->fs_info, bio, BTRFS_WQ_ENDIO_RAID56);
2693 rbio_orig_end_io(rbio, BLK_STS_IOERR);
2701 validate_rbio_for_parity_scrub(rbio);
2706 struct btrfs_raid_bio *rbio;
2708 rbio = container_of(work, struct btrfs_raid_bio, work);
2709 raid56_parity_scrub_stripe(rbio);
2712 void raid56_parity_submit_scrub_rbio(struct btrfs_raid_bio *rbio)
2714 if (!lock_stripe_add(rbio))
2715 start_async_work(rbio, scrub_parity_work);
2724 struct btrfs_raid_bio *rbio;
2726 rbio = alloc_rbio(fs_info, bbio, length);
2727 if (IS_ERR(rbio))
2730 rbio->operation = BTRFS_RBIO_REBUILD_MISSING;
2731 bio_list_add(&rbio->bio_list, bio);
2734 * and make the scrub rbio is similar to the other types
2738 rbio->faila = find_logical_bio_stripe(rbio, bio);
2739 if (rbio->faila == -1) {
2741 kfree(rbio);
2749 rbio->generic_bio_cnt = 1;
2751 return rbio;
2754 void raid56_submit_missing_rbio(struct btrfs_raid_bio *rbio)
2756 if (!lock_stripe_add(rbio))
2757 start_async_work(rbio, read_rebuild_work);