Lines Matching refs:sh
26 * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
112 static inline int raid6_d0(struct stripe_head *sh)
114 if (sh->ddf_layout)
118 if (sh->qd_idx == sh->disks - 1)
121 return sh->qd_idx + 1;
134 static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
139 if (sh->ddf_layout)
141 if (idx == sh->pd_idx)
143 if (idx == sh->qd_idx)
145 if (!sh->ddf_layout)
152 static int stripe_operations_active(struct stripe_head *sh)
154 return sh->check_state || sh->reconstruct_state ||
155 test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
156 test_bit(STRIPE_COMPUTE_RUN, &sh->state);
159 static bool stripe_is_lowprio(struct stripe_head *sh)
161 return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||
162 test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&
163 !test_bit(STRIPE_R5C_CACHING, &sh->state);
166 static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
168 struct r5conf *conf = sh->raid_conf;
171 int i, cpu = sh->cpu;
175 sh->cpu = cpu;
178 if (list_empty(&sh->lru)) {
181 if (stripe_is_lowprio(sh))
182 list_add_tail(&sh->lru, &group->loprio_list);
184 list_add_tail(&sh->lru, &group->handle_list);
186 sh->group = group;
194 group = conf->worker_groups + cpu_to_group(sh->cpu);
198 queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
205 queue_work_on(sh->cpu, raid5_wq,
212 static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
218 BUG_ON(!list_empty(&sh->lru));
222 for (i = sh->disks; i--; )
223 if (test_bit(R5_InJournal, &sh->dev[i].flags))
232 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||
234 !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {
235 if (test_bit(STRIPE_R5C_CACHING, &sh->state))
236 r5c_make_stripe_write_out(sh);
237 set_bit(STRIPE_HANDLE, &sh->state);
240 if (test_bit(STRIPE_HANDLE, &sh->state)) {
241 if (test_bit(STRIPE_DELAYED, &sh->state) &&
242 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
243 list_add_tail(&sh->lru, &conf->delayed_list);
244 else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
245 sh->bm_seq - conf->seq_write > 0)
246 list_add_tail(&sh->lru, &conf->bitmap_list);
248 clear_bit(STRIPE_DELAYED, &sh->state);
249 clear_bit(STRIPE_BIT_DELAY, &sh->state);
251 if (stripe_is_lowprio(sh))
252 list_add_tail(&sh->lru,
255 list_add_tail(&sh->lru,
258 raid5_wakeup_stripe_thread(sh);
264 BUG_ON(stripe_operations_active(sh));
265 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
270 if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
272 list_add_tail(&sh->lru, temp_inactive_list);
274 WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));
276 list_add_tail(&sh->lru, temp_inactive_list);
279 if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))
281 if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
283 list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);
291 list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);
297 static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
300 if (atomic_dec_and_test(&sh->count))
301 do_release_stripe(conf, sh, temp_inactive_list);
357 struct stripe_head *sh, *t;
363 llist_for_each_entry_safe(sh, t, head, release_list) {
366 /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
368 clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
374 hash = sh->hash_lock_index;
375 __release_stripe(conf, sh, &temp_inactive_list[hash]);
382 void raid5_release_stripe(struct stripe_head *sh)
384 struct r5conf *conf = sh->raid_conf;
392 if (atomic_add_unless(&sh->count, -1, 1))
396 test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
398 wakeup = llist_add(&sh->release_list, &conf->released_stripes);
404 if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {
406 hash = sh->hash_lock_index;
407 do_release_stripe(conf, sh, &list);
413 static inline void remove_hash(struct stripe_head *sh)
416 (unsigned long long)sh->sector);
418 hlist_del_init(&sh->hash);
421 static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
423 struct hlist_head *hp = stripe_hash(conf, sh->sector);
426 (unsigned long long)sh->sector);
428 hlist_add_head(&sh->hash, hp);
434 struct stripe_head *sh = NULL;
440 sh = list_entry(first, struct stripe_head, lru);
442 remove_hash(sh);
444 BUG_ON(hash != sh->hash_lock_index);
448 return sh;
452 static void free_stripe_pages(struct stripe_head *sh)
458 if (!sh->pages)
461 for (i = 0; i < sh->nr_pages; i++) {
462 p = sh->pages[i];
465 sh->pages[i] = NULL;
469 static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)
474 for (i = 0; i < sh->nr_pages; i++) {
476 if (sh->pages[i])
481 free_stripe_pages(sh);
484 sh->pages[i] = p;
490 init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)
494 if (sh->pages)
497 /* Each of the sh->dev[i] need one conf->stripe_size */
501 sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
502 if (!sh->pages)
504 sh->nr_pages = nr_pages;
505 sh->stripes_per_page = cnt;
510 static void shrink_buffers(struct stripe_head *sh)
513 int num = sh->raid_conf->pool_size;
519 WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
520 p = sh->dev[i].page;
523 sh->dev[i].page = NULL;
528 sh->dev[i].page = NULL;
529 free_stripe_pages(sh); /* Free pages */
533 static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
536 int num = sh->raid_conf->pool_size;
545 sh->dev[i].page = page;
546 sh->dev[i].orig_page = page;
547 sh->dev[i].offset = 0;
550 if (alloc_stripe_pages(sh, gfp))
554 sh->dev[i].page = raid5_get_dev_page(sh, i);
555 sh->dev[i].orig_page = sh->dev[i].page;
556 sh->dev[i].offset = raid5_get_page_offset(sh, i);
563 struct stripe_head *sh);
565 static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
567 struct r5conf *conf = sh->raid_conf;
570 BUG_ON(atomic_read(&sh->count) != 0);
571 BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
572 BUG_ON(stripe_operations_active(sh));
573 BUG_ON(sh->batch_head);
579 sh->generation = conf->generation - previous;
580 sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
581 sh->sector = sector;
582 stripe_set_idx(sector, conf, previous, sh);
583 sh->state = 0;
585 for (i = sh->disks; i--; ) {
586 struct r5dev *dev = &sh->dev[i];
591 (unsigned long long)sh->sector, i, dev->toread,
597 dev->sector = raid5_compute_blocknr(sh, i, previous);
601 sh->overwrite_disks = 0;
602 insert_hash(conf, sh);
603 sh->cpu = smp_processor_id();
604 set_bit(STRIPE_BATCH_READY, &sh->state);
610 struct stripe_head *sh;
613 hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
614 if (sh->sector == sector && sh->generation == generation)
615 return sh;
706 struct stripe_head *sh;
718 sh = __find_stripe(conf, sector, conf->generation - previous);
719 if (!sh) {
721 sh = get_free_stripe(conf, hash);
722 if (!sh && !test_bit(R5_DID_ALLOC,
727 if (noblock && sh == NULL)
731 if (!sh) {
746 init_stripe(sh, sector, previous);
747 atomic_inc(&sh->count);
749 } else if (!atomic_inc_not_zero(&sh->count)) {
751 if (!atomic_read(&sh->count)) {
752 if (!test_bit(STRIPE_HANDLE, &sh->state))
754 BUG_ON(list_empty(&sh->lru) &&
755 !test_bit(STRIPE_EXPANDING, &sh->state));
759 list_del_init(&sh->lru);
762 if (sh->group) {
763 sh->group->stripes_cnt--;
764 sh->group = NULL;
767 atomic_inc(&sh->count);
770 } while (sh == NULL);
773 return sh;
776 static bool is_full_stripe_write(struct stripe_head *sh)
778 BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
779 return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
804 static bool stripe_can_batch(struct stripe_head *sh)
806 struct r5conf *conf = sh->raid_conf;
810 return test_bit(STRIPE_BATCH_READY, &sh->state) &&
811 !test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&
812 is_full_stripe_write(sh);
816 static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)
825 tmp_sec = sh->sector;
828 head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);
861 lock_two_stripes(head, sh);
863 if (!stripe_can_batch(head) || !stripe_can_batch(sh))
866 if (sh->batch_head)
870 while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
872 if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||
873 bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))
890 sh->batch_head = head->batch_head;
896 list_add(&sh->batch_list, &head->batch_list);
900 sh->batch_head = head->batch_head;
902 list_add_tail(&sh->batch_list, &head->batch_list);
906 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
911 if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {
912 int seq = sh->bm_seq;
913 if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&
914 sh->batch_head->bm_seq > seq)
915 seq = sh->batch_head->bm_seq;
916 set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);
917 sh->batch_head->bm_seq = seq;
920 atomic_inc(&sh->count);
922 unlock_two_stripes(head, sh);
930 static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
940 if (sh->generation == conf->generation - 1)
1057 static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
1059 struct r5conf *conf = sh->raid_conf;
1060 int i, disks = sh->disks;
1061 struct stripe_head *head_sh = sh;
1067 if (log_stripe(sh, s) == 0)
1078 sh = head_sh;
1079 if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
1081 if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
1083 if (test_bit(R5_Discard, &sh->dev[i].flags))
1085 } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
1088 &sh->dev[i].flags)) {
1093 if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
1097 bi = &sh->dev[i].req;
1098 rbi = &sh->dev[i].rreq; /* For writing to replacement */
1138 int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
1172 set_bit(STRIPE_IO_STARTED, &sh->state);
1179 bi->bi_private = sh;
1182 __func__, (unsigned long long)sh->sector,
1184 atomic_inc(&sh->count);
1185 if (sh != head_sh)
1187 if (use_new_offset(conf, sh))
1188 bi->bi_iter.bi_sector = (sh->sector
1191 bi->bi_iter.bi_sector = (sh->sector
1196 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1197 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1200 test_bit(R5_InJournal, &sh->dev[i].flags))
1206 sh->dev[i].vec.bv_page = sh->dev[i].orig_page;
1208 sh->dev[i].vec.bv_page = sh->dev[i].page;
1211 bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
1213 bi->bi_write_hint = sh->dev[i].write_hint;
1215 sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
1223 set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
1228 sh->dev[i].sector);
1239 set_bit(STRIPE_IO_STARTED, &sh->state);
1245 rbi->bi_private = sh;
1249 __func__, (unsigned long long)sh->sector,
1251 atomic_inc(&sh->count);
1252 if (sh != head_sh)
1254 if (use_new_offset(conf, sh))
1255 rbi->bi_iter.bi_sector = (sh->sector
1258 rbi->bi_iter.bi_sector = (sh->sector
1260 if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
1261 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
1262 sh->dev[i].rvec.bv_page = sh->dev[i].page;
1265 rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
1267 rbi->bi_write_hint = sh->dev[i].write_hint;
1268 sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
1278 sh->dev[i].sector);
1286 set_bit(STRIPE_DEGRADED, &sh->state);
1288 bi->bi_opf, i, (unsigned long long)sh->sector);
1289 clear_bit(R5_LOCKED, &sh->dev[i].flags);
1290 set_bit(STRIPE_HANDLE, &sh->state);
1295 sh = list_first_entry(&sh->batch_list, struct stripe_head,
1297 if (sh != head_sh)
1308 struct stripe_head *sh, int no_skipcopy)
1316 struct r5conf *conf = sh->raid_conf;
1372 struct stripe_head *sh = stripe_head_ref;
1374 struct r5conf *conf = sh->raid_conf;
1377 (unsigned long long)sh->sector);
1380 for (i = sh->disks; i--; ) {
1381 struct r5dev *dev = &sh->dev[i];
1402 clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
1404 set_bit(STRIPE_HANDLE, &sh->state);
1405 raid5_release_stripe(sh);
1408 static void ops_run_biofill(struct stripe_head *sh)
1413 struct r5conf *conf = sh->raid_conf;
1415 BUG_ON(sh->batch_head);
1417 (unsigned long long)sh->sector);
1419 for (i = sh->disks; i--; ) {
1420 struct r5dev *dev = &sh->dev[i];
1423 spin_lock_irq(&sh->stripe_lock);
1426 spin_unlock_irq(&sh->stripe_lock);
1431 dev->sector, tx, sh, 0);
1437 atomic_inc(&sh->count);
1438 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
1442 static void mark_target_uptodate(struct stripe_head *sh, int target)
1449 tgt = &sh->dev[target];
1457 struct stripe_head *sh = stripe_head_ref;
1460 (unsigned long long)sh->sector);
1463 mark_target_uptodate(sh, sh->ops.target);
1464 mark_target_uptodate(sh, sh->ops.target2);
1466 clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
1467 if (sh->check_state == check_state_compute_run)
1468 sh->check_state = check_state_compute_result;
1469 set_bit(STRIPE_HANDLE, &sh->state);
1470 raid5_release_stripe(sh);
1480 static addr_conv_t *to_addr_conv(struct stripe_head *sh,
1483 return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
1490 to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)
1492 return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
1496 ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
1498 int disks = sh->disks;
1500 unsigned int *off_srcs = to_addr_offs(sh, percpu);
1501 int target = sh->ops.target;
1502 struct r5dev *tgt = &sh->dev[target];
1510 BUG_ON(sh->batch_head);
1513 __func__, (unsigned long long)sh->sector, target);
1518 off_srcs[count] = sh->dev[i].offset;
1519 xor_srcs[count++] = sh->dev[i].page;
1523 atomic_inc(&sh->count);
1526 ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
1529 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1532 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1538 * @srcs - (struct page *) array of size sh->disks
1540 * @sh - stripe_head to parse
1549 struct stripe_head *sh,
1552 int disks = sh->disks;
1553 int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
1554 int d0_idx = raid6_d0(sh);
1564 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1565 struct r5dev *dev = &sh->dev[i];
1567 if (i == sh->qd_idx || i == sh->pd_idx ||
1576 srcs[slot] = sh->dev[i].orig_page;
1578 srcs[slot] = sh->dev[i].page;
1584 offs[slot] = sh->dev[i].offset;
1593 ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
1595 int disks = sh->disks;
1597 unsigned int *offs = to_addr_offs(sh, percpu);
1599 int qd_idx = sh->qd_idx;
1608 BUG_ON(sh->batch_head);
1609 if (sh->ops.target < 0)
1610 target = sh->ops.target2;
1611 else if (sh->ops.target2 < 0)
1612 target = sh->ops.target;
1618 __func__, (unsigned long long)sh->sector, target);
1620 tgt = &sh->dev[target];
1625 atomic_inc(&sh->count);
1628 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1632 ops_complete_compute, sh,
1633 to_addr_conv(sh, percpu, 0));
1635 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1642 offs[count] = sh->dev[i].offset;
1643 blocks[count++] = sh->dev[i].page;
1647 NULL, ops_complete_compute, sh,
1648 to_addr_conv(sh, percpu, 0));
1650 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1657 ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
1659 int i, count, disks = sh->disks;
1660 int syndrome_disks = sh->ddf_layout ? disks : disks-2;
1661 int d0_idx = raid6_d0(sh);
1663 int target = sh->ops.target;
1664 int target2 = sh->ops.target2;
1665 struct r5dev *tgt = &sh->dev[target];
1666 struct r5dev *tgt2 = &sh->dev[target2];
1669 unsigned int *offs = to_addr_offs(sh, percpu);
1672 BUG_ON(sh->batch_head);
1674 __func__, (unsigned long long)sh->sector, target, target2);
1689 int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
1691 offs[slot] = sh->dev[i].offset;
1692 blocks[slot] = sh->dev[i].page;
1705 __func__, (unsigned long long)sh->sector, faila, failb);
1707 atomic_inc(&sh->count);
1714 ops_complete_compute, sh,
1715 to_addr_conv(sh, percpu, 0));
1717 RAID5_STRIPE_SIZE(sh->raid_conf),
1723 int qd_idx = sh->qd_idx;
1735 offs[count] = sh->dev[i].offset;
1736 blocks[count++] = sh->dev[i].page;
1738 dest = sh->dev[data_target].page;
1739 dest_off = sh->dev[data_target].offset;
1743 to_addr_conv(sh, percpu, 0));
1745 RAID5_STRIPE_SIZE(sh->raid_conf),
1748 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
1750 ops_complete_compute, sh,
1751 to_addr_conv(sh, percpu, 0));
1753 RAID5_STRIPE_SIZE(sh->raid_conf),
1758 ops_complete_compute, sh,
1759 to_addr_conv(sh, percpu, 0));
1763 RAID5_STRIPE_SIZE(sh->raid_conf),
1769 RAID5_STRIPE_SIZE(sh->raid_conf),
1778 struct stripe_head *sh = stripe_head_ref;
1781 (unsigned long long)sh->sector);
1783 if (r5c_is_writeback(sh->raid_conf->log))
1788 r5c_release_extra_page(sh);
1792 ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
1795 int disks = sh->disks;
1797 unsigned int *off_srcs = to_addr_offs(sh, percpu);
1798 int count = 0, pd_idx = sh->pd_idx, i;
1802 unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
1803 struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
1805 BUG_ON(sh->batch_head);
1807 (unsigned long long)sh->sector);
1810 struct r5dev *dev = &sh->dev[i];
1826 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1828 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1834 ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
1838 unsigned int *offs = to_addr_offs(sh, percpu);
1843 (unsigned long long)sh->sector);
1845 count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
1848 ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
1850 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
1856 ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
1858 struct r5conf *conf = sh->raid_conf;
1859 int disks = sh->disks;
1861 struct stripe_head *head_sh = sh;
1864 (unsigned long long)sh->sector);
1870 sh = head_sh;
1875 dev = &sh->dev[i];
1881 spin_lock_irq(&sh->stripe_lock);
1884 sh->overwrite_disks = 0;
1887 spin_unlock_irq(&sh->stripe_lock);
1901 dev->sector, tx, sh,
1914 sh = list_first_entry(&sh->batch_list,
1917 if (sh == head_sh)
1929 struct stripe_head *sh = stripe_head_ref;
1930 int disks = sh->disks;
1931 int pd_idx = sh->pd_idx;
1932 int qd_idx = sh->qd_idx;
1937 (unsigned long long)sh->sector);
1940 fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
1941 sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
1942 discard |= test_bit(R5_Discard, &sh->dev[i].flags);
1946 struct r5dev *dev = &sh->dev[i];
1951 if (test_bit(STRIPE_EXPAND_READY, &sh->state))
1961 if (sh->reconstruct_state == reconstruct_state_drain_run)
1962 sh->reconstruct_state = reconstruct_state_drain_result;
1963 else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
1964 sh->reconstruct_state = reconstruct_state_prexor_drain_result;
1966 BUG_ON(sh->reconstruct_state != reconstruct_state_run);
1967 sh->reconstruct_state = reconstruct_state_result;
1970 set_bit(STRIPE_HANDLE, &sh->state);
1971 raid5_release_stripe(sh);
1975 ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
1978 int disks = sh->disks;
1982 int count, pd_idx = sh->pd_idx, i;
1988 struct stripe_head *head_sh = sh;
1992 (unsigned long long)sh->sector);
1994 for (i = 0; i < sh->disks; i++) {
1997 if (!test_bit(R5_Discard, &sh->dev[i].flags))
2000 if (i >= sh->disks) {
2001 atomic_inc(&sh->count);
2002 set_bit(R5_Discard, &sh->dev[pd_idx].flags);
2003 ops_complete_reconstruct(sh);
2009 off_srcs = to_addr_offs(sh, percpu);
2015 off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
2016 xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
2018 struct r5dev *dev = &sh->dev[i];
2026 xor_dest = sh->dev[pd_idx].page;
2027 off_dest = sh->dev[pd_idx].offset;
2029 struct r5dev *dev = &sh->dev[i];
2043 list_first_entry(&sh->batch_list,
2051 to_addr_conv(sh, percpu, j));
2055 to_addr_conv(sh, percpu, j));
2060 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
2063 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
2066 sh = list_first_entry(&sh->batch_list, struct stripe_head,
2073 ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
2080 struct stripe_head *head_sh = sh;
2085 pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
2087 for (i = 0; i < sh->disks; i++) {
2088 if (sh->pd_idx == i || sh->qd_idx == i)
2090 if (!test_bit(R5_Discard, &sh->dev[i].flags))
2093 if (i >= sh->disks) {
2094 atomic_inc(&sh->count);
2095 set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
2096 set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
2097 ops_complete_reconstruct(sh);
2103 offs = to_addr_offs(sh, percpu);
2105 if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
2113 count = set_syndrome_sources(blocks, offs, sh, synflags);
2115 list_first_entry(&sh->batch_list,
2121 head_sh, to_addr_conv(sh, percpu, j));
2124 to_addr_conv(sh, percpu, j));
2126 RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
2129 sh = list_first_entry(&sh->batch_list, struct stripe_head,
2137 struct stripe_head *sh = stripe_head_ref;
2140 (unsigned long long)sh->sector);
2142 sh->check_state = check_state_check_result;
2143 set_bit(STRIPE_HANDLE, &sh->state);
2144 raid5_release_stripe(sh);
2147 static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
2149 int disks = sh->disks;
2150 int pd_idx = sh->pd_idx;
2151 int qd_idx = sh->qd_idx;
2155 unsigned int *off_srcs = to_addr_offs(sh, percpu);
2162 (unsigned long long)sh->sector);
2164 BUG_ON(sh->batch_head);
2166 xor_dest = sh->dev[pd_idx].page;
2167 off_dest = sh->dev[pd_idx].offset;
2173 off_srcs[count] = sh->dev[i].offset;
2174 xor_srcs[count++] = sh->dev[i].page;
2178 to_addr_conv(sh, percpu, 0));
2180 RAID5_STRIPE_SIZE(sh->raid_conf),
2181 &sh->ops.zero_sum_result, &submit);
2183 atomic_inc(&sh->count);
2184 init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
2188 static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
2191 unsigned int *offs = to_addr_offs(sh, percpu);
2196 (unsigned long long)sh->sector, checkp);
2198 BUG_ON(sh->batch_head);
2199 count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
2203 atomic_inc(&sh->count);
2205 sh, to_addr_conv(sh, percpu, 0));
2207 RAID5_STRIPE_SIZE(sh->raid_conf),
2208 &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
2211 static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
2213 int overlap_clear = 0, i, disks = sh->disks;
2215 struct r5conf *conf = sh->raid_conf;
2223 ops_run_biofill(sh);
2229 tx = ops_run_compute5(sh, percpu);
2231 if (sh->ops.target2 < 0 || sh->ops.target < 0)
2232 tx = ops_run_compute6_1(sh, percpu);
2234 tx = ops_run_compute6_2(sh, percpu);
2243 tx = ops_run_prexor5(sh, percpu, tx);
2245 tx = ops_run_prexor6(sh, percpu, tx);
2249 tx = ops_run_partial_parity(sh, percpu, tx);
2252 tx = ops_run_biodrain(sh, tx);
2258 ops_run_reconstruct5(sh, percpu, tx);
2260 ops_run_reconstruct6(sh, percpu, tx);
2264 if (sh->check_state == check_state_run)
2265 ops_run_check_p(sh, percpu);
2266 else if (sh->check_state == check_state_run_q)
2267 ops_run_check_pq(sh, percpu, 0);
2268 else if (sh->check_state == check_state_run_pq)
2269 ops_run_check_pq(sh, percpu, 1);
2274 if (overlap_clear && !sh->batch_head)
2276 struct r5dev *dev = &sh->dev[i];
2278 wake_up(&sh->raid_conf->wait_for_overlap);
2283 static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)
2286 kfree(sh->pages);
2288 if (sh->ppl_page)
2289 __free_page(sh->ppl_page);
2290 kmem_cache_free(sc, sh);
2296 struct stripe_head *sh;
2299 sh = kmem_cache_zalloc(sc, gfp);
2300 if (sh) {
2301 spin_lock_init(&sh->stripe_lock);
2302 spin_lock_init(&sh->batch_lock);
2303 INIT_LIST_HEAD(&sh->batch_list);
2304 INIT_LIST_HEAD(&sh->lru);
2305 INIT_LIST_HEAD(&sh->r5c);
2306 INIT_LIST_HEAD(&sh->log_list);
2307 atomic_set(&sh->count, 1);
2308 sh->raid_conf = conf;
2309 sh->log_start = MaxSector;
2311 struct r5dev *dev = &sh->dev[i];
2318 sh->ppl_page = alloc_page(gfp);
2319 if (!sh->ppl_page) {
2320 free_stripe(sc, sh);
2325 if (init_stripe_shared_pages(sh, conf, disks)) {
2326 free_stripe(sc, sh);
2331 return sh;
2335 struct stripe_head *sh;
2337 sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);
2338 if (!sh)
2341 if (grow_buffers(sh, gfp)) {
2342 shrink_buffers(sh);
2343 free_stripe(conf->slab_cache, sh);
2346 sh->hash_lock_index =
2351 raid5_release_stripe(sh);
2637 struct stripe_head *sh;
2641 sh = get_free_stripe(conf, hash);
2643 if (!sh)
2645 BUG_ON(atomic_read(&sh->count));
2646 shrink_buffers(sh);
2647 free_stripe(conf->slab_cache, sh);
2665 struct stripe_head *sh = bi->bi_private;
2666 struct r5conf *conf = sh->raid_conf;
2667 int disks = sh->disks, i;
2673 if (bi == &sh->dev[i].req)
2677 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
2684 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2694 if (use_new_offset(conf, sh))
2695 s = sh->sector + rdev->new_data_offset;
2697 s = sh->sector + rdev->data_offset;
2699 set_bit(R5_UPTODATE, &sh->dev[i].flags);
2700 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
2711 clear_bit(R5_ReadError, &sh->dev[i].flags);
2712 clear_bit(R5_ReWrite, &sh->dev[i].flags);
2713 } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2714 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2716 if (test_bit(R5_InJournal, &sh->dev[i].flags))
2721 set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
2730 clear_bit(R5_UPTODATE, &sh->dev[i].flags);
2733 if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
2746 } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
2767 && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
2770 if (sh->qd_idx >= 0 && sh->pd_idx == i)
2771 set_bit(R5_ReadError, &sh->dev[i].flags);
2772 else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
2773 set_bit(R5_ReadError, &sh->dev[i].flags);
2774 clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2776 set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
2778 clear_bit(R5_ReadError, &sh->dev[i].flags);
2779 clear_bit(R5_ReWrite, &sh->dev[i].flags);
2783 rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))
2789 clear_bit(R5_LOCKED, &sh->dev[i].flags);
2790 set_bit(STRIPE_HANDLE, &sh->state);
2791 raid5_release_stripe(sh);
2796 struct stripe_head *sh = bi->bi_private;
2797 struct r5conf *conf = sh->raid_conf;
2798 int disks = sh->disks, i;
2805 if (bi == &sh->dev[i].req) {
2809 if (bi == &sh->dev[i].rreq) {
2823 (unsigned long long)sh->sector, i, atomic_read(&sh->count),
2834 else if (is_badblock(rdev, sh->sector,
2837 set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
2840 set_bit(STRIPE_DEGRADED, &sh->state);
2842 set_bit(R5_WriteError, &sh->dev[i].flags);
2846 } else if (is_badblock(rdev, sh->sector,
2849 set_bit(R5_MadeGood, &sh->dev[i].flags);
2850 if (test_bit(R5_ReadError, &sh->dev[i].flags))
2855 set_bit(R5_ReWrite, &sh->dev[i].flags);
2860 if (sh->batch_head && bi->bi_status && !replacement)
2861 set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
2864 if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
2865 clear_bit(R5_LOCKED, &sh->dev[i].flags);
2866 set_bit(STRIPE_HANDLE, &sh->state);
2868 if (sh->batch_head && sh != sh->batch_head)
2869 raid5_release_stripe(sh->batch_head);
2870 raid5_release_stripe(sh);
2914 struct stripe_head *sh)
3102 if (sh) {
3103 sh->pd_idx = pd_idx;
3104 sh->qd_idx = qd_idx;
3105 sh->ddf_layout = ddf_layout;
3114 sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)
3116 struct r5conf *conf = sh->raid_conf;
3117 int raid_disks = sh->disks;
3119 sector_t new_sector = sh->sector, check;
3134 if (i == sh->pd_idx)
3142 if (i > sh->pd_idx)
3147 if (i < sh->pd_idx)
3149 i -= (sh->pd_idx + 1);
3161 if (i == sh->qd_idx)
3168 if (sh->pd_idx == raid_disks-1)
3170 else if (i > sh->pd_idx)
3175 if (sh->pd_idx == raid_disks-1)
3179 if (i < sh->pd_idx)
3181 i -= (sh->pd_idx + 2);
3191 if (sh->pd_idx == 0)
3195 if (i < sh->pd_idx)
3197 i -= (sh->pd_idx + 1);
3202 if (i > sh->pd_idx)
3207 if (i < sh->pd_idx)
3209 i -= (sh->pd_idx + 1);
3225 if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
3226 || sh2.qd_idx != sh->qd_idx) {
3291 schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
3294 int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
3295 struct r5conf *conf = sh->raid_conf;
3305 r5c_release_extra_page(sh);
3308 struct r5dev *dev = &sh->dev[i];
3329 sh->reconstruct_state = reconstruct_state_drain_run;
3332 sh->reconstruct_state = reconstruct_state_run;
3337 if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
3340 BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
3341 test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
3343 (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
3344 test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
3347 struct r5dev *dev = &sh->dev[i];
3366 sh->reconstruct_state = reconstruct_state_prexor_drain_run;
3375 set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
3376 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
3380 int qd_idx = sh->qd_idx;
3381 struct r5dev *dev = &sh->dev[qd_idx];
3388 if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&
3390 !test_bit(STRIPE_FULL_WRITE, &sh->state) &&
3391 test_bit(R5_Insync, &sh->dev[pd_idx].flags))
3395 __func__, (unsigned long long)sh->sector,
3404 static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,
3408 struct r5conf *conf = sh->raid_conf;
3413 (unsigned long long)sh->sector);
3415 spin_lock_irq(&sh->stripe_lock);
3416 sh->dev[dd_idx].write_hint = bi->bi_write_hint;
3418 if (sh->batch_head)
3421 bip = &sh->dev[dd_idx].towrite;
3425 bip = &sh->dev[dd_idx].toread;
3448 for (i = 0; i < sh->disks; i++) {
3449 if (i != sh->pd_idx &&
3450 (i == dd_idx || sh->dev[i].towrite)) {
3451 sector = sh->dev[i].sector;
3465 clear_bit(STRIPE_BATCH_READY, &sh->state);
3476 sector_t sector = sh->dev[dd_idx].sector;
3477 for (bi=sh->dev[dd_idx].towrite;
3478 sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&
3480 bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {
3484 if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))
3485 if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
3486 sh->overwrite_disks++;
3491 (unsigned long long)sh->sector, dd_idx);
3506 set_bit(STRIPE_BITMAP_PENDING, &sh->state);
3507 spin_unlock_irq(&sh->stripe_lock);
3508 md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,
3510 spin_lock_irq(&sh->stripe_lock);
3511 clear_bit(STRIPE_BITMAP_PENDING, &sh->state);
3512 if (!sh->batch_head) {
3513 sh->bm_seq = conf->seq_flush+1;
3514 set_bit(STRIPE_BIT_DELAY, &sh->state);
3517 spin_unlock_irq(&sh->stripe_lock);
3519 if (stripe_can_batch(sh))
3520 stripe_add_to_batch_list(conf, sh);
3524 set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
3525 spin_unlock_irq(&sh->stripe_lock);
3532 struct stripe_head *sh)
3544 &dd_idx, sh);
3548 handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
3552 BUG_ON(sh->batch_head);
3557 if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
3570 sh->sector,
3576 spin_lock_irq(&sh->stripe_lock);
3578 bi = sh->dev[i].towrite;
3579 sh->dev[i].towrite = NULL;
3580 sh->overwrite_disks = 0;
3581 spin_unlock_irq(&sh->stripe_lock);
3585 log_stripe_write_finished(sh);
3587 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3591 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
3592 struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);
3599 md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3603 bi = sh->dev[i].written;
3604 sh->dev[i].written = NULL;
3605 if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
3606 WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
3607 sh->dev[i].page = sh->dev[i].orig_page;
3612 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
3613 struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);
3623 if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
3625 (!test_bit(R5_Insync, &sh->dev[i].flags) ||
3626 test_bit(R5_ReadError, &sh->dev[i].flags))) {
3627 spin_lock_irq(&sh->stripe_lock);
3628 bi = sh->dev[i].toread;
3629 sh->dev[i].toread = NULL;
3630 spin_unlock_irq(&sh->stripe_lock);
3631 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
3636 sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
3638 r5_next_bio(conf, bi, sh->dev[i].sector);
3645 md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3650 clear_bit(R5_LOCKED, &sh->dev[i].flags);
3655 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
3661 handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
3667 BUG_ON(sh->batch_head);
3668 clear_bit(STRIPE_SYNCING, &sh->state);
3669 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
3690 && !rdev_set_badblocks(rdev, sh->sector,
3697 && !rdev_set_badblocks(rdev, sh->sector,
3709 static int want_replace(struct stripe_head *sh, int disk_idx)
3715 rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement);
3719 && (rdev->recovery_offset <= sh->sector
3720 || rdev->mddev->recovery_cp <= sh->sector))
3726 static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
3729 struct r5dev *dev = &sh->dev[disk_idx];
3730 struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
3731 &sh->dev[s->failed_num[1]] };
3733 bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);
3749 (s->replacing && want_replace(sh, disk_idx)))
3774 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
3795 s->failed_num[i] == sh->pd_idx ||
3796 s->failed_num[i] == sh->qd_idx) &&
3813 sh->sector < sh->raid_conf->mddev->recovery_cp)
3817 if (s->failed_num[i] != sh->pd_idx &&
3818 s->failed_num[i] != sh->qd_idx &&
3833 static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
3836 struct r5dev *dev = &sh->dev[disk_idx];
3839 if (need_this_block(sh, s, disk_idx, disks)) {
3845 BUG_ON(sh->batch_head);
3857 ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) ||
3864 (unsigned long long)sh->sector, disk_idx);
3865 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3868 sh->ops.target = disk_idx;
3869 sh->ops.target2 = -1; /* no 2nd target */
3888 &sh->dev[other].flags))
3893 (unsigned long long)sh->sector,
3895 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
3897 set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
3898 set_bit(R5_Wantcompute, &sh->dev[other].flags);
3899 sh->ops.target = disk_idx;
3900 sh->ops.target2 = other;
3919 static void handle_stripe_fill(struct stripe_head *sh,
3929 if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
3930 !sh->reconstruct_state) {
3940 if (test_bit(STRIPE_R5C_CACHING, &sh->state))
3941 r5c_make_stripe_write_out(sh);
3946 if (fetch_block(sh, s, i, disks))
3950 set_bit(STRIPE_HANDLE, &sh->state);
3961 struct stripe_head *sh, int disks)
3966 struct stripe_head *head_sh = sh;
3970 if (sh->dev[i].written) {
3971 dev = &sh->dev[i];
3997 md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
3999 !test_bit(STRIPE_DEGRADED, &sh->state),
4002 sh = list_first_entry(&sh->batch_list,
4005 if (sh != head_sh) {
4006 dev = &sh->dev[i];
4010 sh = head_sh;
4011 dev = &sh->dev[i];
4016 log_stripe_write_finished(sh);
4019 test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
4021 clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
4022 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
4023 if (sh->qd_idx >= 0) {
4024 clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
4025 clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
4028 clear_bit(STRIPE_DISCARD, &sh->state);
4035 hash = sh->hash_lock_index;
4037 remove_hash(sh);
4040 sh = list_first_entry(&sh->batch_list,
4042 if (sh != head_sh)
4045 sh = head_sh;
4047 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
4048 set_bit(STRIPE_HANDLE, &sh->state);
4052 if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
4076 struct stripe_head *sh,
4091 (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
4097 pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",
4099 (unsigned long long)sh->sector);
4102 struct r5dev *dev = &sh->dev[i];
4104 i == sh->pd_idx || i == sh->qd_idx ||
4116 i != sh->pd_idx && i != sh->qd_idx &&
4128 (unsigned long long)sh->sector, sh->state, rmw, rcw);
4129 set_bit(STRIPE_HANDLE, &sh->state);
4135 (unsigned long long)sh->sector, rmw);
4137 struct r5dev *dev = &sh->dev[i];
4140 !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {
4155 r5c_use_extra_page(sh);
4160 set_bit(STRIPE_DELAYED, &sh->state);
4167 struct r5dev *dev = &sh->dev[i];
4169 i == sh->pd_idx || i == sh->qd_idx ||
4176 &sh->state)) {
4183 set_bit(STRIPE_DELAYED, &sh->state);
4192 struct r5dev *dev = &sh->dev[i];
4194 i != sh->pd_idx && i != sh->qd_idx &&
4201 &sh->state)) {
4209 set_bit(STRIPE_DELAYED, &sh->state);
4214 (unsigned long long)sh->sector,
4215 rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
4219 !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
4220 set_bit(STRIPE_DELAYED, &sh->state);
4232 if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
4234 !test_bit(STRIPE_BIT_DELAY, &sh->state)))
4235 schedule_reconstruction(sh, s, rcw == 0, 0);
4239 static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
4244 BUG_ON(sh->batch_head);
4245 set_bit(STRIPE_HANDLE, &sh->state);
4247 switch (sh->check_state) {
4252 sh->check_state = check_state_run;
4254 clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
4258 dev = &sh->dev[s->failed_num[0]];
4261 sh->check_state = check_state_idle;
4263 dev = &sh->dev[sh->pd_idx];
4266 if (test_bit(STRIPE_INSYNC, &sh->state))
4277 clear_bit(STRIPE_DEGRADED, &sh->state);
4278 set_bit(STRIPE_INSYNC, &sh->state);
4283 sh->check_state = check_state_idle;
4295 if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
4299 set_bit(STRIPE_INSYNC, &sh->state);
4304 set_bit(STRIPE_INSYNC, &sh->state);
4307 (unsigned long long) sh->sector,
4308 (unsigned long long) sh->sector +
4311 sh->check_state = check_state_compute_run;
4312 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
4315 &sh->dev[sh->pd_idx].flags);
4316 sh->ops.target = sh->pd_idx;
4317 sh->ops.target2 = -1;
4326 __func__, sh->check_state,
4327 (unsigned long long) sh->sector);
4332 static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
4336 int pd_idx = sh->pd_idx;
4337 int qd_idx = sh->qd_idx;
4340 BUG_ON(sh->batch_head);
4341 set_bit(STRIPE_HANDLE, &sh->state);
4351 switch (sh->check_state) {
4359 sh->check_state = check_state_run;
4365 if (sh->check_state == check_state_run)
4366 sh->check_state = check_state_run_pq;
4368 sh->check_state = check_state_run_q;
4372 sh->ops.zero_sum_result = 0;
4374 if (sh->check_state == check_state_run) {
4376 clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
4379 if (sh->check_state >= check_state_run &&
4380 sh->check_state <= check_state_run_pq) {
4392 sh->check_state = check_state_idle;
4395 if (test_bit(STRIPE_INSYNC, &sh->state))
4403 dev = &sh->dev[s->failed_num[1]];
4409 dev = &sh->dev[s->failed_num[0]];
4414 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
4415 dev = &sh->dev[pd_idx];
4420 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
4421 dev = &sh->dev[qd_idx];
4429 dev - (struct r5dev *) &sh->dev)) {
4434 clear_bit(STRIPE_DEGRADED, &sh->state);
4436 set_bit(STRIPE_INSYNC, &sh->state);
4443 sh->check_state = check_state_idle;
4449 if (sh->ops.zero_sum_result == 0) {
4452 set_bit(STRIPE_INSYNC, &sh->state);
4458 sh->check_state = check_state_compute_result;
4469 set_bit(STRIPE_INSYNC, &sh->state);
4472 (unsigned long long) sh->sector,
4473 (unsigned long long) sh->sector +
4476 int *target = &sh->ops.target;
4478 sh->ops.target = -1;
4479 sh->ops.target2 = -1;
4480 sh->check_state = check_state_compute_run;
4481 set_bit(STRIPE_COMPUTE_RUN, &sh->state);
4483 if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
4485 &sh->dev[pd_idx].flags);
4487 target = &sh->ops.target2;
4490 if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
4492 &sh->dev[qd_idx].flags);
4503 __func__, sh->check_state,
4504 (unsigned long long) sh->sector);
4509 static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
4517 BUG_ON(sh->batch_head);
4518 clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
4519 for (i = 0; i < sh->disks; i++)
4520 if (i != sh->pd_idx && i != sh->qd_idx) {
4525 sector_t bn = raid5_compute_blocknr(sh, i, 1);
4545 sh->dev[i].page, sh2->dev[dd_idx].offset,
4546 sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
4581 static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
4583 struct r5conf *conf = sh->raid_conf;
4584 int disks = sh->disks;
4591 s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;
4592 s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;
4605 dev = &sh->dev[i];
4616 !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
4645 rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
4646 !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
4660 is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
4687 else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)
4757 if (test_bit(STRIPE_SYNCING, &sh->state)) {
4767 sh->sector >= conf->mddev->recovery_cp ||
4780 static int clear_batch_ready(struct stripe_head *sh)
4783 if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
4784 return (sh->batch_head && sh->batch_head != sh);
4785 spin_lock(&sh->stripe_lock);
4786 if (!sh->batch_head) {
4787 spin_unlock(&sh->stripe_lock);
4795 if (sh->batch_head != sh) {
4796 spin_unlock(&sh->stripe_lock);
4799 spin_lock(&sh->batch_lock);
4800 list_for_each_entry(tmp, &sh->batch_list, batch_list)
4802 spin_unlock(&sh->batch_lock);
4803 spin_unlock(&sh->stripe_lock);
4815 struct stripe_head *sh, *next;
4819 list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {
4821 list_del_init(&sh->batch_list);
4823 WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |
4835 "stripe state: %lx\n", sh->state);
4840 set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |
4846 sh->check_state = head_sh->check_state;
4847 sh->reconstruct_state = head_sh->reconstruct_state;
4848 spin_lock_irq(&sh->stripe_lock);
4849 sh->batch_head = NULL;
4850 spin_unlock_irq(&sh->stripe_lock);
4851 for (i = 0; i < sh->disks; i++) {
4852 if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
4854 sh->dev[i].flags = head_sh->dev[i].flags &
4858 sh->state & handle_flags)
4859 set_bit(STRIPE_HANDLE, &sh->state);
4860 raid5_release_stripe(sh);
4875 static void handle_stripe(struct stripe_head *sh)
4878 struct r5conf *conf = sh->raid_conf;
4881 int disks = sh->disks;
4884 clear_bit(STRIPE_HANDLE, &sh->state);
4892 if (clear_batch_ready(sh))
4895 if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
4898 set_bit(STRIPE_HANDLE, &sh->state);
4902 if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
4903 break_stripe_batch_list(sh, 0);
4905 if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {
4906 spin_lock(&sh->stripe_lock);
4911 if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&
4912 !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) &&
4913 !test_bit(STRIPE_DISCARD, &sh->state) &&
4914 test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
4915 set_bit(STRIPE_SYNCING, &sh->state);
4916 clear_bit(STRIPE_INSYNC, &sh->state);
4917 clear_bit(STRIPE_REPLACED, &sh->state);
4919 spin_unlock(&sh->stripe_lock);
4921 clear_bit(STRIPE_DELAYED, &sh->state);
4925 (unsigned long long)sh->sector, sh->state,
4926 atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
4927 sh->check_state, sh->reconstruct_state);
4929 analyse_stripe(sh, &s);
4931 if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
4936 set_bit(STRIPE_HANDLE, &sh->state);
4943 set_bit(STRIPE_HANDLE, &sh->state);
4951 if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
4953 set_bit(STRIPE_BIOFILL_RUN, &sh->state);
4969 sh->check_state = 0;
4970 sh->reconstruct_state = 0;
4971 break_stripe_batch_list(sh, 0);
4973 handle_failed_stripe(conf, sh, &s, disks);
4975 handle_failed_sync(conf, sh, &s);
4982 if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
4984 if (sh->reconstruct_state == reconstruct_state_drain_result ||
4985 sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
4986 sh->reconstruct_state = reconstruct_state_idle;
4991 BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
4992 !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
4993 BUG_ON(sh->qd_idx >= 0 &&
4994 !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
4995 !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
4997 struct r5dev *dev = &sh->dev[i];
4999 (i == sh->pd_idx || i == sh->qd_idx ||
5009 ((i == sh->pd_idx || i == sh->qd_idx) &&
5011 set_bit(STRIPE_INSYNC, &sh->state);
5014 if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5022 pdev = &sh->dev[sh->pd_idx];
5023 s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
5024 || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
5025 qdev = &sh->dev[sh->qd_idx];
5026 s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
5027 || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
5039 handle_stripe_clean_event(conf, sh, disks);
5042 r5c_handle_cached_data_endio(conf, sh, disks);
5043 log_stripe_write_finished(sh);
5054 handle_stripe_fill(sh, &s, disks);
5061 r5c_finish_stripe_write_out(conf, sh, &s);
5072 if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {
5075 handle_stripe_dirtying(conf, sh, &s, disks);
5081 ret = r5c_try_caching_write(conf, sh, &s,
5092 (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
5094 ret = handle_stripe_dirtying(conf, sh, &s,
5107 if (sh->check_state ||
5109 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
5110 !test_bit(STRIPE_INSYNC, &sh->state))) {
5112 handle_parity_checks6(conf, sh, &s, disks);
5114 handle_parity_checks5(conf, sh, &s, disks);
5118 && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
5119 && !test_bit(STRIPE_REPLACED, &sh->state)) {
5122 if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
5123 WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
5124 set_bit(R5_WantReplace, &sh->dev[i].flags);
5125 set_bit(R5_LOCKED, &sh->dev[i].flags);
5129 set_bit(STRIPE_INSYNC, &sh->state);
5130 set_bit(STRIPE_REPLACED, &sh->state);
5133 !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
5134 test_bit(STRIPE_INSYNC, &sh->state)) {
5136 clear_bit(STRIPE_SYNCING, &sh->state);
5137 if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
5146 struct r5dev *dev = &sh->dev[s.failed_num[i]];
5163 if (sh->reconstruct_state == reconstruct_state_result) {
5165 = raid5_get_active_stripe(conf, sh->sector, 1, 1, 1);
5167 /* sh cannot be written until sh_src has been read.
5168 * so arrange for sh to be delayed a little
5170 set_bit(STRIPE_DELAYED, &sh->state);
5171 set_bit(STRIPE_HANDLE, &sh->state);
5181 sh->reconstruct_state = reconstruct_state_idle;
5182 clear_bit(STRIPE_EXPANDING, &sh->state);
5184 set_bit(R5_Wantwrite, &sh->dev[i].flags);
5185 set_bit(R5_LOCKED, &sh->dev[i].flags);
5190 if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
5191 !sh->reconstruct_state) {
5193 sh->disks = conf->raid_disks;
5194 stripe_set_idx(sh->sector, conf, 0, sh);
5195 schedule_reconstruction(sh, &s, 1, 1);
5196 } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
5197 clear_bit(STRIPE_EXPAND_READY, &sh->state);
5204 !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
5205 handle_stripe_expansion(conf, sh);
5225 struct r5dev *dev = &sh->dev[i];
5229 if (!rdev_set_badblocks(rdev, sh->sector,
5236 rdev_clear_badblocks(rdev, sh->sector,
5245 rdev_clear_badblocks(rdev, sh->sector,
5252 raid_run_ops(sh, s.ops_request);
5254 ops_run_io(sh, &s);
5267 clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
5275 struct stripe_head *sh;
5276 sh = list_entry(l, struct stripe_head, lru);
5278 clear_bit(STRIPE_DELAYED, &sh->state);
5279 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5281 list_add_tail(&sh->lru, &conf->hold_list);
5282 raid5_wakeup_stripe_thread(sh);
5295 struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
5297 list_del_init(&sh->lru);
5298 atomic_inc(&sh->count);
5299 hash = sh->hash_lock_index;
5300 __release_stripe(conf, sh, &temp_inactive_list[hash]);
5514 struct stripe_head *sh, *tmp;
5524 sh = NULL;
5550 sh = list_entry(handle_list->next, typeof(*sh), lru);
5554 else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
5574 sh = tmp;
5579 if (sh) {
5587 if (!sh) {
5597 sh->group = NULL;
5599 list_del_init(&sh->lru);
5600 BUG_ON(atomic_inc_return(&sh->count) != 1);
5601 return sh;
5614 struct stripe_head *sh;
5623 sh = list_first_entry(&cb->list, struct stripe_head, lru);
5624 list_del_init(&sh->lru);
5631 clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
5636 hash = sh->hash_lock_index;
5637 __release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
5650 struct stripe_head *sh)
5658 raid5_release_stripe(sh);
5671 if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
5672 list_add_tail(&sh->lru, &cb->list);
5674 raid5_release_stripe(sh);
5681 struct stripe_head *sh;
5707 sh = raid5_get_active_stripe(conf, logical_sector, 0, 0, 0);
5710 set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
5711 if (test_bit(STRIPE_SYNCING, &sh->state)) {
5712 raid5_release_stripe(sh);
5716 clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
5717 spin_lock_irq(&sh->stripe_lock);
5719 if (d == sh->pd_idx || d == sh->qd_idx)
5721 if (sh->dev[d].towrite || sh->dev[d].toread) {
5722 set_bit(R5_Overlap, &sh->dev[d].flags);
5723 spin_unlock_irq(&sh->stripe_lock);
5724 raid5_release_stripe(sh);
5729 set_bit(STRIPE_DISCARD, &sh->state);
5731 sh->overwrite_disks = 0;
5733 if (d == sh->pd_idx || d == sh->qd_idx)
5735 sh->dev[d].towrite = bi;
5736 set_bit(R5_OVERWRITE, &sh->dev[d].flags);
5739 sh->overwrite_disks++;
5741 spin_unlock_irq(&sh->stripe_lock);
5747 sh->sector,
5750 sh->bm_seq = conf->seq_flush + 1;
5751 set_bit(STRIPE_BIT_DELAY, &sh->state);
5754 set_bit(STRIPE_HANDLE, &sh->state);
5755 clear_bit(STRIPE_DELAYED, &sh->state);
5756 if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5758 release_stripe_plug(mddev, sh);
5770 struct stripe_head *sh;
5863 sh = raid5_get_active_stripe(conf, new_sector, previous,
5865 if (sh) {
5871 * 'sh', we know that if that happens,
5884 raid5_release_stripe(sh);
5894 raid5_release_stripe(sh);
5898 if (test_bit(STRIPE_EXPANDING, &sh->state) ||
5899 !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {
5905 raid5_release_stripe(sh);
5911 set_bit(STRIPE_R5C_PREFLUSH, &sh->state);
5916 set_bit(STRIPE_HANDLE, &sh->state);
5917 clear_bit(STRIPE_DELAYED, &sh->state);
5918 if ((!sh->batch_head || sh == sh->batch_head) &&
5920 !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
5922 release_stripe_plug(mddev, sh);
5951 struct stripe_head *sh;
6102 sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
6103 set_bit(STRIPE_EXPANDING, &sh->state);
6108 for (j=sh->disks; j--;) {
6110 if (j == sh->pd_idx)
6113 j == sh->qd_idx)
6115 s = raid5_compute_blocknr(sh, j, 0);
6120 memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));
6121 set_bit(R5_Expanded, &sh->dev[j].flags);
6122 set_bit(R5_UPTODATE, &sh->dev[j].flags);
6125 set_bit(STRIPE_EXPAND_READY, &sh->state);
6126 set_bit(STRIPE_HANDLE, &sh->state);
6128 list_add(&sh->lru, &stripes);
6151 sh = raid5_get_active_stripe(conf, first_sector, 1, 0, 1);
6152 set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
6153 set_bit(STRIPE_HANDLE, &sh->state);
6154 raid5_release_stripe(sh);
6161 sh = list_entry(stripes.next, struct stripe_head, lru);
6162 list_del_init(&sh->lru);
6163 raid5_release_stripe(sh);
6212 struct stripe_head *sh;
6271 sh = raid5_get_active_stripe(conf, sector_nr, 0, 1, 0);
6272 if (sh == NULL) {
6273 sh = raid5_get_active_stripe(conf, sector_nr, 0, 0, 0);
6294 set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
6295 set_bit(STRIPE_HANDLE, &sh->state);
6297 raid5_release_stripe(sh);
6315 struct stripe_head *sh;
6336 sh = raid5_get_active_stripe(conf, sector, 0, 1, 1);
6338 if (!sh) {
6345 if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
6346 raid5_release_stripe(sh);
6352 set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
6353 handle_stripe(sh);
6354 raid5_release_stripe(sh);
6371 struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
6376 (sh = __get_priority_stripe(conf, group)) != NULL)
6377 batch[batch_size++] = sh;