Lines Matching refs:dio
374 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map);
1465 static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1471 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1494 static void do_endio_flush(struct dm_integrity_c *ic, struct dm_integrity_io *dio)
1496 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1498 if (unlikely(dio->fua) && likely(!bio->bi_status) && likely(!dm_integrity_failed(ic)))
1499 submit_flush_bio(ic, dio);
1504 static void dec_in_flight(struct dm_integrity_io *dio)
1506 if (atomic_dec_and_test(&dio->in_flight)) {
1507 struct dm_integrity_c *ic = dio->ic;
1510 remove_range(ic, &dio->range);
1512 if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))
1515 bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1517 if (unlikely(dio->bi_status) && !bio->bi_status)
1518 bio->bi_status = dio->bi_status;
1519 if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) {
1520 dio->range.logical_sector += dio->range.n_sectors;
1521 bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT);
1522 INIT_WORK(&dio->work, integrity_bio_wait);
1523 queue_work(ic->offload_wq, &dio->work);
1526 do_endio_flush(ic, dio);
1532 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1534 dm_bio_restore(&dio->bio_details, bio);
1538 if (dio->completion)
1539 complete(dio->completion);
1541 dec_in_flight(dio);
1591 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
1592 struct dm_integrity_c *ic = dio->ic;
1600 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1610 if (likely(dio->op != REQ_OP_DISCARD))
1624 if (unlikely(dio->op == REQ_OP_DISCARD)) {
1625 sector_t bi_sector = dio->bio_details.bi_iter.bi_sector;
1626 unsigned bi_size = dio->bio_details.bi_iter.bi_size;
1634 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1656 sector = dio->range.logical_sector;
1657 sectors_to_process = dio->range.n_sectors;
1659 __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) {
1677 r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset,
1678 checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE);
1705 struct bio_integrity_payload *bip = dio->bio_details.bi_integrity;
1710 unsigned data_to_process = dio->range.n_sectors;
1721 r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset,
1722 this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE);
1732 dec_in_flight(dio);
1735 dio->bi_status = errno_to_blk_status(r);
1736 dec_in_flight(dio);
1742 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
1747 dio->ic = ic;
1748 dio->bi_status = 0;
1749 dio->op = bio_op(bio);
1751 if (unlikely(dio->op == REQ_OP_DISCARD)) {
1765 submit_flush_bio(ic, dio);
1769 dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1770 dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA;
1771 if (unlikely(dio->fua)) {
1778 if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) {
1780 dio->range.logical_sector, bio_sectors(bio),
1784 if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1))) {
1787 dio->range.logical_sector, bio_sectors(bio));
1791 if (ic->sectors_per_block > 1 && likely(dio->op != REQ_OP_DISCARD)) {
1824 if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ))
1827 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1828 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
1831 dm_integrity_map_continue(dio, true);
1835 static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio,
1838 struct dm_integrity_c *ic = dio->ic;
1842 logical_sector = dio->range.logical_sector;
1843 n_sectors = dio->range.n_sectors;
1854 if (likely(dio->op == REQ_OP_WRITE))
1860 if (unlikely(dio->op == REQ_OP_READ)) {
1907 if (likely(dio->op == REQ_OP_WRITE))
1915 if (likely(dio->op == REQ_OP_WRITE))
1920 if (likely(dio->op == REQ_OP_WRITE)) {
1956 if (unlikely(dio->op == REQ_OP_READ))
1961 if (likely(dio->op == REQ_OP_WRITE)) {
1971 remove_range(ic, &dio->range);
1977 dio->range.logical_sector = logical_sector;
1978 get_area_and_offset(ic, dio->range.logical_sector, &area, &offset);
1979 dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, &dio->metadata_offset);
1986 static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map)
1988 struct dm_integrity_c *ic = dio->ic;
1989 struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io));
1994 bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ;
1995 if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D')
1999 INIT_WORK(&dio->work, integrity_bio_wait);
2000 queue_work(ic->offload_wq, &dio->work);
2012 dio->range.n_sectors = bio_sectors(bio);
2014 if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) {
2015 if (dio->op == REQ_OP_WRITE) {
2019 dio->range.n_sectors = min(dio->range.n_sectors,
2021 if (unlikely(!dio->range.n_sectors)) {
2027 range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block;
2045 add_journal_node(ic, &ic->journal_tree[pos], dio->range.logical_sector + i);
2059 } while ((i += ic->sectors_per_block) < dio->range.n_sectors);
2065 journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2067 if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector))
2068 dio->range.n_sectors = next_sector - dio->range.logical_sector;
2072 for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) {
2073 if (!test_journal_node(ic, jp, dio->range.logical_sector + i))
2076 dio->range.n_sectors = i;
2080 if (unlikely(!add_new_range(ic, &dio->range, true))) {
2089 INIT_WORK(&dio->work, integrity_bio_wait);
2090 queue_work(ic->wait_wq, &dio->work);
2094 dio->range.n_sectors = ic->sectors_per_block;
2095 wait_and_add_new_range(ic, &dio->range);
2103 unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2105 remove_range_unlocked(ic, &dio->range);
2110 if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) {
2112 unsigned new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector);
2114 unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) {
2115 remove_range_unlocked(ic, &dio->range);
2133 if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) {
2134 if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
2135 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {
2138 bbs = sector_to_bitmap_block(ic, dio->range.logical_sector);
2147 dio->in_flight = (atomic_t)ATOMIC_INIT(2);
2151 dio->completion = &read_comp;
2153 dio->completion = NULL;
2155 dm_bio_record(&dio->bio_details, bio);
2160 bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT;
2162 if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) {
2163 integrity_metadata(&dio->work);
2166 dio->in_flight = (atomic_t)ATOMIC_INIT(1);
2167 dio->completion = NULL;
2179 dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector))
2182 if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector,
2183 dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR))
2188 integrity_metadata(&dio->work);
2191 dec_in_flight(dio);
2194 INIT_WORK(&dio->work, integrity_metadata);
2195 queue_work(ic->metadata_wq, &dio->work);
2201 if (unlikely(__journal_read_write(dio, bio, journal_section, journal_entry)))
2204 do_endio_flush(ic, dio);
2210 struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work);
2212 dm_integrity_map_continue(dio, false);
2664 struct dm_integrity_io *dio;
2666 dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
2668 if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
2669 dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) {
2670 remove_range(ic, &dio->range);
2671 INIT_WORK(&dio->work, integrity_bio_wait);
2672 queue_work(ic->offload_wq, &dio->work);
2674 block_bitmap_op(ic, ic->journal, dio->range.logical_sector,
2675 dio->range.n_sectors, BITMAP_OP_SET);
2688 struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io));
2690 block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector,
2691 dio->range.n_sectors, BITMAP_OP_SET);
2693 remove_range(ic, &dio->range);
2694 INIT_WORK(&dio->work, integrity_bio_wait);
2695 queue_work(ic->offload_wq, &dio->work);