Lines Matching refs:sdio
161 static inline unsigned dio_pages_present(struct dio_submit *sdio)
163 return sdio->tail - sdio->head;
169 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)
173 ret = iov_iter_get_pages(sdio->iter, dio->pages, LONG_MAX, DIO_PAGES,
174 &sdio->from);
176 if (ret < 0 && sdio->blocks_available && (dio->op == REQ_OP_WRITE)) {
187 sdio->head = 0;
188 sdio->tail = 1;
189 sdio->from = 0;
190 sdio->to = PAGE_SIZE;
195 iov_iter_advance(sdio->iter, ret);
196 ret += sdio->from;
197 sdio->head = 0;
198 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
199 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1;
212 struct dio_submit *sdio)
214 if (dio_pages_present(sdio) == 0) {
217 ret = dio_refill_pages(dio, sdio);
220 BUG_ON(dio_pages_present(sdio) == 0);
222 return dio->pages[sdio->head];
390 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
412 sdio->bio = bio;
413 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset;
423 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)
425 struct bio *bio = sdio->bio;
439 if (sdio->submit_io) {
440 sdio->submit_io(bio, dio->inode, sdio->logical_offset_in_bio);
445 sdio->bio = NULL;
446 sdio->boundary = 0;
447 sdio->logical_offset_in_bio = 0;
453 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)
455 while (sdio->head < sdio->tail)
456 put_page(dio->pages[sdio->head++]);
545 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)
549 if (sdio->reap_counter++ >= 64) {
563 sdio->reap_counter = 0;
606 * of available blocks at sdio->blocks_available. These are in units of the
627 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,
635 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor;
644 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request);
645 fs_startblk = sdio->block_in_file >> sdio->blkfactor;
646 fs_endblk = (sdio->final_block_in_request - 1) >>
647 sdio->blkfactor;
671 ret = (*sdio->get_block)(dio->inode, fs_startblk,
686 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,
692 ret = dio_bio_reap(dio, sdio);
695 sector = start_sector << (sdio->blkbits - 9);
696 nr_pages = min(sdio->pages_in_io, BIO_MAX_PAGES);
698 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);
699 sdio->boundary = 0;
711 static inline int dio_bio_add_page(struct dio_submit *sdio)
715 ret = bio_add_page(sdio->bio, sdio->cur_page,
716 sdio->cur_page_len, sdio->cur_page_offset);
717 if (ret == sdio->cur_page_len) {
721 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE)
722 sdio->pages_in_io--;
723 get_page(sdio->cur_page);
724 sdio->final_block_in_bio = sdio->cur_page_block +
725 (sdio->cur_page_len >> sdio->blkbits);
743 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,
748 if (sdio->bio) {
749 loff_t cur_offset = sdio->cur_page_fs_offset;
750 loff_t bio_next_offset = sdio->logical_offset_in_bio +
751 sdio->bio->bi_iter.bi_size;
767 if (sdio->final_block_in_bio != sdio->cur_page_block ||
769 dio_bio_submit(dio, sdio);
772 if (sdio->bio == NULL) {
773 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
778 if (dio_bio_add_page(sdio) != 0) {
779 dio_bio_submit(dio, sdio);
780 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
782 ret = dio_bio_add_page(sdio);
808 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,
813 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */
825 if (sdio->cur_page == page &&
826 sdio->cur_page_offset + sdio->cur_page_len == offset &&
827 sdio->cur_page_block +
828 (sdio->cur_page_len >> sdio->blkbits) == blocknr) {
829 sdio->cur_page_len += len;
836 if (sdio->cur_page) {
837 ret = dio_send_cur_page(dio, sdio, map_bh);
838 put_page(sdio->cur_page);
839 sdio->cur_page = NULL;
845 sdio->cur_page = page;
846 sdio->cur_page_offset = offset;
847 sdio->cur_page_len = len;
848 sdio->cur_page_block = blocknr;
849 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits;
856 ret = dio_send_cur_page(dio, sdio, map_bh);
857 if (sdio->bio)
858 dio_bio_submit(dio, sdio);
859 put_page(sdio->cur_page);
860 sdio->cur_page = NULL;
874 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,
882 sdio->start_zero_done = 1;
883 if (!sdio->blkfactor || !buffer_new(map_bh))
886 dio_blocks_per_fs_block = 1 << sdio->blkfactor;
887 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1);
899 this_chunk_bytes = this_chunk_blocks << sdio->blkbits;
902 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,
903 sdio->next_block_for_io, map_bh))
906 sdio->next_block_for_io += this_chunk_blocks;
925 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,
928 const unsigned blkbits = sdio->blkbits;
929 const unsigned i_blkbits = blkbits + sdio->blkfactor;
932 while (sdio->block_in_file < sdio->final_block_in_request) {
936 page = dio_get_page(dio, sdio);
941 from = sdio->head ? 0 : sdio->from;
942 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE;
943 sdio->head++;
950 if (sdio->blocks_available == 0) {
957 ret = get_more_blocks(dio, sdio, map_bh);
965 sdio->blocks_available =
967 sdio->next_block_for_io =
968 map_bh->b_blocknr << sdio->blkfactor;
976 if (!sdio->blkfactor)
979 blkmask = (1 << sdio->blkfactor) - 1;
980 dio_remainder = (sdio->block_in_file & blkmask);
994 sdio->next_block_for_io += dio_remainder;
995 sdio->blocks_available -= dio_remainder;
1014 if (sdio->block_in_file >=
1021 sdio->block_in_file++;
1032 if (unlikely(sdio->blkfactor && !sdio->start_zero_done))
1033 dio_zero_block(dio, sdio, 0, map_bh);
1039 this_chunk_blocks = sdio->blocks_available;
1043 u = sdio->final_block_in_request - sdio->block_in_file;
1049 if (this_chunk_blocks == sdio->blocks_available)
1050 sdio->boundary = buffer_boundary(map_bh);
1051 ret = submit_page_section(dio, sdio, page,
1054 sdio->next_block_for_io,
1060 sdio->next_block_for_io += this_chunk_blocks;
1062 sdio->block_in_file += this_chunk_blocks;
1065 sdio->blocks_available -= this_chunk_blocks;
1067 BUG_ON(sdio->block_in_file > sdio->final_block_in_request);
1068 if (sdio->block_in_file == sdio->final_block_in_request)
1121 * NOTE: if you pass "sdio" to anything by pointer make sure that function
1140 struct dio_submit sdio = { 0, };
1244 sdio.blkbits = blkbits;
1245 sdio.blkfactor = i_blkbits - blkbits;
1246 sdio.block_in_file = offset >> blkbits;
1248 sdio.get_block = get_block;
1250 sdio.submit_io = submit_io;
1251 sdio.final_block_in_bio = -1;
1252 sdio.next_block_for_io = -1;
1260 sdio.iter = iter;
1261 sdio.final_block_in_request = end >> blkbits;
1267 if (unlikely(sdio.blkfactor))
1268 sdio.pages_in_io = 2;
1270 sdio.pages_in_io += iov_iter_npages(iter, INT_MAX);
1274 retval = do_direct_IO(dio, &sdio, &map_bh);
1276 dio_cleanup(dio, &sdio);
1289 dio_zero_block(dio, &sdio, 1, &map_bh);
1291 if (sdio.cur_page) {
1294 ret2 = dio_send_cur_page(dio, &sdio, &map_bh);
1297 put_page(sdio.cur_page);
1298 sdio.cur_page = NULL;
1300 if (sdio.bio)
1301 dio_bio_submit(dio, &sdio);
1309 dio_cleanup(dio, &sdio);