Lines Matching refs:sdio
159 static inline unsigned dio_pages_present(struct dio_submit *sdio)
161 return sdio->tail - sdio->head;
167 static inline int dio_refill_pages(struct dio *dio, struct dio_submit *sdio)
173 ret = iov_iter_extract_pages(sdio->iter, &pages, LONG_MAX,
174 DIO_PAGES, 0, &sdio->from);
176 if (ret < 0 && sdio->blocks_available && dio_op == REQ_OP_WRITE) {
185 sdio->head = 0;
186 sdio->tail = 1;
187 sdio->from = 0;
188 sdio->to = PAGE_SIZE;
193 ret += sdio->from;
194 sdio->head = 0;
195 sdio->tail = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
196 sdio->to = ((ret - 1) & (PAGE_SIZE - 1)) + 1;
209 struct dio_submit *sdio)
211 if (dio_pages_present(sdio) == 0) {
214 ret = dio_refill_pages(dio, sdio);
217 BUG_ON(dio_pages_present(sdio) == 0);
219 return dio->pages[sdio->head];
395 dio_bio_alloc(struct dio *dio, struct dio_submit *sdio,
413 sdio->bio = bio;
414 sdio->logical_offset_in_bio = sdio->cur_page_fs_offset;
424 static inline void dio_bio_submit(struct dio *dio, struct dio_submit *sdio)
427 struct bio *bio = sdio->bio;
443 sdio->bio = NULL;
444 sdio->boundary = 0;
445 sdio->logical_offset_in_bio = 0;
451 static inline void dio_cleanup(struct dio *dio, struct dio_submit *sdio)
454 unpin_user_pages(dio->pages + sdio->head,
455 sdio->tail - sdio->head);
456 sdio->head = sdio->tail;
544 static inline int dio_bio_reap(struct dio *dio, struct dio_submit *sdio)
548 if (sdio->reap_counter++ >= 64) {
562 sdio->reap_counter = 0;
581 * of available blocks at sdio->blocks_available. These are in units of the
602 static int get_more_blocks(struct dio *dio, struct dio_submit *sdio,
611 unsigned int i_blkbits = sdio->blkbits + sdio->blkfactor;
620 BUG_ON(sdio->block_in_file >= sdio->final_block_in_request);
621 fs_startblk = sdio->block_in_file >> sdio->blkfactor;
622 fs_endblk = (sdio->final_block_in_request - 1) >>
623 sdio->blkfactor;
647 ret = (*sdio->get_block)(dio->inode, fs_startblk,
662 static inline int dio_new_bio(struct dio *dio, struct dio_submit *sdio,
668 ret = dio_bio_reap(dio, sdio);
671 sector = start_sector << (sdio->blkbits - 9);
672 nr_pages = bio_max_segs(sdio->pages_in_io);
674 dio_bio_alloc(dio, sdio, map_bh->b_bdev, sector, nr_pages);
675 sdio->boundary = 0;
687 static inline int dio_bio_add_page(struct dio *dio, struct dio_submit *sdio)
691 ret = bio_add_page(sdio->bio, sdio->cur_page,
692 sdio->cur_page_len, sdio->cur_page_offset);
693 if (ret == sdio->cur_page_len) {
697 if ((sdio->cur_page_len + sdio->cur_page_offset) == PAGE_SIZE)
698 sdio->pages_in_io--;
699 dio_pin_page(dio, sdio->cur_page);
700 sdio->final_block_in_bio = sdio->cur_page_block +
701 (sdio->cur_page_len >> sdio->blkbits);
719 static inline int dio_send_cur_page(struct dio *dio, struct dio_submit *sdio,
724 if (sdio->bio) {
725 loff_t cur_offset = sdio->cur_page_fs_offset;
726 loff_t bio_next_offset = sdio->logical_offset_in_bio +
727 sdio->bio->bi_iter.bi_size;
743 if (sdio->final_block_in_bio != sdio->cur_page_block ||
745 dio_bio_submit(dio, sdio);
748 if (sdio->bio == NULL) {
749 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
754 if (dio_bio_add_page(dio, sdio) != 0) {
755 dio_bio_submit(dio, sdio);
756 ret = dio_new_bio(dio, sdio, sdio->cur_page_block, map_bh);
758 ret = dio_bio_add_page(dio, sdio);
784 submit_page_section(struct dio *dio, struct dio_submit *sdio, struct page *page,
790 int boundary = sdio->boundary; /* dio_send_cur_page may clear it */
802 if (sdio->cur_page == page &&
803 sdio->cur_page_offset + sdio->cur_page_len == offset &&
804 sdio->cur_page_block +
805 (sdio->cur_page_len >> sdio->blkbits) == blocknr) {
806 sdio->cur_page_len += len;
813 if (sdio->cur_page) {
814 ret = dio_send_cur_page(dio, sdio, map_bh);
815 dio_unpin_page(dio, sdio->cur_page);
816 sdio->cur_page = NULL;
822 sdio->cur_page = page;
823 sdio->cur_page_offset = offset;
824 sdio->cur_page_len = len;
825 sdio->cur_page_block = blocknr;
826 sdio->cur_page_fs_offset = sdio->block_in_file << sdio->blkbits;
833 ret = dio_send_cur_page(dio, sdio, map_bh);
834 if (sdio->bio)
835 dio_bio_submit(dio, sdio);
836 dio_unpin_page(dio, sdio->cur_page);
837 sdio->cur_page = NULL;
851 static inline void dio_zero_block(struct dio *dio, struct dio_submit *sdio,
859 sdio->start_zero_done = 1;
860 if (!sdio->blkfactor || !buffer_new(map_bh))
863 dio_blocks_per_fs_block = 1 << sdio->blkfactor;
864 this_chunk_blocks = sdio->block_in_file & (dio_blocks_per_fs_block - 1);
876 this_chunk_bytes = this_chunk_blocks << sdio->blkbits;
879 if (submit_page_section(dio, sdio, page, 0, this_chunk_bytes,
880 sdio->next_block_for_io, map_bh))
883 sdio->next_block_for_io += this_chunk_blocks;
902 static int do_direct_IO(struct dio *dio, struct dio_submit *sdio,
906 const unsigned blkbits = sdio->blkbits;
907 const unsigned i_blkbits = blkbits + sdio->blkfactor;
910 while (sdio->block_in_file < sdio->final_block_in_request) {
914 page = dio_get_page(dio, sdio);
919 from = sdio->head ? 0 : sdio->from;
920 to = (sdio->head == sdio->tail - 1) ? sdio->to : PAGE_SIZE;
921 sdio->head++;
928 if (sdio->blocks_available == 0) {
935 ret = get_more_blocks(dio, sdio, map_bh);
943 sdio->blocks_available =
945 sdio->next_block_for_io =
946 map_bh->b_blocknr << sdio->blkfactor;
954 if (!sdio->blkfactor)
957 blkmask = (1 << sdio->blkfactor) - 1;
958 dio_remainder = (sdio->block_in_file & blkmask);
972 sdio->next_block_for_io += dio_remainder;
973 sdio->blocks_available -= dio_remainder;
992 if (sdio->block_in_file >=
999 sdio->block_in_file++;
1010 if (unlikely(sdio->blkfactor && !sdio->start_zero_done))
1011 dio_zero_block(dio, sdio, 0, map_bh);
1017 this_chunk_blocks = sdio->blocks_available;
1021 u = sdio->final_block_in_request - sdio->block_in_file;
1027 if (this_chunk_blocks == sdio->blocks_available)
1028 sdio->boundary = buffer_boundary(map_bh);
1029 ret = submit_page_section(dio, sdio, page,
1032 sdio->next_block_for_io,
1038 sdio->next_block_for_io += this_chunk_blocks;
1040 sdio->block_in_file += this_chunk_blocks;
1043 sdio->blocks_available -= this_chunk_blocks;
1045 BUG_ON(sdio->block_in_file > sdio->final_block_in_request);
1046 if (sdio->block_in_file == sdio->final_block_in_request)
1099 * NOTE: if you pass "sdio" to anything by pointer make sure that function
1117 struct dio_submit sdio = { 0, };
1219 sdio.blkbits = blkbits;
1220 sdio.blkfactor = i_blkbits - blkbits;
1221 sdio.block_in_file = offset >> blkbits;
1223 sdio.get_block = get_block;
1225 sdio.final_block_in_bio = -1;
1226 sdio.next_block_for_io = -1;
1234 sdio.iter = iter;
1235 sdio.final_block_in_request = end >> blkbits;
1241 if (unlikely(sdio.blkfactor))
1242 sdio.pages_in_io = 2;
1244 sdio.pages_in_io += iov_iter_npages(iter, INT_MAX);
1248 retval = do_direct_IO(dio, &sdio, &map_bh);
1250 dio_cleanup(dio, &sdio);
1263 dio_zero_block(dio, &sdio, 1, &map_bh);
1265 if (sdio.cur_page) {
1268 ret2 = dio_send_cur_page(dio, &sdio, &map_bh);
1271 dio_unpin_page(dio, sdio.cur_page);
1272 sdio.cur_page = NULL;
1274 if (sdio.bio)
1275 dio_bio_submit(dio, &sdio);
1283 dio_cleanup(dio, &sdio);