Lines Matching refs:dio
56 struct iomap_dio *dio, unsigned short nr_vecs, blk_opf_t opf)
58 if (dio->dops && dio->dops->bio_set)
60 GFP_KERNEL, dio->dops->bio_set);
65 struct iomap_dio *dio, struct bio *bio, loff_t pos)
67 struct kiocb *iocb = dio->iocb;
69 atomic_inc(&dio->ref);
71 /* Sync dio can't be polled reliably */
77 if (dio->dops && dio->dops->submit_io)
78 dio->dops->submit_io(iter, bio, pos);
83 ssize_t iomap_dio_complete(struct iomap_dio *dio)
85 const struct iomap_dio_ops *dops = dio->dops;
86 struct kiocb *iocb = dio->iocb;
88 ssize_t ret = dio->error;
91 ret = dops->end_io(iocb, dio->size, ret, dio->flags);
94 ret = dio->size;
96 if (offset + ret > dio->i_size &&
97 !(dio->flags & IOMAP_DIO_WRITE))
98 ret = dio->i_size - offset;
113 if (!dio->error && dio->size && (dio->flags & IOMAP_DIO_WRITE))
114 kiocb_invalidate_post_direct_write(iocb, dio->size);
125 if (dio->flags & IOMAP_DIO_NEED_SYNC)
128 ret += dio->done_before;
130 trace_iomap_dio_complete(iocb, dio->error, ret);
131 kfree(dio);
143 struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);
144 struct kiocb *iocb = dio->iocb;
146 iocb->ki_complete(iocb, iomap_dio_complete(dio));
150 * Set an error in the dio if none is set yet. We have to use cmpxchg
154 static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)
156 cmpxchg(&dio->error, 0, ret);
161 struct iomap_dio *dio = bio->bi_private;
162 bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);
163 struct kiocb *iocb = dio->iocb;
166 iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));
167 if (!atomic_dec_and_test(&dio->ref))
171 * Synchronous dio, task itself will handle any completion work
174 if (dio->wait_for_completion) {
175 struct task_struct *waiter = dio->submit.waiter;
177 WRITE_ONCE(dio->submit.waiter, NULL);
185 if (dio->flags & IOMAP_DIO_INLINE_COMP) {
187 iomap_dio_complete_work(&dio->aio.work);
192 * If this dio is flagged with IOMAP_DIO_CALLER_COMP, then schedule
195 if (dio->flags & IOMAP_DIO_CALLER_COMP) {
197 iocb->private = dio;
222 INIT_WORK(&dio->aio.work, iomap_dio_complete_work);
224 &dio->aio.work);
235 static void iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio,
238 struct inode *inode = file_inode(dio->iocb->ki_filp);
242 bio = iomap_dio_alloc_bio(iter, dio, 1, REQ_OP_WRITE | REQ_SYNC | REQ_IDLE);
246 bio->bi_private = dio;
250 iomap_dio_submit_bio(iter, dio, bio, pos);
254 * Figure out the bio's operation flags from the dio request, the
256 * clearing the WRITE_THROUGH flag in the dio request.
258 static inline blk_opf_t iomap_dio_bio_opflags(struct iomap_dio *dio,
263 if (!(dio->flags & IOMAP_DIO_WRITE))
270 dio->flags &= ~IOMAP_DIO_WRITE_THROUGH;
276 struct iomap_dio *dio)
292 !bdev_iter_is_aligned(iomap->bdev, dio->submit.iter))
296 dio->flags |= IOMAP_DIO_UNWRITTEN;
301 dio->flags |= IOMAP_DIO_COW;
313 * sync, disable in-task completions as dio completion will
318 (dio->flags & IOMAP_DIO_WRITE_THROUGH) &&
321 else if (dio->flags & IOMAP_DIO_NEED_SYNC)
322 dio->flags &= ~IOMAP_DIO_CALLER_COMP;
330 orig_count = iov_iter_count(dio->submit.iter);
331 iov_iter_truncate(dio->submit.iter, length);
333 if (!iov_iter_count(dio->submit.iter))
344 ((dio->flags & IOMAP_DIO_NEED_SYNC) && !use_fua) ||
345 ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode)))
346 dio->flags &= ~IOMAP_DIO_CALLER_COMP;
353 if (!(dio->flags & (IOMAP_DIO_INLINE_COMP|IOMAP_DIO_CALLER_COMP)))
354 dio->iocb->ki_flags &= ~IOCB_HIPRI;
360 iomap_dio_zero(iter, dio, pos - pad, pad);
368 bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua);
370 nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS);
373 if (dio->error) {
374 iov_iter_revert(dio->submit.iter, copied);
379 bio = iomap_dio_alloc_bio(iter, dio, nr_pages, bio_opf);
383 bio->bi_ioprio = dio->iocb->ki_ioprio;
384 bio->bi_private = dio;
387 ret = bio_iov_iter_get_pages(bio, dio->submit.iter);
400 if (dio->flags & IOMAP_DIO_WRITE) {
403 if (dio->flags & IOMAP_DIO_DIRTY)
407 dio->size += n;
410 nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter,
416 dio->iocb->ki_flags &= ~IOCB_HIPRI;
417 iomap_dio_submit_bio(iter, dio, bio, pos);
429 ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {
433 iomap_dio_zero(iter, dio, pos, fs_block_size - pad);
437 iov_iter_reexpand(dio->submit.iter, orig_count - copied);
444 struct iomap_dio *dio)
446 loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter);
448 dio->size += length;
455 struct iomap_dio *dio)
458 struct iov_iter *iter = dio->submit.iter;
467 if (dio->flags & IOMAP_DIO_WRITE) {
481 dio->size += copied;
488 struct iomap_dio *dio)
492 if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))
494 return iomap_dio_hole_iter(iter, dio);
496 if (!(dio->flags & IOMAP_DIO_WRITE))
497 return iomap_dio_hole_iter(iter, dio);
498 return iomap_dio_bio_iter(iter, dio);
500 return iomap_dio_bio_iter(iter, dio);
502 return iomap_dio_inline_iter(iter, dio);
511 dio->iocb->ki_filp, current->comm);
555 struct iomap_dio *dio;
563 dio = kmalloc(sizeof(*dio), GFP_KERNEL);
564 if (!dio)
567 dio->iocb = iocb;
568 atomic_set(&dio->ref, 1);
569 dio->size = 0;
570 dio->i_size = i_size_read(inode);
571 dio->dops = dops;
572 dio->error = 0;
573 dio->flags = 0;
574 dio->done_before = done_before;
576 dio->submit.iter = iter;
577 dio->submit.waiter = current;
584 dio->flags |= IOMAP_DIO_INLINE_COMP;
586 if (iomi.pos >= dio->i_size)
590 dio->flags |= IOMAP_DIO_DIRTY;
597 dio->flags |= IOMAP_DIO_WRITE;
606 dio->flags |= IOMAP_DIO_CALLER_COMP;
610 if (iomi.pos >= dio->i_size ||
611 iomi.pos + iomi.len > dio->i_size)
618 dio->flags |= IOMAP_DIO_NEED_SYNC;
630 dio->flags |= IOMAP_DIO_WRITE_THROUGH;
659 iomi.processed = iomap_dio_iter(&iomi, dio);
674 if (iov_iter_rw(iter) == READ && iomi.pos >= dio->i_size)
675 iov_iter_revert(iter, iomi.pos - dio->i_size);
677 if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) {
689 iomap_dio_set_error(dio, ret);
696 if (dio->flags & IOMAP_DIO_WRITE_THROUGH)
697 dio->flags &= ~IOMAP_DIO_NEED_SYNC;
701 * might be the last reference to the dio. There are three different
705 * the dio ourselves.
707 * iocb, we must never touch the dio after the decrement, the
714 dio->wait_for_completion = wait_for_completion;
715 if (!atomic_dec_and_test(&dio->ref)) {
723 if (!READ_ONCE(dio->submit.waiter))
731 return dio;
734 kfree(dio);
746 struct iomap_dio *dio;
748 dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private,
750 if (IS_ERR_OR_NULL(dio))
751 return PTR_ERR_OR_ZERO(dio);
752 return iomap_dio_complete(dio);