1// SPDX-License-Identifier: GPL-2.0-or-later 2/* kiocb-using read/write 3 * 4 * Copyright (C) 2021 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8#include <linux/mount.h> 9#include <linux/slab.h> 10#include <linux/file.h> 11#include <linux/uio.h> 12#include <linux/falloc.h> 13#include <linux/sched/mm.h> 14#include <trace/events/fscache.h> 15#include "internal.h" 16 17struct cachefiles_kiocb { 18 struct kiocb iocb; 19 refcount_t ki_refcnt; 20 loff_t start; 21 union { 22 size_t skipped; 23 size_t len; 24 }; 25 struct cachefiles_object *object; 26 netfs_io_terminated_t term_func; 27 void *term_func_priv; 28 bool was_async; 29 unsigned int inval_counter; /* Copy of cookie->inval_counter */ 30 u64 b_writing; 31}; 32 33static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki) 34{ 35 if (refcount_dec_and_test(&ki->ki_refcnt)) { 36 cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq); 37 fput(ki->iocb.ki_filp); 38 kfree(ki); 39 } 40} 41 42/* 43 * Handle completion of a read from the cache. 44 */ 45static void cachefiles_read_complete(struct kiocb *iocb, long ret) 46{ 47 struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb); 48 struct inode *inode = file_inode(ki->iocb.ki_filp); 49 50 _enter("%ld", ret); 51 52 if (ret < 0) 53 trace_cachefiles_io_error(ki->object, inode, ret, 54 cachefiles_trace_read_error); 55 56 if (ki->term_func) { 57 if (ret >= 0) { 58 if (ki->object->cookie->inval_counter == ki->inval_counter) 59 ki->skipped += ret; 60 else 61 ret = -ESTALE; 62 } 63 64 ki->term_func(ki->term_func_priv, ret, ki->was_async); 65 } 66 67 cachefiles_put_kiocb(ki); 68} 69 70/* 71 * Initiate a read from the cache. 72 */ 73static int cachefiles_read(struct netfs_cache_resources *cres, 74 loff_t start_pos, 75 struct iov_iter *iter, 76 enum netfs_read_from_hole read_hole, 77 netfs_io_terminated_t term_func, 78 void *term_func_priv) 79{ 80 struct cachefiles_object *object; 81 struct cachefiles_kiocb *ki; 82 struct file *file; 83 unsigned int old_nofs; 84 ssize_t ret = -ENOBUFS; 85 size_t len = iov_iter_count(iter), skipped = 0; 86 87 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) 88 goto presubmission_error; 89 90 fscache_count_read(); 91 object = cachefiles_cres_object(cres); 92 file = cachefiles_cres_file(cres); 93 94 _enter("%pD,%li,%llx,%zx/%llx", 95 file, file_inode(file)->i_ino, start_pos, len, 96 i_size_read(file_inode(file))); 97 98 /* If the caller asked us to seek for data before doing the read, then 99 * we should do that now. If we find a gap, we fill it with zeros. 100 */ 101 if (read_hole != NETFS_READ_HOLE_IGNORE) { 102 loff_t off = start_pos, off2; 103 104 off2 = cachefiles_inject_read_error(); 105 if (off2 == 0) 106 off2 = vfs_llseek(file, off, SEEK_DATA); 107 if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO && off2 != -ENXIO) { 108 skipped = 0; 109 ret = off2; 110 goto presubmission_error; 111 } 112 113 if (off2 == -ENXIO || off2 >= start_pos + len) { 114 /* The region is beyond the EOF or there's no more data 115 * in the region, so clear the rest of the buffer and 116 * return success. 117 */ 118 ret = -ENODATA; 119 if (read_hole == NETFS_READ_HOLE_FAIL) 120 goto presubmission_error; 121 122 iov_iter_zero(len, iter); 123 skipped = len; 124 ret = 0; 125 goto presubmission_error; 126 } 127 128 skipped = off2 - off; 129 iov_iter_zero(skipped, iter); 130 } 131 132 ret = -ENOMEM; 133 ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL); 134 if (!ki) 135 goto presubmission_error; 136 137 refcount_set(&ki->ki_refcnt, 2); 138 ki->iocb.ki_filp = file; 139 ki->iocb.ki_pos = start_pos + skipped; 140 ki->iocb.ki_flags = IOCB_DIRECT; 141 ki->iocb.ki_ioprio = get_current_ioprio(); 142 ki->skipped = skipped; 143 ki->object = object; 144 ki->inval_counter = cres->inval_counter; 145 ki->term_func = term_func; 146 ki->term_func_priv = term_func_priv; 147 ki->was_async = true; 148 149 if (ki->term_func) 150 ki->iocb.ki_complete = cachefiles_read_complete; 151 152 get_file(ki->iocb.ki_filp); 153 cachefiles_grab_object(object, cachefiles_obj_get_ioreq); 154 155 trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped); 156 old_nofs = memalloc_nofs_save(); 157 ret = cachefiles_inject_read_error(); 158 if (ret == 0) 159 ret = vfs_iocb_iter_read(file, &ki->iocb, iter); 160 memalloc_nofs_restore(old_nofs); 161 switch (ret) { 162 case -EIOCBQUEUED: 163 goto in_progress; 164 165 case -ERESTARTSYS: 166 case -ERESTARTNOINTR: 167 case -ERESTARTNOHAND: 168 case -ERESTART_RESTARTBLOCK: 169 /* There's no easy way to restart the syscall since other AIO's 170 * may be already running. Just fail this IO with EINTR. 171 */ 172 ret = -EINTR; 173 fallthrough; 174 default: 175 ki->was_async = false; 176 cachefiles_read_complete(&ki->iocb, ret); 177 if (ret > 0) 178 ret = 0; 179 break; 180 } 181 182in_progress: 183 cachefiles_put_kiocb(ki); 184 _leave(" = %zd", ret); 185 return ret; 186 187presubmission_error: 188 if (term_func) 189 term_func(term_func_priv, ret < 0 ? ret : skipped, false); 190 return ret; 191} 192 193/* 194 * Query the occupancy of the cache in a region, returning where the next chunk 195 * of data starts and how long it is. 196 */ 197static int cachefiles_query_occupancy(struct netfs_cache_resources *cres, 198 loff_t start, size_t len, size_t granularity, 199 loff_t *_data_start, size_t *_data_len) 200{ 201 struct cachefiles_object *object; 202 struct file *file; 203 loff_t off, off2; 204 205 *_data_start = -1; 206 *_data_len = 0; 207 208 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) 209 return -ENOBUFS; 210 211 object = cachefiles_cres_object(cres); 212 file = cachefiles_cres_file(cres); 213 granularity = max_t(size_t, object->volume->cache->bsize, granularity); 214 215 _enter("%pD,%li,%llx,%zx/%llx", 216 file, file_inode(file)->i_ino, start, len, 217 i_size_read(file_inode(file))); 218 219 off = cachefiles_inject_read_error(); 220 if (off == 0) 221 off = vfs_llseek(file, start, SEEK_DATA); 222 if (off == -ENXIO) 223 return -ENODATA; /* Beyond EOF */ 224 if (off < 0 && off >= (loff_t)-MAX_ERRNO) 225 return -ENOBUFS; /* Error. */ 226 if (round_up(off, granularity) >= start + len) 227 return -ENODATA; /* No data in range */ 228 229 off2 = cachefiles_inject_read_error(); 230 if (off2 == 0) 231 off2 = vfs_llseek(file, off, SEEK_HOLE); 232 if (off2 == -ENXIO) 233 return -ENODATA; /* Beyond EOF */ 234 if (off2 < 0 && off2 >= (loff_t)-MAX_ERRNO) 235 return -ENOBUFS; /* Error. */ 236 237 /* Round away partial blocks */ 238 off = round_up(off, granularity); 239 off2 = round_down(off2, granularity); 240 if (off2 <= off) 241 return -ENODATA; 242 243 *_data_start = off; 244 if (off2 > start + len) 245 *_data_len = len; 246 else 247 *_data_len = off2 - off; 248 return 0; 249} 250 251/* 252 * Handle completion of a write to the cache. 253 */ 254static void cachefiles_write_complete(struct kiocb *iocb, long ret) 255{ 256 struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb); 257 struct cachefiles_object *object = ki->object; 258 struct inode *inode = file_inode(ki->iocb.ki_filp); 259 260 _enter("%ld", ret); 261 262 kiocb_end_write(iocb); 263 264 if (ret < 0) 265 trace_cachefiles_io_error(object, inode, ret, 266 cachefiles_trace_write_error); 267 268 atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing); 269 set_bit(FSCACHE_COOKIE_HAVE_DATA, &object->cookie->flags); 270 if (ki->term_func) 271 ki->term_func(ki->term_func_priv, ret, ki->was_async); 272 cachefiles_put_kiocb(ki); 273} 274 275/* 276 * Initiate a write to the cache. 277 */ 278int __cachefiles_write(struct cachefiles_object *object, 279 struct file *file, 280 loff_t start_pos, 281 struct iov_iter *iter, 282 netfs_io_terminated_t term_func, 283 void *term_func_priv) 284{ 285 struct cachefiles_cache *cache; 286 struct cachefiles_kiocb *ki; 287 unsigned int old_nofs; 288 ssize_t ret; 289 size_t len = iov_iter_count(iter); 290 291 fscache_count_write(); 292 cache = object->volume->cache; 293 294 _enter("%pD,%li,%llx,%zx/%llx", 295 file, file_inode(file)->i_ino, start_pos, len, 296 i_size_read(file_inode(file))); 297 298 ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL); 299 if (!ki) { 300 if (term_func) 301 term_func(term_func_priv, -ENOMEM, false); 302 return -ENOMEM; 303 } 304 305 refcount_set(&ki->ki_refcnt, 2); 306 ki->iocb.ki_filp = file; 307 ki->iocb.ki_pos = start_pos; 308 ki->iocb.ki_flags = IOCB_DIRECT | IOCB_WRITE; 309 ki->iocb.ki_ioprio = get_current_ioprio(); 310 ki->object = object; 311 ki->start = start_pos; 312 ki->len = len; 313 ki->term_func = term_func; 314 ki->term_func_priv = term_func_priv; 315 ki->was_async = true; 316 ki->b_writing = (len + (1 << cache->bshift) - 1) >> cache->bshift; 317 318 if (ki->term_func) 319 ki->iocb.ki_complete = cachefiles_write_complete; 320 atomic_long_add(ki->b_writing, &cache->b_writing); 321 322 kiocb_start_write(&ki->iocb); 323 324 get_file(ki->iocb.ki_filp); 325 cachefiles_grab_object(object, cachefiles_obj_get_ioreq); 326 327 trace_cachefiles_write(object, file_inode(file), ki->iocb.ki_pos, len); 328 old_nofs = memalloc_nofs_save(); 329 ret = cachefiles_inject_write_error(); 330 if (ret == 0) 331 ret = vfs_iocb_iter_write(file, &ki->iocb, iter); 332 memalloc_nofs_restore(old_nofs); 333 switch (ret) { 334 case -EIOCBQUEUED: 335 goto in_progress; 336 337 case -ERESTARTSYS: 338 case -ERESTARTNOINTR: 339 case -ERESTARTNOHAND: 340 case -ERESTART_RESTARTBLOCK: 341 /* There's no easy way to restart the syscall since other AIO's 342 * may be already running. Just fail this IO with EINTR. 343 */ 344 ret = -EINTR; 345 fallthrough; 346 default: 347 ki->was_async = false; 348 cachefiles_write_complete(&ki->iocb, ret); 349 if (ret > 0) 350 ret = 0; 351 break; 352 } 353 354in_progress: 355 cachefiles_put_kiocb(ki); 356 _leave(" = %zd", ret); 357 return ret; 358} 359 360static int cachefiles_write(struct netfs_cache_resources *cres, 361 loff_t start_pos, 362 struct iov_iter *iter, 363 netfs_io_terminated_t term_func, 364 void *term_func_priv) 365{ 366 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) { 367 if (term_func) 368 term_func(term_func_priv, -ENOBUFS, false); 369 return -ENOBUFS; 370 } 371 372 return __cachefiles_write(cachefiles_cres_object(cres), 373 cachefiles_cres_file(cres), 374 start_pos, iter, 375 term_func, term_func_priv); 376} 377 378static inline enum netfs_io_source 379cachefiles_do_prepare_read(struct netfs_cache_resources *cres, 380 loff_t start, size_t *_len, loff_t i_size, 381 unsigned long *_flags, ino_t netfs_ino) 382{ 383 enum cachefiles_prepare_read_trace why; 384 struct cachefiles_object *object = NULL; 385 struct cachefiles_cache *cache; 386 struct fscache_cookie *cookie = fscache_cres_cookie(cres); 387 const struct cred *saved_cred; 388 struct file *file = cachefiles_cres_file(cres); 389 enum netfs_io_source ret = NETFS_DOWNLOAD_FROM_SERVER; 390 size_t len = *_len; 391 loff_t off, to; 392 ino_t ino = file ? file_inode(file)->i_ino : 0; 393 int rc; 394 395 _enter("%zx @%llx/%llx", len, start, i_size); 396 397 if (start >= i_size) { 398 ret = NETFS_FILL_WITH_ZEROES; 399 why = cachefiles_trace_read_after_eof; 400 goto out_no_object; 401 } 402 403 if (test_bit(FSCACHE_COOKIE_NO_DATA_TO_READ, &cookie->flags)) { 404 __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags); 405 why = cachefiles_trace_read_no_data; 406 if (!test_bit(NETFS_SREQ_ONDEMAND, _flags)) 407 goto out_no_object; 408 } 409 410 /* The object and the file may be being created in the background. */ 411 if (!file) { 412 why = cachefiles_trace_read_no_file; 413 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_READ)) 414 goto out_no_object; 415 file = cachefiles_cres_file(cres); 416 if (!file) 417 goto out_no_object; 418 ino = file_inode(file)->i_ino; 419 } 420 421 object = cachefiles_cres_object(cres); 422 cache = object->volume->cache; 423 cachefiles_begin_secure(cache, &saved_cred); 424retry: 425 off = cachefiles_inject_read_error(); 426 if (off == 0) 427 off = vfs_llseek(file, start, SEEK_DATA); 428 if (off < 0 && off >= (loff_t)-MAX_ERRNO) { 429 if (off == (loff_t)-ENXIO) { 430 why = cachefiles_trace_read_seek_nxio; 431 goto download_and_store; 432 } 433 trace_cachefiles_io_error(object, file_inode(file), off, 434 cachefiles_trace_seek_error); 435 why = cachefiles_trace_read_seek_error; 436 goto out; 437 } 438 439 if (off >= start + len) { 440 why = cachefiles_trace_read_found_hole; 441 goto download_and_store; 442 } 443 444 if (off > start) { 445 off = round_up(off, cache->bsize); 446 len = off - start; 447 *_len = len; 448 why = cachefiles_trace_read_found_part; 449 goto download_and_store; 450 } 451 452 to = cachefiles_inject_read_error(); 453 if (to == 0) 454 to = vfs_llseek(file, start, SEEK_HOLE); 455 if (to < 0 && to >= (loff_t)-MAX_ERRNO) { 456 trace_cachefiles_io_error(object, file_inode(file), to, 457 cachefiles_trace_seek_error); 458 why = cachefiles_trace_read_seek_error; 459 goto out; 460 } 461 462 if (to < start + len) { 463 if (start + len >= i_size) 464 to = round_up(to, cache->bsize); 465 else 466 to = round_down(to, cache->bsize); 467 len = to - start; 468 *_len = len; 469 } 470 471 why = cachefiles_trace_read_have_data; 472 ret = NETFS_READ_FROM_CACHE; 473 goto out; 474 475download_and_store: 476 __set_bit(NETFS_SREQ_COPY_TO_CACHE, _flags); 477 if (test_bit(NETFS_SREQ_ONDEMAND, _flags)) { 478 rc = cachefiles_ondemand_read(object, start, len); 479 if (!rc) { 480 __clear_bit(NETFS_SREQ_ONDEMAND, _flags); 481 goto retry; 482 } 483 ret = NETFS_INVALID_READ; 484 } 485out: 486 cachefiles_end_secure(cache, saved_cred); 487out_no_object: 488 trace_cachefiles_prep_read(object, start, len, *_flags, ret, why, ino, netfs_ino); 489 return ret; 490} 491 492/* 493 * Prepare a read operation, shortening it to a cached/uncached 494 * boundary as appropriate. 495 */ 496static enum netfs_io_source cachefiles_prepare_read(struct netfs_io_subrequest *subreq, 497 loff_t i_size) 498{ 499 return cachefiles_do_prepare_read(&subreq->rreq->cache_resources, 500 subreq->start, &subreq->len, i_size, 501 &subreq->flags, subreq->rreq->inode->i_ino); 502} 503 504/* 505 * Prepare an on-demand read operation, shortening it to a cached/uncached 506 * boundary as appropriate. 507 */ 508static enum netfs_io_source 509cachefiles_prepare_ondemand_read(struct netfs_cache_resources *cres, 510 loff_t start, size_t *_len, loff_t i_size, 511 unsigned long *_flags, ino_t ino) 512{ 513 return cachefiles_do_prepare_read(cres, start, _len, i_size, _flags, ino); 514} 515 516/* 517 * Prepare for a write to occur. 518 */ 519int __cachefiles_prepare_write(struct cachefiles_object *object, 520 struct file *file, 521 loff_t *_start, size_t *_len, 522 bool no_space_allocated_yet) 523{ 524 struct cachefiles_cache *cache = object->volume->cache; 525 loff_t start = *_start, pos; 526 size_t len = *_len, down; 527 int ret; 528 529 /* Round to DIO size */ 530 down = start - round_down(start, PAGE_SIZE); 531 *_start = start - down; 532 *_len = round_up(down + len, PAGE_SIZE); 533 534 /* We need to work out whether there's sufficient disk space to perform 535 * the write - but we can skip that check if we have space already 536 * allocated. 537 */ 538 if (no_space_allocated_yet) 539 goto check_space; 540 541 pos = cachefiles_inject_read_error(); 542 if (pos == 0) 543 pos = vfs_llseek(file, *_start, SEEK_DATA); 544 if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) { 545 if (pos == -ENXIO) 546 goto check_space; /* Unallocated tail */ 547 trace_cachefiles_io_error(object, file_inode(file), pos, 548 cachefiles_trace_seek_error); 549 return pos; 550 } 551 if ((u64)pos >= (u64)*_start + *_len) 552 goto check_space; /* Unallocated region */ 553 554 /* We have a block that's at least partially filled - if we're low on 555 * space, we need to see if it's fully allocated. If it's not, we may 556 * want to cull it. 557 */ 558 if (cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, 559 cachefiles_has_space_check) == 0) 560 return 0; /* Enough space to simply overwrite the whole block */ 561 562 pos = cachefiles_inject_read_error(); 563 if (pos == 0) 564 pos = vfs_llseek(file, *_start, SEEK_HOLE); 565 if (pos < 0 && pos >= (loff_t)-MAX_ERRNO) { 566 trace_cachefiles_io_error(object, file_inode(file), pos, 567 cachefiles_trace_seek_error); 568 return pos; 569 } 570 if ((u64)pos >= (u64)*_start + *_len) 571 return 0; /* Fully allocated */ 572 573 /* Partially allocated, but insufficient space: cull. */ 574 fscache_count_no_write_space(); 575 ret = cachefiles_inject_remove_error(); 576 if (ret == 0) 577 ret = vfs_fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 578 *_start, *_len); 579 if (ret < 0) { 580 trace_cachefiles_io_error(object, file_inode(file), ret, 581 cachefiles_trace_fallocate_error); 582 cachefiles_io_error_obj(object, 583 "CacheFiles: fallocate failed (%d)\n", ret); 584 ret = -EIO; 585 } 586 587 return ret; 588 589check_space: 590 return cachefiles_has_space(cache, 0, *_len / PAGE_SIZE, 591 cachefiles_has_space_for_write); 592} 593 594static int cachefiles_prepare_write(struct netfs_cache_resources *cres, 595 loff_t *_start, size_t *_len, loff_t i_size, 596 bool no_space_allocated_yet) 597{ 598 struct cachefiles_object *object = cachefiles_cres_object(cres); 599 struct cachefiles_cache *cache = object->volume->cache; 600 const struct cred *saved_cred; 601 int ret; 602 603 if (!cachefiles_cres_file(cres)) { 604 if (!fscache_wait_for_operation(cres, FSCACHE_WANT_WRITE)) 605 return -ENOBUFS; 606 if (!cachefiles_cres_file(cres)) 607 return -ENOBUFS; 608 } 609 610 cachefiles_begin_secure(cache, &saved_cred); 611 ret = __cachefiles_prepare_write(object, cachefiles_cres_file(cres), 612 _start, _len, 613 no_space_allocated_yet); 614 cachefiles_end_secure(cache, saved_cred); 615 return ret; 616} 617 618/* 619 * Clean up an operation. 620 */ 621static void cachefiles_end_operation(struct netfs_cache_resources *cres) 622{ 623 struct file *file = cachefiles_cres_file(cres); 624 625 if (file) 626 fput(file); 627 fscache_end_cookie_access(fscache_cres_cookie(cres), fscache_access_io_end); 628} 629 630static const struct netfs_cache_ops cachefiles_netfs_cache_ops = { 631 .end_operation = cachefiles_end_operation, 632 .read = cachefiles_read, 633 .write = cachefiles_write, 634 .prepare_read = cachefiles_prepare_read, 635 .prepare_write = cachefiles_prepare_write, 636 .prepare_ondemand_read = cachefiles_prepare_ondemand_read, 637 .query_occupancy = cachefiles_query_occupancy, 638}; 639 640/* 641 * Open the cache file when beginning a cache operation. 642 */ 643bool cachefiles_begin_operation(struct netfs_cache_resources *cres, 644 enum fscache_want_state want_state) 645{ 646 struct cachefiles_object *object = cachefiles_cres_object(cres); 647 648 if (!cachefiles_cres_file(cres)) { 649 cres->ops = &cachefiles_netfs_cache_ops; 650 if (object->file) { 651 spin_lock(&object->lock); 652 if (!cres->cache_priv2 && object->file) 653 cres->cache_priv2 = get_file(object->file); 654 spin_unlock(&object->lock); 655 } 656 } 657 658 if (!cachefiles_cres_file(cres) && want_state != FSCACHE_WANT_PARAMS) { 659 pr_err("failed to get cres->file\n"); 660 return false; 661 } 662 663 return true; 664} 665