1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright 1993 by Theodore Ts'o. 4 */ 5#include <linux/module.h> 6#include <linux/moduleparam.h> 7#include <linux/sched.h> 8#include <linux/fs.h> 9#include <linux/pagemap.h> 10#include <linux/file.h> 11#include <linux/stat.h> 12#include <linux/errno.h> 13#include <linux/major.h> 14#include <linux/wait.h> 15#include <linux/blkpg.h> 16#include <linux/init.h> 17#include <linux/swap.h> 18#include <linux/slab.h> 19#include <linux/compat.h> 20#include <linux/suspend.h> 21#include <linux/freezer.h> 22#include <linux/mutex.h> 23#include <linux/writeback.h> 24#include <linux/completion.h> 25#include <linux/highmem.h> 26#include <linux/splice.h> 27#include <linux/sysfs.h> 28#include <linux/miscdevice.h> 29#include <linux/falloc.h> 30#include <linux/uio.h> 31#include <linux/ioprio.h> 32#include <linux/blk-cgroup.h> 33#include <linux/sched/mm.h> 34#include <linux/statfs.h> 35#include <linux/uaccess.h> 36#include <linux/blk-mq.h> 37#include <linux/spinlock.h> 38#include <uapi/linux/loop.h> 39 40/* Possible states of device */ 41enum { 42 Lo_unbound, 43 Lo_bound, 44 Lo_rundown, 45 Lo_deleting, 46}; 47 48struct loop_func_table; 49 50struct loop_device { 51 int lo_number; 52 loff_t lo_offset; 53 loff_t lo_sizelimit; 54 int lo_flags; 55 char lo_file_name[LO_NAME_SIZE]; 56 57 struct file * lo_backing_file; 58 struct block_device *lo_device; 59 60 gfp_t old_gfp_mask; 61 62 spinlock_t lo_lock; 63 int lo_state; 64 spinlock_t lo_work_lock; 65 struct workqueue_struct *workqueue; 66 struct work_struct rootcg_work; 67 struct list_head rootcg_cmd_list; 68 struct list_head idle_worker_list; 69 struct rb_root worker_tree; 70 struct timer_list timer; 71 bool use_dio; 72 bool sysfs_inited; 73 74 struct request_queue *lo_queue; 75 struct blk_mq_tag_set tag_set; 76 struct gendisk *lo_disk; 77 struct mutex lo_mutex; 78 bool idr_visible; 79}; 80 81struct loop_cmd { 82 struct list_head list_entry; 83 bool use_aio; /* use AIO interface to handle I/O */ 84 atomic_t ref; /* only for aio */ 85 long ret; 86 struct kiocb iocb; 87 struct bio_vec *bvec; 88 struct cgroup_subsys_state *blkcg_css; 89 struct cgroup_subsys_state *memcg_css; 90}; 91 92#define LOOP_IDLE_WORKER_TIMEOUT (60 * HZ) 93#define LOOP_DEFAULT_HW_Q_DEPTH 128 94 95static DEFINE_IDR(loop_index_idr); 96static DEFINE_MUTEX(loop_ctl_mutex); 97static DEFINE_MUTEX(loop_validate_mutex); 98 99/** 100 * loop_global_lock_killable() - take locks for safe loop_validate_file() test 101 * 102 * @lo: struct loop_device 103 * @global: true if @lo is about to bind another "struct loop_device", false otherwise 104 * 105 * Returns 0 on success, -EINTR otherwise. 106 * 107 * Since loop_validate_file() traverses on other "struct loop_device" if 108 * is_loop_device() is true, we need a global lock for serializing concurrent 109 * loop_configure()/loop_change_fd()/__loop_clr_fd() calls. 110 */ 111static int loop_global_lock_killable(struct loop_device *lo, bool global) 112{ 113 int err; 114 115 if (global) { 116 err = mutex_lock_killable(&loop_validate_mutex); 117 if (err) 118 return err; 119 } 120 err = mutex_lock_killable(&lo->lo_mutex); 121 if (err && global) 122 mutex_unlock(&loop_validate_mutex); 123 return err; 124} 125 126/** 127 * loop_global_unlock() - release locks taken by loop_global_lock_killable() 128 * 129 * @lo: struct loop_device 130 * @global: true if @lo was about to bind another "struct loop_device", false otherwise 131 */ 132static void loop_global_unlock(struct loop_device *lo, bool global) 133{ 134 mutex_unlock(&lo->lo_mutex); 135 if (global) 136 mutex_unlock(&loop_validate_mutex); 137} 138 139static int max_part; 140static int part_shift; 141 142static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file) 143{ 144 loff_t loopsize; 145 146 /* Compute loopsize in bytes */ 147 loopsize = i_size_read(file->f_mapping->host); 148 if (offset > 0) 149 loopsize -= offset; 150 /* offset is beyond i_size, weird but possible */ 151 if (loopsize < 0) 152 return 0; 153 154 if (sizelimit > 0 && sizelimit < loopsize) 155 loopsize = sizelimit; 156 /* 157 * Unfortunately, if we want to do I/O on the device, 158 * the number of 512-byte sectors has to fit into a sector_t. 159 */ 160 return loopsize >> 9; 161} 162 163static loff_t get_loop_size(struct loop_device *lo, struct file *file) 164{ 165 return get_size(lo->lo_offset, lo->lo_sizelimit, file); 166} 167 168/* 169 * We support direct I/O only if lo_offset is aligned with the logical I/O size 170 * of backing device, and the logical block size of loop is bigger than that of 171 * the backing device. 172 */ 173static bool lo_bdev_can_use_dio(struct loop_device *lo, 174 struct block_device *backing_bdev) 175{ 176 unsigned short sb_bsize = bdev_logical_block_size(backing_bdev); 177 178 if (queue_logical_block_size(lo->lo_queue) < sb_bsize) 179 return false; 180 if (lo->lo_offset & (sb_bsize - 1)) 181 return false; 182 return true; 183} 184 185static void __loop_update_dio(struct loop_device *lo, bool dio) 186{ 187 struct file *file = lo->lo_backing_file; 188 struct inode *inode = file->f_mapping->host; 189 struct block_device *backing_bdev = NULL; 190 bool use_dio; 191 192 if (S_ISBLK(inode->i_mode)) 193 backing_bdev = I_BDEV(inode); 194 else if (inode->i_sb->s_bdev) 195 backing_bdev = inode->i_sb->s_bdev; 196 197 use_dio = dio && (file->f_mode & FMODE_CAN_ODIRECT) && 198 (!backing_bdev || lo_bdev_can_use_dio(lo, backing_bdev)); 199 200 if (lo->use_dio == use_dio) 201 return; 202 203 /* flush dirty pages before changing direct IO */ 204 vfs_fsync(file, 0); 205 206 /* 207 * The flag of LO_FLAGS_DIRECT_IO is handled similarly with 208 * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup 209 * will get updated by ioctl(LOOP_GET_STATUS) 210 */ 211 if (lo->lo_state == Lo_bound) 212 blk_mq_freeze_queue(lo->lo_queue); 213 lo->use_dio = use_dio; 214 if (use_dio) { 215 blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, lo->lo_queue); 216 lo->lo_flags |= LO_FLAGS_DIRECT_IO; 217 } else { 218 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 219 lo->lo_flags &= ~LO_FLAGS_DIRECT_IO; 220 } 221 if (lo->lo_state == Lo_bound) 222 blk_mq_unfreeze_queue(lo->lo_queue); 223} 224 225/** 226 * loop_set_size() - sets device size and notifies userspace 227 * @lo: struct loop_device to set the size for 228 * @size: new size of the loop device 229 * 230 * Callers must validate that the size passed into this function fits into 231 * a sector_t, eg using loop_validate_size() 232 */ 233static void loop_set_size(struct loop_device *lo, loff_t size) 234{ 235 if (!set_capacity_and_notify(lo->lo_disk, size)) 236 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE); 237} 238 239static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos) 240{ 241 struct iov_iter i; 242 ssize_t bw; 243 244 iov_iter_bvec(&i, ITER_SOURCE, bvec, 1, bvec->bv_len); 245 246 file_start_write(file); 247 bw = vfs_iter_write(file, &i, ppos, 0); 248 file_end_write(file); 249 250 if (likely(bw == bvec->bv_len)) 251 return 0; 252 253 printk_ratelimited(KERN_ERR 254 "loop: Write error at byte offset %llu, length %i.\n", 255 (unsigned long long)*ppos, bvec->bv_len); 256 if (bw >= 0) 257 bw = -EIO; 258 return bw; 259} 260 261static int lo_write_simple(struct loop_device *lo, struct request *rq, 262 loff_t pos) 263{ 264 struct bio_vec bvec; 265 struct req_iterator iter; 266 int ret = 0; 267 268 rq_for_each_segment(bvec, rq, iter) { 269 ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos); 270 if (ret < 0) 271 break; 272 cond_resched(); 273 } 274 275 return ret; 276} 277 278static int lo_read_simple(struct loop_device *lo, struct request *rq, 279 loff_t pos) 280{ 281 struct bio_vec bvec; 282 struct req_iterator iter; 283 struct iov_iter i; 284 ssize_t len; 285 286 rq_for_each_segment(bvec, rq, iter) { 287 iov_iter_bvec(&i, ITER_DEST, &bvec, 1, bvec.bv_len); 288 len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); 289 if (len < 0) 290 return len; 291 292 flush_dcache_page(bvec.bv_page); 293 294 if (len != bvec.bv_len) { 295 struct bio *bio; 296 297 __rq_for_each_bio(bio, rq) 298 zero_fill_bio(bio); 299 break; 300 } 301 cond_resched(); 302 } 303 304 return 0; 305} 306 307static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos, 308 int mode) 309{ 310 /* 311 * We use fallocate to manipulate the space mappings used by the image 312 * a.k.a. discard/zerorange. 313 */ 314 struct file *file = lo->lo_backing_file; 315 int ret; 316 317 mode |= FALLOC_FL_KEEP_SIZE; 318 319 if (!bdev_max_discard_sectors(lo->lo_device)) 320 return -EOPNOTSUPP; 321 322 ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq)); 323 if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP)) 324 return -EIO; 325 return ret; 326} 327 328static int lo_req_flush(struct loop_device *lo, struct request *rq) 329{ 330 int ret = vfs_fsync(lo->lo_backing_file, 0); 331 if (unlikely(ret && ret != -EINVAL)) 332 ret = -EIO; 333 334 return ret; 335} 336 337static void lo_complete_rq(struct request *rq) 338{ 339 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 340 blk_status_t ret = BLK_STS_OK; 341 342 if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) || 343 req_op(rq) != REQ_OP_READ) { 344 if (cmd->ret < 0) 345 ret = errno_to_blk_status(cmd->ret); 346 goto end_io; 347 } 348 349 /* 350 * Short READ - if we got some data, advance our request and 351 * retry it. If we got no data, end the rest with EIO. 352 */ 353 if (cmd->ret) { 354 blk_update_request(rq, BLK_STS_OK, cmd->ret); 355 cmd->ret = 0; 356 blk_mq_requeue_request(rq, true); 357 } else { 358 if (cmd->use_aio) { 359 struct bio *bio = rq->bio; 360 361 while (bio) { 362 zero_fill_bio(bio); 363 bio = bio->bi_next; 364 } 365 } 366 ret = BLK_STS_IOERR; 367end_io: 368 blk_mq_end_request(rq, ret); 369 } 370} 371 372static void lo_rw_aio_do_completion(struct loop_cmd *cmd) 373{ 374 struct request *rq = blk_mq_rq_from_pdu(cmd); 375 376 if (!atomic_dec_and_test(&cmd->ref)) 377 return; 378 kfree(cmd->bvec); 379 cmd->bvec = NULL; 380 if (likely(!blk_should_fake_timeout(rq->q))) 381 blk_mq_complete_request(rq); 382} 383 384static void lo_rw_aio_complete(struct kiocb *iocb, long ret) 385{ 386 struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb); 387 388 cmd->ret = ret; 389 lo_rw_aio_do_completion(cmd); 390} 391 392static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, 393 loff_t pos, int rw) 394{ 395 struct iov_iter iter; 396 struct req_iterator rq_iter; 397 struct bio_vec *bvec; 398 struct request *rq = blk_mq_rq_from_pdu(cmd); 399 struct bio *bio = rq->bio; 400 struct file *file = lo->lo_backing_file; 401 struct bio_vec tmp; 402 unsigned int offset; 403 int nr_bvec = 0; 404 int ret; 405 406 rq_for_each_bvec(tmp, rq, rq_iter) 407 nr_bvec++; 408 409 if (rq->bio != rq->biotail) { 410 411 bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec), 412 GFP_NOIO); 413 if (!bvec) 414 return -EIO; 415 cmd->bvec = bvec; 416 417 /* 418 * The bios of the request may be started from the middle of 419 * the 'bvec' because of bio splitting, so we can't directly 420 * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec 421 * API will take care of all details for us. 422 */ 423 rq_for_each_bvec(tmp, rq, rq_iter) { 424 *bvec = tmp; 425 bvec++; 426 } 427 bvec = cmd->bvec; 428 offset = 0; 429 } else { 430 /* 431 * Same here, this bio may be started from the middle of the 432 * 'bvec' because of bio splitting, so offset from the bvec 433 * must be passed to iov iterator 434 */ 435 offset = bio->bi_iter.bi_bvec_done; 436 bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 437 } 438 atomic_set(&cmd->ref, 2); 439 440 iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq)); 441 iter.iov_offset = offset; 442 443 cmd->iocb.ki_pos = pos; 444 cmd->iocb.ki_filp = file; 445 cmd->iocb.ki_complete = lo_rw_aio_complete; 446 cmd->iocb.ki_flags = IOCB_DIRECT; 447 cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0); 448 449 if (rw == ITER_SOURCE) 450 ret = call_write_iter(file, &cmd->iocb, &iter); 451 else 452 ret = call_read_iter(file, &cmd->iocb, &iter); 453 454 lo_rw_aio_do_completion(cmd); 455 456 if (ret != -EIOCBQUEUED) 457 lo_rw_aio_complete(&cmd->iocb, ret); 458 return 0; 459} 460 461static int do_req_filebacked(struct loop_device *lo, struct request *rq) 462{ 463 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 464 loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset; 465 466 /* 467 * lo_write_simple and lo_read_simple should have been covered 468 * by io submit style function like lo_rw_aio(), one blocker 469 * is that lo_read_simple() need to call flush_dcache_page after 470 * the page is written from kernel, and it isn't easy to handle 471 * this in io submit style function which submits all segments 472 * of the req at one time. And direct read IO doesn't need to 473 * run flush_dcache_page(). 474 */ 475 switch (req_op(rq)) { 476 case REQ_OP_FLUSH: 477 return lo_req_flush(lo, rq); 478 case REQ_OP_WRITE_ZEROES: 479 /* 480 * If the caller doesn't want deallocation, call zeroout to 481 * write zeroes the range. Otherwise, punch them out. 482 */ 483 return lo_fallocate(lo, rq, pos, 484 (rq->cmd_flags & REQ_NOUNMAP) ? 485 FALLOC_FL_ZERO_RANGE : 486 FALLOC_FL_PUNCH_HOLE); 487 case REQ_OP_DISCARD: 488 return lo_fallocate(lo, rq, pos, FALLOC_FL_PUNCH_HOLE); 489 case REQ_OP_WRITE: 490 if (cmd->use_aio) 491 return lo_rw_aio(lo, cmd, pos, ITER_SOURCE); 492 else 493 return lo_write_simple(lo, rq, pos); 494 case REQ_OP_READ: 495 if (cmd->use_aio) 496 return lo_rw_aio(lo, cmd, pos, ITER_DEST); 497 else 498 return lo_read_simple(lo, rq, pos); 499 default: 500 WARN_ON_ONCE(1); 501 return -EIO; 502 } 503} 504 505static inline void loop_update_dio(struct loop_device *lo) 506{ 507 __loop_update_dio(lo, (lo->lo_backing_file->f_flags & O_DIRECT) | 508 lo->use_dio); 509} 510 511static void loop_reread_partitions(struct loop_device *lo) 512{ 513 int rc; 514 515 mutex_lock(&lo->lo_disk->open_mutex); 516 rc = bdev_disk_changed(lo->lo_disk, false); 517 mutex_unlock(&lo->lo_disk->open_mutex); 518 if (rc) 519 pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n", 520 __func__, lo->lo_number, lo->lo_file_name, rc); 521} 522 523static inline int is_loop_device(struct file *file) 524{ 525 struct inode *i = file->f_mapping->host; 526 527 return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR; 528} 529 530static int loop_validate_file(struct file *file, struct block_device *bdev) 531{ 532 struct inode *inode = file->f_mapping->host; 533 struct file *f = file; 534 535 /* Avoid recursion */ 536 while (is_loop_device(f)) { 537 struct loop_device *l; 538 539 lockdep_assert_held(&loop_validate_mutex); 540 if (f->f_mapping->host->i_rdev == bdev->bd_dev) 541 return -EBADF; 542 543 l = I_BDEV(f->f_mapping->host)->bd_disk->private_data; 544 if (l->lo_state != Lo_bound) 545 return -EINVAL; 546 /* Order wrt setting lo->lo_backing_file in loop_configure(). */ 547 rmb(); 548 f = l->lo_backing_file; 549 } 550 if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode)) 551 return -EINVAL; 552 return 0; 553} 554 555/* 556 * loop_change_fd switched the backing store of a loopback device to 557 * a new file. This is useful for operating system installers to free up 558 * the original file and in High Availability environments to switch to 559 * an alternative location for the content in case of server meltdown. 560 * This can only work if the loop device is used read-only, and if the 561 * new backing store is the same size and type as the old backing store. 562 */ 563static int loop_change_fd(struct loop_device *lo, struct block_device *bdev, 564 unsigned int arg) 565{ 566 struct file *file = fget(arg); 567 struct file *old_file; 568 int error; 569 bool partscan; 570 bool is_loop; 571 572 if (!file) 573 return -EBADF; 574 575 /* suppress uevents while reconfiguring the device */ 576 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1); 577 578 is_loop = is_loop_device(file); 579 error = loop_global_lock_killable(lo, is_loop); 580 if (error) 581 goto out_putf; 582 error = -ENXIO; 583 if (lo->lo_state != Lo_bound) 584 goto out_err; 585 586 /* the loop device has to be read-only */ 587 error = -EINVAL; 588 if (!(lo->lo_flags & LO_FLAGS_READ_ONLY)) 589 goto out_err; 590 591 error = loop_validate_file(file, bdev); 592 if (error) 593 goto out_err; 594 595 old_file = lo->lo_backing_file; 596 597 error = -EINVAL; 598 599 /* size of the new backing store needs to be the same */ 600 if (get_loop_size(lo, file) != get_loop_size(lo, old_file)) 601 goto out_err; 602 603 /* and ... switch */ 604 disk_force_media_change(lo->lo_disk); 605 blk_mq_freeze_queue(lo->lo_queue); 606 mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask); 607 lo->lo_backing_file = file; 608 lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping); 609 mapping_set_gfp_mask(file->f_mapping, 610 lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 611 loop_update_dio(lo); 612 blk_mq_unfreeze_queue(lo->lo_queue); 613 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; 614 loop_global_unlock(lo, is_loop); 615 616 /* 617 * Flush loop_validate_file() before fput(), for l->lo_backing_file 618 * might be pointing at old_file which might be the last reference. 619 */ 620 if (!is_loop) { 621 mutex_lock(&loop_validate_mutex); 622 mutex_unlock(&loop_validate_mutex); 623 } 624 /* 625 * We must drop file reference outside of lo_mutex as dropping 626 * the file ref can take open_mutex which creates circular locking 627 * dependency. 628 */ 629 fput(old_file); 630 if (partscan) 631 loop_reread_partitions(lo); 632 633 error = 0; 634done: 635 /* enable and uncork uevent now that we are done */ 636 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0); 637 return error; 638 639out_err: 640 loop_global_unlock(lo, is_loop); 641out_putf: 642 fput(file); 643 goto done; 644} 645 646/* loop sysfs attributes */ 647 648static ssize_t loop_attr_show(struct device *dev, char *page, 649 ssize_t (*callback)(struct loop_device *, char *)) 650{ 651 struct gendisk *disk = dev_to_disk(dev); 652 struct loop_device *lo = disk->private_data; 653 654 return callback(lo, page); 655} 656 657#define LOOP_ATTR_RO(_name) \ 658static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \ 659static ssize_t loop_attr_do_show_##_name(struct device *d, \ 660 struct device_attribute *attr, char *b) \ 661{ \ 662 return loop_attr_show(d, b, loop_attr_##_name##_show); \ 663} \ 664static struct device_attribute loop_attr_##_name = \ 665 __ATTR(_name, 0444, loop_attr_do_show_##_name, NULL); 666 667static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf) 668{ 669 ssize_t ret; 670 char *p = NULL; 671 672 spin_lock_irq(&lo->lo_lock); 673 if (lo->lo_backing_file) 674 p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1); 675 spin_unlock_irq(&lo->lo_lock); 676 677 if (IS_ERR_OR_NULL(p)) 678 ret = PTR_ERR(p); 679 else { 680 ret = strlen(p); 681 memmove(buf, p, ret); 682 buf[ret++] = '\n'; 683 buf[ret] = 0; 684 } 685 686 return ret; 687} 688 689static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf) 690{ 691 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_offset); 692} 693 694static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf) 695{ 696 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit); 697} 698 699static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf) 700{ 701 int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR); 702 703 return sysfs_emit(buf, "%s\n", autoclear ? "1" : "0"); 704} 705 706static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf) 707{ 708 int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN); 709 710 return sysfs_emit(buf, "%s\n", partscan ? "1" : "0"); 711} 712 713static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf) 714{ 715 int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO); 716 717 return sysfs_emit(buf, "%s\n", dio ? "1" : "0"); 718} 719 720LOOP_ATTR_RO(backing_file); 721LOOP_ATTR_RO(offset); 722LOOP_ATTR_RO(sizelimit); 723LOOP_ATTR_RO(autoclear); 724LOOP_ATTR_RO(partscan); 725LOOP_ATTR_RO(dio); 726 727static struct attribute *loop_attrs[] = { 728 &loop_attr_backing_file.attr, 729 &loop_attr_offset.attr, 730 &loop_attr_sizelimit.attr, 731 &loop_attr_autoclear.attr, 732 &loop_attr_partscan.attr, 733 &loop_attr_dio.attr, 734 NULL, 735}; 736 737static struct attribute_group loop_attribute_group = { 738 .name = "loop", 739 .attrs= loop_attrs, 740}; 741 742static void loop_sysfs_init(struct loop_device *lo) 743{ 744 lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj, 745 &loop_attribute_group); 746} 747 748static void loop_sysfs_exit(struct loop_device *lo) 749{ 750 if (lo->sysfs_inited) 751 sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj, 752 &loop_attribute_group); 753} 754 755static void loop_config_discard(struct loop_device *lo) 756{ 757 struct file *file = lo->lo_backing_file; 758 struct inode *inode = file->f_mapping->host; 759 struct request_queue *q = lo->lo_queue; 760 u32 granularity, max_discard_sectors; 761 762 /* 763 * If the backing device is a block device, mirror its zeroing 764 * capability. Set the discard sectors to the block device's zeroing 765 * capabilities because loop discards result in blkdev_issue_zeroout(), 766 * not blkdev_issue_discard(). This maintains consistent behavior with 767 * file-backed loop devices: discarded regions read back as zero. 768 */ 769 if (S_ISBLK(inode->i_mode)) { 770 struct request_queue *backingq = bdev_get_queue(I_BDEV(inode)); 771 772 max_discard_sectors = backingq->limits.max_write_zeroes_sectors; 773 granularity = bdev_discard_granularity(I_BDEV(inode)) ?: 774 queue_physical_block_size(backingq); 775 776 /* 777 * We use punch hole to reclaim the free space used by the 778 * image a.k.a. discard. 779 */ 780 } else if (!file->f_op->fallocate) { 781 max_discard_sectors = 0; 782 granularity = 0; 783 784 } else { 785 struct kstatfs sbuf; 786 787 max_discard_sectors = UINT_MAX >> 9; 788 if (!vfs_statfs(&file->f_path, &sbuf)) 789 granularity = sbuf.f_bsize; 790 else 791 max_discard_sectors = 0; 792 } 793 794 if (max_discard_sectors) { 795 q->limits.discard_granularity = granularity; 796 blk_queue_max_discard_sectors(q, max_discard_sectors); 797 blk_queue_max_write_zeroes_sectors(q, max_discard_sectors); 798 } else { 799 q->limits.discard_granularity = 0; 800 blk_queue_max_discard_sectors(q, 0); 801 blk_queue_max_write_zeroes_sectors(q, 0); 802 } 803} 804 805struct loop_worker { 806 struct rb_node rb_node; 807 struct work_struct work; 808 struct list_head cmd_list; 809 struct list_head idle_list; 810 struct loop_device *lo; 811 struct cgroup_subsys_state *blkcg_css; 812 unsigned long last_ran_at; 813}; 814 815static void loop_workfn(struct work_struct *work); 816 817#ifdef CONFIG_BLK_CGROUP 818static inline int queue_on_root_worker(struct cgroup_subsys_state *css) 819{ 820 return !css || css == blkcg_root_css; 821} 822#else 823static inline int queue_on_root_worker(struct cgroup_subsys_state *css) 824{ 825 return !css; 826} 827#endif 828 829static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd) 830{ 831 struct rb_node **node, *parent = NULL; 832 struct loop_worker *cur_worker, *worker = NULL; 833 struct work_struct *work; 834 struct list_head *cmd_list; 835 836 spin_lock_irq(&lo->lo_work_lock); 837 838 if (queue_on_root_worker(cmd->blkcg_css)) 839 goto queue_work; 840 841 node = &lo->worker_tree.rb_node; 842 843 while (*node) { 844 parent = *node; 845 cur_worker = container_of(*node, struct loop_worker, rb_node); 846 if (cur_worker->blkcg_css == cmd->blkcg_css) { 847 worker = cur_worker; 848 break; 849 } else if ((long)cur_worker->blkcg_css < (long)cmd->blkcg_css) { 850 node = &(*node)->rb_left; 851 } else { 852 node = &(*node)->rb_right; 853 } 854 } 855 if (worker) 856 goto queue_work; 857 858 worker = kzalloc(sizeof(struct loop_worker), GFP_NOWAIT | __GFP_NOWARN); 859 /* 860 * In the event we cannot allocate a worker, just queue on the 861 * rootcg worker and issue the I/O as the rootcg 862 */ 863 if (!worker) { 864 cmd->blkcg_css = NULL; 865 if (cmd->memcg_css) 866 css_put(cmd->memcg_css); 867 cmd->memcg_css = NULL; 868 goto queue_work; 869 } 870 871 worker->blkcg_css = cmd->blkcg_css; 872 css_get(worker->blkcg_css); 873 INIT_WORK(&worker->work, loop_workfn); 874 INIT_LIST_HEAD(&worker->cmd_list); 875 INIT_LIST_HEAD(&worker->idle_list); 876 worker->lo = lo; 877 rb_link_node(&worker->rb_node, parent, node); 878 rb_insert_color(&worker->rb_node, &lo->worker_tree); 879queue_work: 880 if (worker) { 881 /* 882 * We need to remove from the idle list here while 883 * holding the lock so that the idle timer doesn't 884 * free the worker 885 */ 886 if (!list_empty(&worker->idle_list)) 887 list_del_init(&worker->idle_list); 888 work = &worker->work; 889 cmd_list = &worker->cmd_list; 890 } else { 891 work = &lo->rootcg_work; 892 cmd_list = &lo->rootcg_cmd_list; 893 } 894 list_add_tail(&cmd->list_entry, cmd_list); 895 queue_work(lo->workqueue, work); 896 spin_unlock_irq(&lo->lo_work_lock); 897} 898 899static void loop_set_timer(struct loop_device *lo) 900{ 901 timer_reduce(&lo->timer, jiffies + LOOP_IDLE_WORKER_TIMEOUT); 902} 903 904static void loop_free_idle_workers(struct loop_device *lo, bool delete_all) 905{ 906 struct loop_worker *pos, *worker; 907 908 spin_lock_irq(&lo->lo_work_lock); 909 list_for_each_entry_safe(worker, pos, &lo->idle_worker_list, 910 idle_list) { 911 if (!delete_all && 912 time_is_after_jiffies(worker->last_ran_at + 913 LOOP_IDLE_WORKER_TIMEOUT)) 914 break; 915 list_del(&worker->idle_list); 916 rb_erase(&worker->rb_node, &lo->worker_tree); 917 css_put(worker->blkcg_css); 918 kfree(worker); 919 } 920 if (!list_empty(&lo->idle_worker_list)) 921 loop_set_timer(lo); 922 spin_unlock_irq(&lo->lo_work_lock); 923} 924 925static void loop_free_idle_workers_timer(struct timer_list *timer) 926{ 927 struct loop_device *lo = container_of(timer, struct loop_device, timer); 928 929 return loop_free_idle_workers(lo, false); 930} 931 932static void loop_update_rotational(struct loop_device *lo) 933{ 934 struct file *file = lo->lo_backing_file; 935 struct inode *file_inode = file->f_mapping->host; 936 struct block_device *file_bdev = file_inode->i_sb->s_bdev; 937 struct request_queue *q = lo->lo_queue; 938 bool nonrot = true; 939 940 /* not all filesystems (e.g. tmpfs) have a sb->s_bdev */ 941 if (file_bdev) 942 nonrot = bdev_nonrot(file_bdev); 943 944 if (nonrot) 945 blk_queue_flag_set(QUEUE_FLAG_NONROT, q); 946 else 947 blk_queue_flag_clear(QUEUE_FLAG_NONROT, q); 948} 949 950/** 951 * loop_set_status_from_info - configure device from loop_info 952 * @lo: struct loop_device to configure 953 * @info: struct loop_info64 to configure the device with 954 * 955 * Configures the loop device parameters according to the passed 956 * in loop_info64 configuration. 957 */ 958static int 959loop_set_status_from_info(struct loop_device *lo, 960 const struct loop_info64 *info) 961{ 962 if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) 963 return -EINVAL; 964 965 switch (info->lo_encrypt_type) { 966 case LO_CRYPT_NONE: 967 break; 968 case LO_CRYPT_XOR: 969 pr_warn("support for the xor transformation has been removed.\n"); 970 return -EINVAL; 971 case LO_CRYPT_CRYPTOAPI: 972 pr_warn("support for cryptoloop has been removed. Use dm-crypt instead.\n"); 973 return -EINVAL; 974 default: 975 return -EINVAL; 976 } 977 978 /* Avoid assigning overflow values */ 979 if (info->lo_offset > LLONG_MAX || info->lo_sizelimit > LLONG_MAX) 980 return -EOVERFLOW; 981 982 lo->lo_offset = info->lo_offset; 983 lo->lo_sizelimit = info->lo_sizelimit; 984 985 memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); 986 lo->lo_file_name[LO_NAME_SIZE-1] = 0; 987 lo->lo_flags = info->lo_flags; 988 return 0; 989} 990 991static int loop_configure(struct loop_device *lo, blk_mode_t mode, 992 struct block_device *bdev, 993 const struct loop_config *config) 994{ 995 struct file *file = fget(config->fd); 996 struct inode *inode; 997 struct address_space *mapping; 998 int error; 999 loff_t size; 1000 bool partscan; 1001 unsigned short bsize; 1002 bool is_loop; 1003 1004 if (!file) 1005 return -EBADF; 1006 is_loop = is_loop_device(file); 1007 1008 /* This is safe, since we have a reference from open(). */ 1009 __module_get(THIS_MODULE); 1010 1011 /* 1012 * If we don't hold exclusive handle for the device, upgrade to it 1013 * here to avoid changing device under exclusive owner. 1014 */ 1015 if (!(mode & BLK_OPEN_EXCL)) { 1016 error = bd_prepare_to_claim(bdev, loop_configure, NULL); 1017 if (error) 1018 goto out_putf; 1019 } 1020 1021 error = loop_global_lock_killable(lo, is_loop); 1022 if (error) 1023 goto out_bdev; 1024 1025 error = -EBUSY; 1026 if (lo->lo_state != Lo_unbound) 1027 goto out_unlock; 1028 1029 error = loop_validate_file(file, bdev); 1030 if (error) 1031 goto out_unlock; 1032 1033 mapping = file->f_mapping; 1034 inode = mapping->host; 1035 1036 if ((config->info.lo_flags & ~LOOP_CONFIGURE_SETTABLE_FLAGS) != 0) { 1037 error = -EINVAL; 1038 goto out_unlock; 1039 } 1040 1041 if (config->block_size) { 1042 error = blk_validate_block_size(config->block_size); 1043 if (error) 1044 goto out_unlock; 1045 } 1046 1047 error = loop_set_status_from_info(lo, &config->info); 1048 if (error) 1049 goto out_unlock; 1050 1051 if (!(file->f_mode & FMODE_WRITE) || !(mode & BLK_OPEN_WRITE) || 1052 !file->f_op->write_iter) 1053 lo->lo_flags |= LO_FLAGS_READ_ONLY; 1054 1055 if (!lo->workqueue) { 1056 lo->workqueue = alloc_workqueue("loop%d", 1057 WQ_UNBOUND | WQ_FREEZABLE, 1058 0, lo->lo_number); 1059 if (!lo->workqueue) { 1060 error = -ENOMEM; 1061 goto out_unlock; 1062 } 1063 } 1064 1065 /* suppress uevents while reconfiguring the device */ 1066 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1); 1067 1068 disk_force_media_change(lo->lo_disk); 1069 set_disk_ro(lo->lo_disk, (lo->lo_flags & LO_FLAGS_READ_ONLY) != 0); 1070 1071 lo->use_dio = lo->lo_flags & LO_FLAGS_DIRECT_IO; 1072 lo->lo_device = bdev; 1073 lo->lo_backing_file = file; 1074 lo->old_gfp_mask = mapping_gfp_mask(mapping); 1075 mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 1076 1077 if (!(lo->lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) 1078 blk_queue_write_cache(lo->lo_queue, true, false); 1079 1080 if (config->block_size) 1081 bsize = config->block_size; 1082 else if ((lo->lo_backing_file->f_flags & O_DIRECT) && inode->i_sb->s_bdev) 1083 /* In case of direct I/O, match underlying block size */ 1084 bsize = bdev_logical_block_size(inode->i_sb->s_bdev); 1085 else 1086 bsize = 512; 1087 1088 blk_queue_logical_block_size(lo->lo_queue, bsize); 1089 blk_queue_physical_block_size(lo->lo_queue, bsize); 1090 blk_queue_io_min(lo->lo_queue, bsize); 1091 1092 loop_config_discard(lo); 1093 loop_update_rotational(lo); 1094 loop_update_dio(lo); 1095 loop_sysfs_init(lo); 1096 1097 size = get_loop_size(lo, file); 1098 loop_set_size(lo, size); 1099 1100 /* Order wrt reading lo_state in loop_validate_file(). */ 1101 wmb(); 1102 1103 lo->lo_state = Lo_bound; 1104 if (part_shift) 1105 lo->lo_flags |= LO_FLAGS_PARTSCAN; 1106 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; 1107 if (partscan) 1108 clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state); 1109 1110 /* enable and uncork uevent now that we are done */ 1111 dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0); 1112 1113 loop_global_unlock(lo, is_loop); 1114 if (partscan) 1115 loop_reread_partitions(lo); 1116 1117 if (!(mode & BLK_OPEN_EXCL)) 1118 bd_abort_claiming(bdev, loop_configure); 1119 1120 return 0; 1121 1122out_unlock: 1123 loop_global_unlock(lo, is_loop); 1124out_bdev: 1125 if (!(mode & BLK_OPEN_EXCL)) 1126 bd_abort_claiming(bdev, loop_configure); 1127out_putf: 1128 fput(file); 1129 /* This is safe: open() is still holding a reference. */ 1130 module_put(THIS_MODULE); 1131 return error; 1132} 1133 1134static void __loop_clr_fd(struct loop_device *lo, bool release) 1135{ 1136 struct file *filp; 1137 gfp_t gfp = lo->old_gfp_mask; 1138 1139 if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags)) 1140 blk_queue_write_cache(lo->lo_queue, false, false); 1141 1142 /* 1143 * Freeze the request queue when unbinding on a live file descriptor and 1144 * thus an open device. When called from ->release we are guaranteed 1145 * that there is no I/O in progress already. 1146 */ 1147 if (!release) 1148 blk_mq_freeze_queue(lo->lo_queue); 1149 1150 spin_lock_irq(&lo->lo_lock); 1151 filp = lo->lo_backing_file; 1152 lo->lo_backing_file = NULL; 1153 spin_unlock_irq(&lo->lo_lock); 1154 1155 lo->lo_device = NULL; 1156 lo->lo_offset = 0; 1157 lo->lo_sizelimit = 0; 1158 memset(lo->lo_file_name, 0, LO_NAME_SIZE); 1159 blk_queue_logical_block_size(lo->lo_queue, 512); 1160 blk_queue_physical_block_size(lo->lo_queue, 512); 1161 blk_queue_io_min(lo->lo_queue, 512); 1162 invalidate_disk(lo->lo_disk); 1163 loop_sysfs_exit(lo); 1164 /* let user-space know about this change */ 1165 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE); 1166 mapping_set_gfp_mask(filp->f_mapping, gfp); 1167 /* This is safe: open() is still holding a reference. */ 1168 module_put(THIS_MODULE); 1169 if (!release) 1170 blk_mq_unfreeze_queue(lo->lo_queue); 1171 1172 disk_force_media_change(lo->lo_disk); 1173 1174 if (lo->lo_flags & LO_FLAGS_PARTSCAN) { 1175 int err; 1176 1177 /* 1178 * open_mutex has been held already in release path, so don't 1179 * acquire it if this function is called in such case. 1180 * 1181 * If the reread partition isn't from release path, lo_refcnt 1182 * must be at least one and it can only become zero when the 1183 * current holder is released. 1184 */ 1185 if (!release) 1186 mutex_lock(&lo->lo_disk->open_mutex); 1187 err = bdev_disk_changed(lo->lo_disk, false); 1188 if (!release) 1189 mutex_unlock(&lo->lo_disk->open_mutex); 1190 if (err) 1191 pr_warn("%s: partition scan of loop%d failed (rc=%d)\n", 1192 __func__, lo->lo_number, err); 1193 /* Device is gone, no point in returning error */ 1194 } 1195 1196 /* 1197 * lo->lo_state is set to Lo_unbound here after above partscan has 1198 * finished. There cannot be anybody else entering __loop_clr_fd() as 1199 * Lo_rundown state protects us from all the other places trying to 1200 * change the 'lo' device. 1201 */ 1202 lo->lo_flags = 0; 1203 if (!part_shift) 1204 set_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state); 1205 mutex_lock(&lo->lo_mutex); 1206 lo->lo_state = Lo_unbound; 1207 mutex_unlock(&lo->lo_mutex); 1208 1209 /* 1210 * Need not hold lo_mutex to fput backing file. Calling fput holding 1211 * lo_mutex triggers a circular lock dependency possibility warning as 1212 * fput can take open_mutex which is usually taken before lo_mutex. 1213 */ 1214 fput(filp); 1215} 1216 1217static int loop_clr_fd(struct loop_device *lo) 1218{ 1219 int err; 1220 1221 /* 1222 * Since lo_ioctl() is called without locks held, it is possible that 1223 * loop_configure()/loop_change_fd() and loop_clr_fd() run in parallel. 1224 * 1225 * Therefore, use global lock when setting Lo_rundown state in order to 1226 * make sure that loop_validate_file() will fail if the "struct file" 1227 * which loop_configure()/loop_change_fd() found via fget() was this 1228 * loop device. 1229 */ 1230 err = loop_global_lock_killable(lo, true); 1231 if (err) 1232 return err; 1233 if (lo->lo_state != Lo_bound) { 1234 loop_global_unlock(lo, true); 1235 return -ENXIO; 1236 } 1237 /* 1238 * If we've explicitly asked to tear down the loop device, 1239 * and it has an elevated reference count, set it for auto-teardown when 1240 * the last reference goes away. This stops $!~#$@ udev from 1241 * preventing teardown because it decided that it needs to run blkid on 1242 * the loopback device whenever they appear. xfstests is notorious for 1243 * failing tests because blkid via udev races with a losetup 1244 * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d 1245 * command to fail with EBUSY. 1246 */ 1247 if (disk_openers(lo->lo_disk) > 1) { 1248 lo->lo_flags |= LO_FLAGS_AUTOCLEAR; 1249 loop_global_unlock(lo, true); 1250 return 0; 1251 } 1252 lo->lo_state = Lo_rundown; 1253 loop_global_unlock(lo, true); 1254 1255 __loop_clr_fd(lo, false); 1256 return 0; 1257} 1258 1259static int 1260loop_set_status(struct loop_device *lo, const struct loop_info64 *info) 1261{ 1262 int err; 1263 int prev_lo_flags; 1264 bool partscan = false; 1265 bool size_changed = false; 1266 1267 err = mutex_lock_killable(&lo->lo_mutex); 1268 if (err) 1269 return err; 1270 if (lo->lo_state != Lo_bound) { 1271 err = -ENXIO; 1272 goto out_unlock; 1273 } 1274 1275 if (lo->lo_offset != info->lo_offset || 1276 lo->lo_sizelimit != info->lo_sizelimit) { 1277 size_changed = true; 1278 sync_blockdev(lo->lo_device); 1279 invalidate_bdev(lo->lo_device); 1280 } 1281 1282 /* I/O need to be drained during transfer transition */ 1283 blk_mq_freeze_queue(lo->lo_queue); 1284 1285 prev_lo_flags = lo->lo_flags; 1286 1287 err = loop_set_status_from_info(lo, info); 1288 if (err) 1289 goto out_unfreeze; 1290 1291 /* Mask out flags that can't be set using LOOP_SET_STATUS. */ 1292 lo->lo_flags &= LOOP_SET_STATUS_SETTABLE_FLAGS; 1293 /* For those flags, use the previous values instead */ 1294 lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_SETTABLE_FLAGS; 1295 /* For flags that can't be cleared, use previous values too */ 1296 lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_CLEARABLE_FLAGS; 1297 1298 if (size_changed) { 1299 loff_t new_size = get_size(lo->lo_offset, lo->lo_sizelimit, 1300 lo->lo_backing_file); 1301 loop_set_size(lo, new_size); 1302 } 1303 1304 loop_config_discard(lo); 1305 1306 /* update dio if lo_offset or transfer is changed */ 1307 __loop_update_dio(lo, lo->use_dio); 1308 1309out_unfreeze: 1310 blk_mq_unfreeze_queue(lo->lo_queue); 1311 1312 if (!err && (lo->lo_flags & LO_FLAGS_PARTSCAN) && 1313 !(prev_lo_flags & LO_FLAGS_PARTSCAN)) { 1314 clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state); 1315 partscan = true; 1316 } 1317out_unlock: 1318 mutex_unlock(&lo->lo_mutex); 1319 if (partscan) 1320 loop_reread_partitions(lo); 1321 1322 return err; 1323} 1324 1325static int 1326loop_get_status(struct loop_device *lo, struct loop_info64 *info) 1327{ 1328 struct path path; 1329 struct kstat stat; 1330 int ret; 1331 1332 ret = mutex_lock_killable(&lo->lo_mutex); 1333 if (ret) 1334 return ret; 1335 if (lo->lo_state != Lo_bound) { 1336 mutex_unlock(&lo->lo_mutex); 1337 return -ENXIO; 1338 } 1339 1340 memset(info, 0, sizeof(*info)); 1341 info->lo_number = lo->lo_number; 1342 info->lo_offset = lo->lo_offset; 1343 info->lo_sizelimit = lo->lo_sizelimit; 1344 info->lo_flags = lo->lo_flags; 1345 memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE); 1346 1347 /* Drop lo_mutex while we call into the filesystem. */ 1348 path = lo->lo_backing_file->f_path; 1349 path_get(&path); 1350 mutex_unlock(&lo->lo_mutex); 1351 ret = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT); 1352 if (!ret) { 1353 info->lo_device = huge_encode_dev(stat.dev); 1354 info->lo_inode = stat.ino; 1355 info->lo_rdevice = huge_encode_dev(stat.rdev); 1356 } 1357 path_put(&path); 1358 return ret; 1359} 1360 1361static void 1362loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64) 1363{ 1364 memset(info64, 0, sizeof(*info64)); 1365 info64->lo_number = info->lo_number; 1366 info64->lo_device = info->lo_device; 1367 info64->lo_inode = info->lo_inode; 1368 info64->lo_rdevice = info->lo_rdevice; 1369 info64->lo_offset = info->lo_offset; 1370 info64->lo_sizelimit = 0; 1371 info64->lo_flags = info->lo_flags; 1372 memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE); 1373} 1374 1375static int 1376loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info) 1377{ 1378 memset(info, 0, sizeof(*info)); 1379 info->lo_number = info64->lo_number; 1380 info->lo_device = info64->lo_device; 1381 info->lo_inode = info64->lo_inode; 1382 info->lo_rdevice = info64->lo_rdevice; 1383 info->lo_offset = info64->lo_offset; 1384 info->lo_flags = info64->lo_flags; 1385 memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE); 1386 1387 /* error in case values were truncated */ 1388 if (info->lo_device != info64->lo_device || 1389 info->lo_rdevice != info64->lo_rdevice || 1390 info->lo_inode != info64->lo_inode || 1391 info->lo_offset != info64->lo_offset) 1392 return -EOVERFLOW; 1393 1394 return 0; 1395} 1396 1397static int 1398loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg) 1399{ 1400 struct loop_info info; 1401 struct loop_info64 info64; 1402 1403 if (copy_from_user(&info, arg, sizeof (struct loop_info))) 1404 return -EFAULT; 1405 loop_info64_from_old(&info, &info64); 1406 return loop_set_status(lo, &info64); 1407} 1408 1409static int 1410loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg) 1411{ 1412 struct loop_info64 info64; 1413 1414 if (copy_from_user(&info64, arg, sizeof (struct loop_info64))) 1415 return -EFAULT; 1416 return loop_set_status(lo, &info64); 1417} 1418 1419static int 1420loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) { 1421 struct loop_info info; 1422 struct loop_info64 info64; 1423 int err; 1424 1425 if (!arg) 1426 return -EINVAL; 1427 err = loop_get_status(lo, &info64); 1428 if (!err) 1429 err = loop_info64_to_old(&info64, &info); 1430 if (!err && copy_to_user(arg, &info, sizeof(info))) 1431 err = -EFAULT; 1432 1433 return err; 1434} 1435 1436static int 1437loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) { 1438 struct loop_info64 info64; 1439 int err; 1440 1441 if (!arg) 1442 return -EINVAL; 1443 err = loop_get_status(lo, &info64); 1444 if (!err && copy_to_user(arg, &info64, sizeof(info64))) 1445 err = -EFAULT; 1446 1447 return err; 1448} 1449 1450static int loop_set_capacity(struct loop_device *lo) 1451{ 1452 loff_t size; 1453 1454 if (unlikely(lo->lo_state != Lo_bound)) 1455 return -ENXIO; 1456 1457 size = get_loop_size(lo, lo->lo_backing_file); 1458 loop_set_size(lo, size); 1459 1460 return 0; 1461} 1462 1463static int loop_set_dio(struct loop_device *lo, unsigned long arg) 1464{ 1465 int error = -ENXIO; 1466 if (lo->lo_state != Lo_bound) 1467 goto out; 1468 1469 __loop_update_dio(lo, !!arg); 1470 if (lo->use_dio == !!arg) 1471 return 0; 1472 error = -EINVAL; 1473 out: 1474 return error; 1475} 1476 1477static int loop_set_block_size(struct loop_device *lo, unsigned long arg) 1478{ 1479 int err = 0; 1480 1481 if (lo->lo_state != Lo_bound) 1482 return -ENXIO; 1483 1484 err = blk_validate_block_size(arg); 1485 if (err) 1486 return err; 1487 1488 if (lo->lo_queue->limits.logical_block_size == arg) 1489 return 0; 1490 1491 sync_blockdev(lo->lo_device); 1492 invalidate_bdev(lo->lo_device); 1493 1494 blk_mq_freeze_queue(lo->lo_queue); 1495 blk_queue_logical_block_size(lo->lo_queue, arg); 1496 blk_queue_physical_block_size(lo->lo_queue, arg); 1497 blk_queue_io_min(lo->lo_queue, arg); 1498 loop_update_dio(lo); 1499 blk_mq_unfreeze_queue(lo->lo_queue); 1500 1501 return err; 1502} 1503 1504static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd, 1505 unsigned long arg) 1506{ 1507 int err; 1508 1509 err = mutex_lock_killable(&lo->lo_mutex); 1510 if (err) 1511 return err; 1512 switch (cmd) { 1513 case LOOP_SET_CAPACITY: 1514 err = loop_set_capacity(lo); 1515 break; 1516 case LOOP_SET_DIRECT_IO: 1517 err = loop_set_dio(lo, arg); 1518 break; 1519 case LOOP_SET_BLOCK_SIZE: 1520 err = loop_set_block_size(lo, arg); 1521 break; 1522 default: 1523 err = -EINVAL; 1524 } 1525 mutex_unlock(&lo->lo_mutex); 1526 return err; 1527} 1528 1529static int lo_ioctl(struct block_device *bdev, blk_mode_t mode, 1530 unsigned int cmd, unsigned long arg) 1531{ 1532 struct loop_device *lo = bdev->bd_disk->private_data; 1533 void __user *argp = (void __user *) arg; 1534 int err; 1535 1536 switch (cmd) { 1537 case LOOP_SET_FD: { 1538 /* 1539 * Legacy case - pass in a zeroed out struct loop_config with 1540 * only the file descriptor set , which corresponds with the 1541 * default parameters we'd have used otherwise. 1542 */ 1543 struct loop_config config; 1544 1545 memset(&config, 0, sizeof(config)); 1546 config.fd = arg; 1547 1548 return loop_configure(lo, mode, bdev, &config); 1549 } 1550 case LOOP_CONFIGURE: { 1551 struct loop_config config; 1552 1553 if (copy_from_user(&config, argp, sizeof(config))) 1554 return -EFAULT; 1555 1556 return loop_configure(lo, mode, bdev, &config); 1557 } 1558 case LOOP_CHANGE_FD: 1559 return loop_change_fd(lo, bdev, arg); 1560 case LOOP_CLR_FD: 1561 return loop_clr_fd(lo); 1562 case LOOP_SET_STATUS: 1563 err = -EPERM; 1564 if ((mode & BLK_OPEN_WRITE) || capable(CAP_SYS_ADMIN)) 1565 err = loop_set_status_old(lo, argp); 1566 break; 1567 case LOOP_GET_STATUS: 1568 return loop_get_status_old(lo, argp); 1569 case LOOP_SET_STATUS64: 1570 err = -EPERM; 1571 if ((mode & BLK_OPEN_WRITE) || capable(CAP_SYS_ADMIN)) 1572 err = loop_set_status64(lo, argp); 1573 break; 1574 case LOOP_GET_STATUS64: 1575 return loop_get_status64(lo, argp); 1576 case LOOP_SET_CAPACITY: 1577 case LOOP_SET_DIRECT_IO: 1578 case LOOP_SET_BLOCK_SIZE: 1579 if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN)) 1580 return -EPERM; 1581 fallthrough; 1582 default: 1583 err = lo_simple_ioctl(lo, cmd, arg); 1584 break; 1585 } 1586 1587 return err; 1588} 1589 1590#ifdef CONFIG_COMPAT 1591struct compat_loop_info { 1592 compat_int_t lo_number; /* ioctl r/o */ 1593 compat_dev_t lo_device; /* ioctl r/o */ 1594 compat_ulong_t lo_inode; /* ioctl r/o */ 1595 compat_dev_t lo_rdevice; /* ioctl r/o */ 1596 compat_int_t lo_offset; 1597 compat_int_t lo_encrypt_type; /* obsolete, ignored */ 1598 compat_int_t lo_encrypt_key_size; /* ioctl w/o */ 1599 compat_int_t lo_flags; /* ioctl r/o */ 1600 char lo_name[LO_NAME_SIZE]; 1601 unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */ 1602 compat_ulong_t lo_init[2]; 1603 char reserved[4]; 1604}; 1605 1606/* 1607 * Transfer 32-bit compatibility structure in userspace to 64-bit loop info 1608 * - noinlined to reduce stack space usage in main part of driver 1609 */ 1610static noinline int 1611loop_info64_from_compat(const struct compat_loop_info __user *arg, 1612 struct loop_info64 *info64) 1613{ 1614 struct compat_loop_info info; 1615 1616 if (copy_from_user(&info, arg, sizeof(info))) 1617 return -EFAULT; 1618 1619 memset(info64, 0, sizeof(*info64)); 1620 info64->lo_number = info.lo_number; 1621 info64->lo_device = info.lo_device; 1622 info64->lo_inode = info.lo_inode; 1623 info64->lo_rdevice = info.lo_rdevice; 1624 info64->lo_offset = info.lo_offset; 1625 info64->lo_sizelimit = 0; 1626 info64->lo_flags = info.lo_flags; 1627 memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE); 1628 return 0; 1629} 1630 1631/* 1632 * Transfer 64-bit loop info to 32-bit compatibility structure in userspace 1633 * - noinlined to reduce stack space usage in main part of driver 1634 */ 1635static noinline int 1636loop_info64_to_compat(const struct loop_info64 *info64, 1637 struct compat_loop_info __user *arg) 1638{ 1639 struct compat_loop_info info; 1640 1641 memset(&info, 0, sizeof(info)); 1642 info.lo_number = info64->lo_number; 1643 info.lo_device = info64->lo_device; 1644 info.lo_inode = info64->lo_inode; 1645 info.lo_rdevice = info64->lo_rdevice; 1646 info.lo_offset = info64->lo_offset; 1647 info.lo_flags = info64->lo_flags; 1648 memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE); 1649 1650 /* error in case values were truncated */ 1651 if (info.lo_device != info64->lo_device || 1652 info.lo_rdevice != info64->lo_rdevice || 1653 info.lo_inode != info64->lo_inode || 1654 info.lo_offset != info64->lo_offset) 1655 return -EOVERFLOW; 1656 1657 if (copy_to_user(arg, &info, sizeof(info))) 1658 return -EFAULT; 1659 return 0; 1660} 1661 1662static int 1663loop_set_status_compat(struct loop_device *lo, 1664 const struct compat_loop_info __user *arg) 1665{ 1666 struct loop_info64 info64; 1667 int ret; 1668 1669 ret = loop_info64_from_compat(arg, &info64); 1670 if (ret < 0) 1671 return ret; 1672 return loop_set_status(lo, &info64); 1673} 1674 1675static int 1676loop_get_status_compat(struct loop_device *lo, 1677 struct compat_loop_info __user *arg) 1678{ 1679 struct loop_info64 info64; 1680 int err; 1681 1682 if (!arg) 1683 return -EINVAL; 1684 err = loop_get_status(lo, &info64); 1685 if (!err) 1686 err = loop_info64_to_compat(&info64, arg); 1687 return err; 1688} 1689 1690static int lo_compat_ioctl(struct block_device *bdev, blk_mode_t mode, 1691 unsigned int cmd, unsigned long arg) 1692{ 1693 struct loop_device *lo = bdev->bd_disk->private_data; 1694 int err; 1695 1696 switch(cmd) { 1697 case LOOP_SET_STATUS: 1698 err = loop_set_status_compat(lo, 1699 (const struct compat_loop_info __user *)arg); 1700 break; 1701 case LOOP_GET_STATUS: 1702 err = loop_get_status_compat(lo, 1703 (struct compat_loop_info __user *)arg); 1704 break; 1705 case LOOP_SET_CAPACITY: 1706 case LOOP_CLR_FD: 1707 case LOOP_GET_STATUS64: 1708 case LOOP_SET_STATUS64: 1709 case LOOP_CONFIGURE: 1710 arg = (unsigned long) compat_ptr(arg); 1711 fallthrough; 1712 case LOOP_SET_FD: 1713 case LOOP_CHANGE_FD: 1714 case LOOP_SET_BLOCK_SIZE: 1715 case LOOP_SET_DIRECT_IO: 1716 err = lo_ioctl(bdev, mode, cmd, arg); 1717 break; 1718 default: 1719 err = -ENOIOCTLCMD; 1720 break; 1721 } 1722 return err; 1723} 1724#endif 1725 1726static void lo_release(struct gendisk *disk) 1727{ 1728 struct loop_device *lo = disk->private_data; 1729 1730 if (disk_openers(disk) > 0) 1731 return; 1732 1733 mutex_lock(&lo->lo_mutex); 1734 if (lo->lo_state == Lo_bound && (lo->lo_flags & LO_FLAGS_AUTOCLEAR)) { 1735 lo->lo_state = Lo_rundown; 1736 mutex_unlock(&lo->lo_mutex); 1737 /* 1738 * In autoclear mode, stop the loop thread 1739 * and remove configuration after last close. 1740 */ 1741 __loop_clr_fd(lo, true); 1742 return; 1743 } 1744 mutex_unlock(&lo->lo_mutex); 1745} 1746 1747static void lo_free_disk(struct gendisk *disk) 1748{ 1749 struct loop_device *lo = disk->private_data; 1750 1751 if (lo->workqueue) 1752 destroy_workqueue(lo->workqueue); 1753 loop_free_idle_workers(lo, true); 1754 timer_shutdown_sync(&lo->timer); 1755 mutex_destroy(&lo->lo_mutex); 1756 kfree(lo); 1757} 1758 1759static const struct block_device_operations lo_fops = { 1760 .owner = THIS_MODULE, 1761 .release = lo_release, 1762 .ioctl = lo_ioctl, 1763#ifdef CONFIG_COMPAT 1764 .compat_ioctl = lo_compat_ioctl, 1765#endif 1766 .free_disk = lo_free_disk, 1767}; 1768 1769/* 1770 * And now the modules code and kernel interface. 1771 */ 1772 1773/* 1774 * If max_loop is specified, create that many devices upfront. 1775 * This also becomes a hard limit. If max_loop is not specified, 1776 * the default isn't a hard limit (as before commit 85c50197716c 1777 * changed the default value from 0 for max_loop=0 reasons), just 1778 * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module 1779 * init time. Loop devices can be requested on-demand with the 1780 * /dev/loop-control interface, or be instantiated by accessing 1781 * a 'dead' device node. 1782 */ 1783static int max_loop = CONFIG_BLK_DEV_LOOP_MIN_COUNT; 1784 1785#ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD 1786static bool max_loop_specified; 1787 1788static int max_loop_param_set_int(const char *val, 1789 const struct kernel_param *kp) 1790{ 1791 int ret; 1792 1793 ret = param_set_int(val, kp); 1794 if (ret < 0) 1795 return ret; 1796 1797 max_loop_specified = true; 1798 return 0; 1799} 1800 1801static const struct kernel_param_ops max_loop_param_ops = { 1802 .set = max_loop_param_set_int, 1803 .get = param_get_int, 1804}; 1805 1806module_param_cb(max_loop, &max_loop_param_ops, &max_loop, 0444); 1807MODULE_PARM_DESC(max_loop, "Maximum number of loop devices"); 1808#else 1809module_param(max_loop, int, 0444); 1810MODULE_PARM_DESC(max_loop, "Initial number of loop devices"); 1811#endif 1812 1813module_param(max_part, int, 0444); 1814MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device"); 1815 1816static int hw_queue_depth = LOOP_DEFAULT_HW_Q_DEPTH; 1817 1818static int loop_set_hw_queue_depth(const char *s, const struct kernel_param *p) 1819{ 1820 int qd, ret; 1821 1822 ret = kstrtoint(s, 0, &qd); 1823 if (ret < 0) 1824 return ret; 1825 if (qd < 1) 1826 return -EINVAL; 1827 hw_queue_depth = qd; 1828 return 0; 1829} 1830 1831static const struct kernel_param_ops loop_hw_qdepth_param_ops = { 1832 .set = loop_set_hw_queue_depth, 1833 .get = param_get_int, 1834}; 1835 1836device_param_cb(hw_queue_depth, &loop_hw_qdepth_param_ops, &hw_queue_depth, 0444); 1837MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: " __stringify(LOOP_DEFAULT_HW_Q_DEPTH)); 1838 1839MODULE_LICENSE("GPL"); 1840MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR); 1841 1842static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx, 1843 const struct blk_mq_queue_data *bd) 1844{ 1845 struct request *rq = bd->rq; 1846 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 1847 struct loop_device *lo = rq->q->queuedata; 1848 1849 blk_mq_start_request(rq); 1850 1851 if (lo->lo_state != Lo_bound) 1852 return BLK_STS_IOERR; 1853 1854 switch (req_op(rq)) { 1855 case REQ_OP_FLUSH: 1856 case REQ_OP_DISCARD: 1857 case REQ_OP_WRITE_ZEROES: 1858 cmd->use_aio = false; 1859 break; 1860 default: 1861 cmd->use_aio = lo->use_dio; 1862 break; 1863 } 1864 1865 /* always use the first bio's css */ 1866 cmd->blkcg_css = NULL; 1867 cmd->memcg_css = NULL; 1868#ifdef CONFIG_BLK_CGROUP 1869 if (rq->bio) { 1870 cmd->blkcg_css = bio_blkcg_css(rq->bio); 1871#ifdef CONFIG_MEMCG 1872 if (cmd->blkcg_css) { 1873 cmd->memcg_css = 1874 cgroup_get_e_css(cmd->blkcg_css->cgroup, 1875 &memory_cgrp_subsys); 1876 } 1877#endif 1878 } 1879#endif 1880 loop_queue_work(lo, cmd); 1881 1882 return BLK_STS_OK; 1883} 1884 1885static void loop_handle_cmd(struct loop_cmd *cmd) 1886{ 1887 struct cgroup_subsys_state *cmd_blkcg_css = cmd->blkcg_css; 1888 struct cgroup_subsys_state *cmd_memcg_css = cmd->memcg_css; 1889 struct request *rq = blk_mq_rq_from_pdu(cmd); 1890 const bool write = op_is_write(req_op(rq)); 1891 struct loop_device *lo = rq->q->queuedata; 1892 int ret = 0; 1893 struct mem_cgroup *old_memcg = NULL; 1894 const bool use_aio = cmd->use_aio; 1895 1896 if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) { 1897 ret = -EIO; 1898 goto failed; 1899 } 1900 1901 if (cmd_blkcg_css) 1902 kthread_associate_blkcg(cmd_blkcg_css); 1903 if (cmd_memcg_css) 1904 old_memcg = set_active_memcg( 1905 mem_cgroup_from_css(cmd_memcg_css)); 1906 1907 /* 1908 * do_req_filebacked() may call blk_mq_complete_request() synchronously 1909 * or asynchronously if using aio. Hence, do not touch 'cmd' after 1910 * do_req_filebacked() has returned unless we are sure that 'cmd' has 1911 * not yet been completed. 1912 */ 1913 ret = do_req_filebacked(lo, rq); 1914 1915 if (cmd_blkcg_css) 1916 kthread_associate_blkcg(NULL); 1917 1918 if (cmd_memcg_css) { 1919 set_active_memcg(old_memcg); 1920 css_put(cmd_memcg_css); 1921 } 1922 failed: 1923 /* complete non-aio request */ 1924 if (!use_aio || ret) { 1925 if (ret == -EOPNOTSUPP) 1926 cmd->ret = ret; 1927 else 1928 cmd->ret = ret ? -EIO : 0; 1929 if (likely(!blk_should_fake_timeout(rq->q))) 1930 blk_mq_complete_request(rq); 1931 } 1932} 1933 1934static void loop_process_work(struct loop_worker *worker, 1935 struct list_head *cmd_list, struct loop_device *lo) 1936{ 1937 int orig_flags = current->flags; 1938 struct loop_cmd *cmd; 1939 1940 current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; 1941 spin_lock_irq(&lo->lo_work_lock); 1942 while (!list_empty(cmd_list)) { 1943 cmd = container_of( 1944 cmd_list->next, struct loop_cmd, list_entry); 1945 list_del(cmd_list->next); 1946 spin_unlock_irq(&lo->lo_work_lock); 1947 1948 loop_handle_cmd(cmd); 1949 cond_resched(); 1950 1951 spin_lock_irq(&lo->lo_work_lock); 1952 } 1953 1954 /* 1955 * We only add to the idle list if there are no pending cmds 1956 * *and* the worker will not run again which ensures that it 1957 * is safe to free any worker on the idle list 1958 */ 1959 if (worker && !work_pending(&worker->work)) { 1960 worker->last_ran_at = jiffies; 1961 list_add_tail(&worker->idle_list, &lo->idle_worker_list); 1962 loop_set_timer(lo); 1963 } 1964 spin_unlock_irq(&lo->lo_work_lock); 1965 current->flags = orig_flags; 1966} 1967 1968static void loop_workfn(struct work_struct *work) 1969{ 1970 struct loop_worker *worker = 1971 container_of(work, struct loop_worker, work); 1972 loop_process_work(worker, &worker->cmd_list, worker->lo); 1973} 1974 1975static void loop_rootcg_workfn(struct work_struct *work) 1976{ 1977 struct loop_device *lo = 1978 container_of(work, struct loop_device, rootcg_work); 1979 loop_process_work(NULL, &lo->rootcg_cmd_list, lo); 1980} 1981 1982static const struct blk_mq_ops loop_mq_ops = { 1983 .queue_rq = loop_queue_rq, 1984 .complete = lo_complete_rq, 1985}; 1986 1987static int loop_add(int i) 1988{ 1989 struct loop_device *lo; 1990 struct gendisk *disk; 1991 int err; 1992 1993 err = -ENOMEM; 1994 lo = kzalloc(sizeof(*lo), GFP_KERNEL); 1995 if (!lo) 1996 goto out; 1997 lo->worker_tree = RB_ROOT; 1998 INIT_LIST_HEAD(&lo->idle_worker_list); 1999 timer_setup(&lo->timer, loop_free_idle_workers_timer, TIMER_DEFERRABLE); 2000 lo->lo_state = Lo_unbound; 2001 2002 err = mutex_lock_killable(&loop_ctl_mutex); 2003 if (err) 2004 goto out_free_dev; 2005 2006 /* allocate id, if @id >= 0, we're requesting that specific id */ 2007 if (i >= 0) { 2008 err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL); 2009 if (err == -ENOSPC) 2010 err = -EEXIST; 2011 } else { 2012 err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL); 2013 } 2014 mutex_unlock(&loop_ctl_mutex); 2015 if (err < 0) 2016 goto out_free_dev; 2017 i = err; 2018 2019 lo->tag_set.ops = &loop_mq_ops; 2020 lo->tag_set.nr_hw_queues = 1; 2021 lo->tag_set.queue_depth = hw_queue_depth; 2022 lo->tag_set.numa_node = NUMA_NO_NODE; 2023 lo->tag_set.cmd_size = sizeof(struct loop_cmd); 2024 lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_STACKING | 2025 BLK_MQ_F_NO_SCHED_BY_DEFAULT; 2026 lo->tag_set.driver_data = lo; 2027 2028 err = blk_mq_alloc_tag_set(&lo->tag_set); 2029 if (err) 2030 goto out_free_idr; 2031 2032 disk = lo->lo_disk = blk_mq_alloc_disk(&lo->tag_set, lo); 2033 if (IS_ERR(disk)) { 2034 err = PTR_ERR(disk); 2035 goto out_cleanup_tags; 2036 } 2037 lo->lo_queue = lo->lo_disk->queue; 2038 2039 blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS); 2040 2041 /* 2042 * By default, we do buffer IO, so it doesn't make sense to enable 2043 * merge because the I/O submitted to backing file is handled page by 2044 * page. For directio mode, merge does help to dispatch bigger request 2045 * to underlayer disk. We will enable merge once directio is enabled. 2046 */ 2047 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 2048 2049 /* 2050 * Disable partition scanning by default. The in-kernel partition 2051 * scanning can be requested individually per-device during its 2052 * setup. Userspace can always add and remove partitions from all 2053 * devices. The needed partition minors are allocated from the 2054 * extended minor space, the main loop device numbers will continue 2055 * to match the loop minors, regardless of the number of partitions 2056 * used. 2057 * 2058 * If max_part is given, partition scanning is globally enabled for 2059 * all loop devices. The minors for the main loop devices will be 2060 * multiples of max_part. 2061 * 2062 * Note: Global-for-all-devices, set-only-at-init, read-only module 2063 * parameteters like 'max_loop' and 'max_part' make things needlessly 2064 * complicated, are too static, inflexible and may surprise 2065 * userspace tools. Parameters like this in general should be avoided. 2066 */ 2067 if (!part_shift) 2068 set_bit(GD_SUPPRESS_PART_SCAN, &disk->state); 2069 mutex_init(&lo->lo_mutex); 2070 lo->lo_number = i; 2071 spin_lock_init(&lo->lo_lock); 2072 spin_lock_init(&lo->lo_work_lock); 2073 INIT_WORK(&lo->rootcg_work, loop_rootcg_workfn); 2074 INIT_LIST_HEAD(&lo->rootcg_cmd_list); 2075 disk->major = LOOP_MAJOR; 2076 disk->first_minor = i << part_shift; 2077 disk->minors = 1 << part_shift; 2078 disk->fops = &lo_fops; 2079 disk->private_data = lo; 2080 disk->queue = lo->lo_queue; 2081 disk->events = DISK_EVENT_MEDIA_CHANGE; 2082 disk->event_flags = DISK_EVENT_FLAG_UEVENT; 2083 sprintf(disk->disk_name, "loop%d", i); 2084 /* Make this loop device reachable from pathname. */ 2085 err = add_disk(disk); 2086 if (err) 2087 goto out_cleanup_disk; 2088 2089 /* Show this loop device. */ 2090 mutex_lock(&loop_ctl_mutex); 2091 lo->idr_visible = true; 2092 mutex_unlock(&loop_ctl_mutex); 2093 2094 return i; 2095 2096out_cleanup_disk: 2097 put_disk(disk); 2098out_cleanup_tags: 2099 blk_mq_free_tag_set(&lo->tag_set); 2100out_free_idr: 2101 mutex_lock(&loop_ctl_mutex); 2102 idr_remove(&loop_index_idr, i); 2103 mutex_unlock(&loop_ctl_mutex); 2104out_free_dev: 2105 kfree(lo); 2106out: 2107 return err; 2108} 2109 2110static void loop_remove(struct loop_device *lo) 2111{ 2112 /* Make this loop device unreachable from pathname. */ 2113 del_gendisk(lo->lo_disk); 2114 blk_mq_free_tag_set(&lo->tag_set); 2115 2116 mutex_lock(&loop_ctl_mutex); 2117 idr_remove(&loop_index_idr, lo->lo_number); 2118 mutex_unlock(&loop_ctl_mutex); 2119 2120 put_disk(lo->lo_disk); 2121} 2122 2123#ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD 2124static void loop_probe(dev_t dev) 2125{ 2126 int idx = MINOR(dev) >> part_shift; 2127 2128 if (max_loop_specified && max_loop && idx >= max_loop) 2129 return; 2130 loop_add(idx); 2131} 2132#else 2133#define loop_probe NULL 2134#endif /* !CONFIG_BLOCK_LEGACY_AUTOLOAD */ 2135 2136static int loop_control_remove(int idx) 2137{ 2138 struct loop_device *lo; 2139 int ret; 2140 2141 if (idx < 0) { 2142 pr_warn_once("deleting an unspecified loop device is not supported.\n"); 2143 return -EINVAL; 2144 } 2145 2146 /* Hide this loop device for serialization. */ 2147 ret = mutex_lock_killable(&loop_ctl_mutex); 2148 if (ret) 2149 return ret; 2150 lo = idr_find(&loop_index_idr, idx); 2151 if (!lo || !lo->idr_visible) 2152 ret = -ENODEV; 2153 else 2154 lo->idr_visible = false; 2155 mutex_unlock(&loop_ctl_mutex); 2156 if (ret) 2157 return ret; 2158 2159 /* Check whether this loop device can be removed. */ 2160 ret = mutex_lock_killable(&lo->lo_mutex); 2161 if (ret) 2162 goto mark_visible; 2163 if (lo->lo_state != Lo_unbound || disk_openers(lo->lo_disk) > 0) { 2164 mutex_unlock(&lo->lo_mutex); 2165 ret = -EBUSY; 2166 goto mark_visible; 2167 } 2168 /* Mark this loop device as no more bound, but not quite unbound yet */ 2169 lo->lo_state = Lo_deleting; 2170 mutex_unlock(&lo->lo_mutex); 2171 2172 loop_remove(lo); 2173 return 0; 2174 2175mark_visible: 2176 /* Show this loop device again. */ 2177 mutex_lock(&loop_ctl_mutex); 2178 lo->idr_visible = true; 2179 mutex_unlock(&loop_ctl_mutex); 2180 return ret; 2181} 2182 2183static int loop_control_get_free(int idx) 2184{ 2185 struct loop_device *lo; 2186 int id, ret; 2187 2188 ret = mutex_lock_killable(&loop_ctl_mutex); 2189 if (ret) 2190 return ret; 2191 idr_for_each_entry(&loop_index_idr, lo, id) { 2192 /* Hitting a race results in creating a new loop device which is harmless. */ 2193 if (lo->idr_visible && data_race(lo->lo_state) == Lo_unbound) 2194 goto found; 2195 } 2196 mutex_unlock(&loop_ctl_mutex); 2197 return loop_add(-1); 2198found: 2199 mutex_unlock(&loop_ctl_mutex); 2200 return id; 2201} 2202 2203static long loop_control_ioctl(struct file *file, unsigned int cmd, 2204 unsigned long parm) 2205{ 2206 switch (cmd) { 2207 case LOOP_CTL_ADD: 2208 return loop_add(parm); 2209 case LOOP_CTL_REMOVE: 2210 return loop_control_remove(parm); 2211 case LOOP_CTL_GET_FREE: 2212 return loop_control_get_free(parm); 2213 default: 2214 return -ENOSYS; 2215 } 2216} 2217 2218static const struct file_operations loop_ctl_fops = { 2219 .open = nonseekable_open, 2220 .unlocked_ioctl = loop_control_ioctl, 2221 .compat_ioctl = loop_control_ioctl, 2222 .owner = THIS_MODULE, 2223 .llseek = noop_llseek, 2224}; 2225 2226static struct miscdevice loop_misc = { 2227 .minor = LOOP_CTRL_MINOR, 2228 .name = "loop-control", 2229 .fops = &loop_ctl_fops, 2230}; 2231 2232MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR); 2233MODULE_ALIAS("devname:loop-control"); 2234 2235static int __init loop_init(void) 2236{ 2237 int i; 2238 int err; 2239 2240 part_shift = 0; 2241 if (max_part > 0) { 2242 part_shift = fls(max_part); 2243 2244 /* 2245 * Adjust max_part according to part_shift as it is exported 2246 * to user space so that user can decide correct minor number 2247 * if [s]he want to create more devices. 2248 * 2249 * Note that -1 is required because partition 0 is reserved 2250 * for the whole disk. 2251 */ 2252 max_part = (1UL << part_shift) - 1; 2253 } 2254 2255 if ((1UL << part_shift) > DISK_MAX_PARTS) { 2256 err = -EINVAL; 2257 goto err_out; 2258 } 2259 2260 if (max_loop > 1UL << (MINORBITS - part_shift)) { 2261 err = -EINVAL; 2262 goto err_out; 2263 } 2264 2265 err = misc_register(&loop_misc); 2266 if (err < 0) 2267 goto err_out; 2268 2269 2270 if (__register_blkdev(LOOP_MAJOR, "loop", loop_probe)) { 2271 err = -EIO; 2272 goto misc_out; 2273 } 2274 2275 /* pre-create number of devices given by config or max_loop */ 2276 for (i = 0; i < max_loop; i++) 2277 loop_add(i); 2278 2279 printk(KERN_INFO "loop: module loaded\n"); 2280 return 0; 2281 2282misc_out: 2283 misc_deregister(&loop_misc); 2284err_out: 2285 return err; 2286} 2287 2288static void __exit loop_exit(void) 2289{ 2290 struct loop_device *lo; 2291 int id; 2292 2293 unregister_blkdev(LOOP_MAJOR, "loop"); 2294 misc_deregister(&loop_misc); 2295 2296 /* 2297 * There is no need to use loop_ctl_mutex here, for nobody else can 2298 * access loop_index_idr when this module is unloading (unless forced 2299 * module unloading is requested). If this is not a clean unloading, 2300 * we have no means to avoid kernel crash. 2301 */ 2302 idr_for_each_entry(&loop_index_idr, lo, id) 2303 loop_remove(lo); 2304 2305 idr_destroy(&loop_index_idr); 2306} 2307 2308module_init(loop_init); 2309module_exit(loop_exit); 2310 2311#ifndef MODULE 2312static int __init max_loop_setup(char *str) 2313{ 2314 max_loop = simple_strtol(str, NULL, 0); 2315#ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD 2316 max_loop_specified = true; 2317#endif 2318 return 1; 2319} 2320 2321__setup("max_loop=", max_loop_setup); 2322#endif 2323