1/* SPDX-License-Identifier: GPL-2.0 */ 2#ifndef BLK_INTERNAL_H 3#define BLK_INTERNAL_H 4 5#include <linux/idr.h> 6#include <linux/blk-mq.h> 7#include <linux/part_stat.h> 8#include <linux/blk-crypto.h> 9#include <xen/xen.h> 10#include "blk-crypto-internal.h" 11#include "blk-mq.h" 12#include "blk-mq-sched.h" 13 14/* Max future timer expiry for timeouts */ 15#define BLK_MAX_TIMEOUT (5 * HZ) 16 17extern struct dentry *blk_debugfs_root; 18 19struct blk_flush_queue { 20 unsigned int flush_pending_idx:1; 21 unsigned int flush_running_idx:1; 22 blk_status_t rq_status; 23 unsigned long flush_pending_since; 24 struct list_head flush_queue[2]; 25 struct list_head flush_data_in_flight; 26 struct request *flush_rq; 27 28 spinlock_t mq_flush_lock; 29}; 30 31extern struct kmem_cache *blk_requestq_cachep; 32extern struct kobj_type blk_queue_ktype; 33extern struct ida blk_queue_ida; 34 35static inline struct blk_flush_queue * 36blk_get_flush_queue(struct request_queue *q, struct blk_mq_ctx *ctx) 37{ 38 return blk_mq_map_queue(q, REQ_OP_FLUSH, ctx)->fq; 39} 40 41static inline void __blk_get_queue(struct request_queue *q) 42{ 43 kobject_get(&q->kobj); 44} 45 46bool is_flush_rq(struct request *req); 47 48struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size, 49 gfp_t flags); 50void blk_free_flush_queue(struct blk_flush_queue *q); 51 52void blk_freeze_queue(struct request_queue *q); 53 54static inline bool biovec_phys_mergeable(struct request_queue *q, 55 struct bio_vec *vec1, struct bio_vec *vec2) 56{ 57 unsigned long mask = queue_segment_boundary(q); 58 phys_addr_t addr1 = page_to_phys(vec1->bv_page) + vec1->bv_offset; 59 phys_addr_t addr2 = page_to_phys(vec2->bv_page) + vec2->bv_offset; 60 61 if (addr1 + vec1->bv_len != addr2) 62 return false; 63 if (xen_domain() && !xen_biovec_phys_mergeable(vec1, vec2->bv_page)) 64 return false; 65 if ((addr1 | mask) != ((addr2 + vec2->bv_len - 1) | mask)) 66 return false; 67 return true; 68} 69 70static inline bool __bvec_gap_to_prev(struct request_queue *q, 71 struct bio_vec *bprv, unsigned int offset) 72{ 73 return (offset & queue_virt_boundary(q)) || 74 ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q)); 75} 76 77/* 78 * Check if adding a bio_vec after bprv with offset would create a gap in 79 * the SG list. Most drivers don't care about this, but some do. 80 */ 81static inline bool bvec_gap_to_prev(struct request_queue *q, 82 struct bio_vec *bprv, unsigned int offset) 83{ 84 if (!queue_virt_boundary(q)) 85 return false; 86 return __bvec_gap_to_prev(q, bprv, offset); 87} 88 89static inline void blk_rq_bio_prep(struct request *rq, struct bio *bio, 90 unsigned int nr_segs) 91{ 92 rq->nr_phys_segments = nr_segs; 93 rq->__data_len = bio->bi_iter.bi_size; 94 rq->bio = rq->biotail = bio; 95 rq->ioprio = bio_prio(bio); 96 97 if (bio->bi_disk) 98 rq->rq_disk = bio->bi_disk; 99} 100 101#ifdef CONFIG_BLK_DEV_INTEGRITY 102void blk_flush_integrity(void); 103bool __bio_integrity_endio(struct bio *); 104void bio_integrity_free(struct bio *bio); 105static inline bool bio_integrity_endio(struct bio *bio) 106{ 107 if (bio_integrity(bio)) 108 return __bio_integrity_endio(bio); 109 return true; 110} 111 112bool blk_integrity_merge_rq(struct request_queue *, struct request *, 113 struct request *); 114bool blk_integrity_merge_bio(struct request_queue *, struct request *, 115 struct bio *); 116 117static inline bool integrity_req_gap_back_merge(struct request *req, 118 struct bio *next) 119{ 120 struct bio_integrity_payload *bip = bio_integrity(req->bio); 121 struct bio_integrity_payload *bip_next = bio_integrity(next); 122 123 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], 124 bip_next->bip_vec[0].bv_offset); 125} 126 127static inline bool integrity_req_gap_front_merge(struct request *req, 128 struct bio *bio) 129{ 130 struct bio_integrity_payload *bip = bio_integrity(bio); 131 struct bio_integrity_payload *bip_next = bio_integrity(req->bio); 132 133 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1], 134 bip_next->bip_vec[0].bv_offset); 135} 136 137void blk_integrity_add(struct gendisk *); 138void blk_integrity_del(struct gendisk *); 139#else /* CONFIG_BLK_DEV_INTEGRITY */ 140static inline bool blk_integrity_merge_rq(struct request_queue *rq, 141 struct request *r1, struct request *r2) 142{ 143 return true; 144} 145static inline bool blk_integrity_merge_bio(struct request_queue *rq, 146 struct request *r, struct bio *b) 147{ 148 return true; 149} 150static inline bool integrity_req_gap_back_merge(struct request *req, 151 struct bio *next) 152{ 153 return false; 154} 155static inline bool integrity_req_gap_front_merge(struct request *req, 156 struct bio *bio) 157{ 158 return false; 159} 160 161static inline void blk_flush_integrity(void) 162{ 163} 164static inline bool bio_integrity_endio(struct bio *bio) 165{ 166 return true; 167} 168static inline void bio_integrity_free(struct bio *bio) 169{ 170} 171static inline void blk_integrity_add(struct gendisk *disk) 172{ 173} 174static inline void blk_integrity_del(struct gendisk *disk) 175{ 176} 177#endif /* CONFIG_BLK_DEV_INTEGRITY */ 178 179unsigned long blk_rq_timeout(unsigned long timeout); 180void blk_add_timer(struct request *req); 181 182bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio, 183 unsigned int nr_segs, struct request **same_queue_rq); 184bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, 185 struct bio *bio, unsigned int nr_segs); 186 187void blk_account_io_start(struct request *req); 188void blk_account_io_done(struct request *req, u64 now); 189 190/* 191 * Plug flush limits 192 */ 193#define BLK_MAX_REQUEST_COUNT 32 194#define BLK_PLUG_FLUSH_SIZE (128 * 1024) 195 196/* 197 * Internal elevator interface 198 */ 199#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED) 200 201void blk_insert_flush(struct request *rq); 202 203void elevator_init_mq(struct request_queue *q); 204int elevator_switch_mq(struct request_queue *q, 205 struct elevator_type *new_e); 206void __elevator_exit(struct request_queue *, struct elevator_queue *); 207int elv_register_queue(struct request_queue *q, bool uevent); 208void elv_unregister_queue(struct request_queue *q); 209 210static inline void elevator_exit(struct request_queue *q, 211 struct elevator_queue *e) 212{ 213 lockdep_assert_held(&q->sysfs_lock); 214 215 blk_mq_sched_free_requests(q); 216 __elevator_exit(q, e); 217} 218 219struct hd_struct *__disk_get_part(struct gendisk *disk, int partno); 220 221ssize_t part_size_show(struct device *dev, struct device_attribute *attr, 222 char *buf); 223ssize_t part_stat_show(struct device *dev, struct device_attribute *attr, 224 char *buf); 225ssize_t part_inflight_show(struct device *dev, struct device_attribute *attr, 226 char *buf); 227ssize_t part_fail_show(struct device *dev, struct device_attribute *attr, 228 char *buf); 229ssize_t part_fail_store(struct device *dev, struct device_attribute *attr, 230 const char *buf, size_t count); 231ssize_t part_timeout_show(struct device *, struct device_attribute *, char *); 232ssize_t part_timeout_store(struct device *, struct device_attribute *, 233 const char *, size_t); 234 235void __blk_queue_split(struct bio **bio, unsigned int *nr_segs); 236int ll_back_merge_fn(struct request *req, struct bio *bio, 237 unsigned int nr_segs); 238bool blk_attempt_req_merge(struct request_queue *q, struct request *rq, 239 struct request *next); 240unsigned int blk_recalc_rq_segments(struct request *rq); 241void blk_rq_set_mixed_merge(struct request *rq); 242bool blk_rq_merge_ok(struct request *rq, struct bio *bio); 243enum elv_merge blk_try_merge(struct request *rq, struct bio *bio); 244 245int blk_dev_init(void); 246 247/* 248 * Contribute to IO statistics IFF: 249 * 250 * a) it's attached to a gendisk, and 251 * b) the queue had IO stats enabled when this request was started 252 */ 253static inline bool blk_do_io_stat(struct request *rq) 254{ 255 return rq->rq_disk && (rq->rq_flags & RQF_IO_STAT); 256} 257 258static inline void req_set_nomerge(struct request_queue *q, struct request *req) 259{ 260 req->cmd_flags |= REQ_NOMERGE; 261 if (req == q->last_merge) 262 q->last_merge = NULL; 263} 264 265/* 266 * The max size one bio can handle is UINT_MAX becasue bvec_iter.bi_size 267 * is defined as 'unsigned int', meantime it has to aligned to with logical 268 * block size which is the minimum accepted unit by hardware. 269 */ 270static inline unsigned int bio_allowed_max_sectors(struct request_queue *q) 271{ 272 return round_down(UINT_MAX, queue_logical_block_size(q)) >> 9; 273} 274 275/* 276 * The max bio size which is aligned to q->limits.discard_granularity. This 277 * is a hint to split large discard bio in generic block layer, then if device 278 * driver needs to split the discard bio into smaller ones, their bi_size can 279 * be very probably and easily aligned to discard_granularity of the device's 280 * queue. 281 */ 282static inline unsigned int bio_aligned_discard_max_sectors( 283 struct request_queue *q) 284{ 285 return round_down(UINT_MAX, q->limits.discard_granularity) >> 286 SECTOR_SHIFT; 287} 288 289/* 290 * Internal io_context interface 291 */ 292void get_io_context(struct io_context *ioc); 293struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q); 294struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q, 295 gfp_t gfp_mask); 296void ioc_clear_queue(struct request_queue *q); 297 298int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node); 299 300/* 301 * Internal throttling interface 302 */ 303#ifdef CONFIG_BLK_DEV_THROTTLING 304extern int blk_throtl_init(struct request_queue *q); 305extern void blk_throtl_exit(struct request_queue *q); 306extern void blk_throtl_register_queue(struct request_queue *q); 307extern void blk_throtl_charge_bio_split(struct bio *bio); 308bool blk_throtl_bio(struct bio *bio); 309#else /* CONFIG_BLK_DEV_THROTTLING */ 310static inline int blk_throtl_init(struct request_queue *q) { return 0; } 311static inline void blk_throtl_exit(struct request_queue *q) { } 312static inline void blk_throtl_register_queue(struct request_queue *q) { } 313static inline void blk_throtl_charge_bio_split(struct bio *bio) { } 314static inline bool blk_throtl_bio(struct bio *bio) { return false; } 315#endif /* CONFIG_BLK_DEV_THROTTLING */ 316#ifdef CONFIG_BLK_DEV_THROTTLING_LOW 317extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page); 318extern ssize_t blk_throtl_sample_time_store(struct request_queue *q, 319 const char *page, size_t count); 320extern void blk_throtl_bio_endio(struct bio *bio); 321extern void blk_throtl_stat_add(struct request *rq, u64 time); 322#else 323static inline void blk_throtl_bio_endio(struct bio *bio) { } 324static inline void blk_throtl_stat_add(struct request *rq, u64 time) { } 325#endif 326 327#ifdef CONFIG_BOUNCE 328extern int init_emergency_isa_pool(void); 329extern void blk_queue_bounce(struct request_queue *q, struct bio **bio); 330#else 331static inline int init_emergency_isa_pool(void) 332{ 333 return 0; 334} 335static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio) 336{ 337} 338#endif /* CONFIG_BOUNCE */ 339 340#ifdef CONFIG_BLK_CGROUP_IOLATENCY 341extern int blk_iolatency_init(struct request_queue *q); 342#else 343static inline int blk_iolatency_init(struct request_queue *q) { return 0; } 344#endif 345 346struct bio *blk_next_bio(struct bio *bio, unsigned int nr_pages, gfp_t gfp); 347 348#ifdef CONFIG_BLK_DEV_ZONED 349void blk_queue_free_zone_bitmaps(struct request_queue *q); 350#else 351static inline void blk_queue_free_zone_bitmaps(struct request_queue *q) {} 352#endif 353 354struct hd_struct *disk_map_sector_rcu(struct gendisk *disk, sector_t sector); 355 356int blk_alloc_devt(struct hd_struct *part, dev_t *devt); 357void blk_free_devt(dev_t devt); 358void blk_invalidate_devt(dev_t devt); 359char *disk_name(struct gendisk *hd, int partno, char *buf); 360#define ADDPART_FLAG_NONE 0 361#define ADDPART_FLAG_RAID 1 362#define ADDPART_FLAG_WHOLEDISK 2 363void delete_partition(struct hd_struct *part); 364int bdev_add_partition(struct block_device *bdev, int partno, 365 sector_t start, sector_t length); 366int bdev_del_partition(struct block_device *bdev, int partno); 367int bdev_resize_partition(struct block_device *bdev, int partno, 368 sector_t start, sector_t length); 369int disk_expand_part_tbl(struct gendisk *disk, int target); 370int hd_ref_init(struct hd_struct *part); 371 372/* no need to get/put refcount of part0 */ 373static inline int hd_struct_try_get(struct hd_struct *part) 374{ 375 if (part->partno) 376 return percpu_ref_tryget_live(&part->ref); 377 return 1; 378} 379 380static inline void hd_struct_put(struct hd_struct *part) 381{ 382 if (part->partno) 383 percpu_ref_put(&part->ref); 384} 385 386static inline void hd_free_part(struct hd_struct *part) 387{ 388 free_percpu(part->dkstats); 389 kfree(part->info); 390 percpu_ref_exit(&part->ref); 391} 392 393/* 394 * Any access of part->nr_sects which is not protected by partition 395 * bd_mutex or gendisk bdev bd_mutex, should be done using this 396 * accessor function. 397 * 398 * Code written along the lines of i_size_read() and i_size_write(). 399 * CONFIG_PREEMPTION case optimizes the case of UP kernel with preemption 400 * on. 401 */ 402static inline sector_t part_nr_sects_read(struct hd_struct *part) 403{ 404#if BITS_PER_LONG==32 && defined(CONFIG_SMP) 405 sector_t nr_sects; 406 unsigned seq; 407 do { 408 seq = read_seqcount_begin(&part->nr_sects_seq); 409 nr_sects = part->nr_sects; 410 } while (read_seqcount_retry(&part->nr_sects_seq, seq)); 411 return nr_sects; 412#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) 413 sector_t nr_sects; 414 415 preempt_disable(); 416 nr_sects = part->nr_sects; 417 preempt_enable(); 418 return nr_sects; 419#else 420 return part->nr_sects; 421#endif 422} 423 424/* 425 * Should be called with mutex lock held (typically bd_mutex) of partition 426 * to provide mutual exlusion among writers otherwise seqcount might be 427 * left in wrong state leaving the readers spinning infinitely. 428 */ 429static inline void part_nr_sects_write(struct hd_struct *part, sector_t size) 430{ 431#if BITS_PER_LONG==32 && defined(CONFIG_SMP) 432 preempt_disable(); 433 write_seqcount_begin(&part->nr_sects_seq); 434 part->nr_sects = size; 435 write_seqcount_end(&part->nr_sects_seq); 436 preempt_enable(); 437#elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) 438 preempt_disable(); 439 part->nr_sects = size; 440 preempt_enable(); 441#else 442 part->nr_sects = size; 443#endif 444} 445 446int bio_add_hw_page(struct request_queue *q, struct bio *bio, 447 struct page *page, unsigned int len, unsigned int offset, 448 unsigned int max_sectors, bool *same_page); 449 450#endif /* BLK_INTERNAL_H */ 451