Lines Matching refs:ac
420 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac);
422 static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
872 static void ext4_mb_choose_next_group_p2_aligned(struct ext4_allocation_context *ac,
875 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
879 if (ac->ac_status == AC_STATUS_FOUND)
882 if (unlikely(sbi->s_mb_stats && ac->ac_flags & EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED))
885 for (i = ac->ac_2order; i < MB_NUM_ORDERS(ac->ac_sb); i++) {
897 if (likely(ext4_mb_good_group(ac, iter->bb_group, CR_POWER2_ALIGNED))) {
899 ac->ac_flags |= EXT4_MB_CR_POWER2_ALIGNED_OPTIMIZED;
915 ext4_mb_find_good_group_avg_frag_lists(struct ext4_allocation_context *ac, int order)
917 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
921 enum criteria cr = ac->ac_criteria;
933 if (likely(ext4_mb_good_group(ac, iter->bb_group, cr))) {
946 static void ext4_mb_choose_next_group_goal_fast(struct ext4_allocation_context *ac,
949 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
953 if (unlikely(ac->ac_flags & EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED)) {
958 for (i = mb_avg_fragment_size_order(ac->ac_sb, ac->ac_g_ex.fe_len);
959 i < MB_NUM_ORDERS(ac->ac_sb); i++) {
960 grp = ext4_mb_find_good_group_avg_frag_lists(ac, i);
963 ac->ac_flags |= EXT4_MB_CR_GOAL_LEN_FAST_OPTIMIZED;
976 if (ac->ac_flags & EXT4_MB_HINT_DATA)
991 static void ext4_mb_choose_next_group_best_avail(struct ext4_allocation_context *ac,
994 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
999 if (unlikely(ac->ac_flags & EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED)) {
1010 order = fls(ac->ac_g_ex.fe_len) - 1;
1029 if (1 << min_order < ac->ac_o_ex.fe_len)
1030 min_order = fls(ac->ac_o_ex.fe_len);
1039 ac->ac_g_ex.fe_len = 1 << i;
1047 ac->ac_g_ex.fe_len = roundup(ac->ac_g_ex.fe_len,
1051 frag_order = mb_avg_fragment_size_order(ac->ac_sb,
1052 ac->ac_g_ex.fe_len);
1054 grp = ext4_mb_find_good_group_avg_frag_lists(ac, frag_order);
1057 ac->ac_flags |= EXT4_MB_CR_BEST_AVAIL_LEN_OPTIMIZED;
1063 ac->ac_g_ex.fe_len = ac->ac_orig_goal_len;
1067 static inline int should_optimize_scan(struct ext4_allocation_context *ac)
1069 if (unlikely(!test_opt2(ac->ac_sb, MB_OPTIMIZE_SCAN)))
1071 if (ac->ac_criteria >= CR_GOAL_LEN_SLOW)
1073 if (!ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS))
1083 next_linear_group(struct ext4_allocation_context *ac, ext4_group_t group,
1086 if (!should_optimize_scan(ac))
1089 if (ac->ac_groups_linear_remaining) {
1090 ac->ac_groups_linear_remaining--;
1106 * @ac Allocation Context
1116 static void ext4_mb_choose_next_group(struct ext4_allocation_context *ac,
1119 *new_cr = ac->ac_criteria;
1121 if (!should_optimize_scan(ac) || ac->ac_groups_linear_remaining) {
1122 *group = next_linear_group(ac, *group, ngroups);
1127 ext4_mb_choose_next_group_p2_aligned(ac, new_cr, group, ngroups);
1129 ext4_mb_choose_next_group_goal_fast(ac, new_cr, group, ngroups);
1131 ext4_mb_choose_next_group_best_avail(ac, new_cr, group, ngroups);
2130 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,
2133 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
2136 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);
2137 BUG_ON(ac->ac_status == AC_STATUS_FOUND);
2139 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);
2140 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;
2141 ret = mb_mark_used(e4b, &ac->ac_b_ex);
2145 ac->ac_f_ex = ac->ac_b_ex;
2147 ac->ac_status = AC_STATUS_FOUND;
2148 ac->ac_tail = ret & 0xffff;
2149 ac->ac_buddy = ret >> 16;
2158 ac->ac_bitmap_page = e4b->bd_bitmap_page;
2159 get_page(ac->ac_bitmap_page);
2160 ac->ac_buddy_page = e4b->bd_buddy_page;
2161 get_page(ac->ac_buddy_page);
2163 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
2165 sbi->s_mb_last_group = ac->ac_f_ex.fe_group;
2166 sbi->s_mb_last_start = ac->ac_f_ex.fe_start;
2174 if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)
2175 ext4_mb_new_preallocation(ac);
2179 static void ext4_mb_check_limits(struct ext4_allocation_context *ac,
2183 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
2184 struct ext4_free_extent *bex = &ac->ac_b_ex;
2185 struct ext4_free_extent *gex = &ac->ac_g_ex;
2187 if (ac->ac_status == AC_STATUS_FOUND)
2192 if (ac->ac_found > sbi->s_mb_max_to_scan &&
2193 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
2194 ac->ac_status = AC_STATUS_BREAK;
2204 if (finish_group || ac->ac_found > sbi->s_mb_min_to_scan)
2205 ext4_mb_use_best_found(ac, e4b);
2232 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac,
2236 struct ext4_free_extent *bex = &ac->ac_b_ex;
2237 struct ext4_free_extent *gex = &ac->ac_g_ex;
2240 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));
2241 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));
2242 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);
2244 ac->ac_found++;
2245 ac->ac_cX_found[ac->ac_criteria]++;
2250 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
2252 ext4_mb_use_best_found(ac, e4b);
2261 ext4_mb_use_best_found(ac, e4b);
2289 ext4_mb_check_limits(ac, e4b, 0);
2293 void ext4_mb_try_best_found(struct ext4_allocation_context *ac,
2296 struct ext4_free_extent ex = ac->ac_b_ex;
2302 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
2306 ext4_lock_group(ac->ac_sb, group);
2313 ac->ac_b_ex = ex;
2314 ext4_mb_use_best_found(ac, e4b);
2318 ext4_unlock_group(ac->ac_sb, group);
2323 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac,
2326 ext4_group_t group = ac->ac_g_ex.fe_group;
2329 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
2330 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
2335 if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY)))
2340 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
2344 ext4_lock_group(ac->ac_sb, group);
2348 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,
2349 ac->ac_g_ex.fe_len, &ex);
2352 if (max >= ac->ac_g_ex.fe_len &&
2353 ac->ac_g_ex.fe_len == EXT4_B2C(sbi, sbi->s_stripe)) {
2356 start = ext4_grp_offs_to_block(ac->ac_sb, &ex);
2359 ac->ac_found++;
2360 ac->ac_b_ex = ex;
2361 ext4_mb_use_best_found(ac, e4b);
2363 } else if (max >= ac->ac_g_ex.fe_len) {
2365 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
2366 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
2367 ac->ac_found++;
2368 ac->ac_b_ex = ex;
2369 ext4_mb_use_best_found(ac, e4b);
2370 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {
2374 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
2375 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
2376 ac->ac_found++;
2377 ac->ac_b_ex = ex;
2378 ext4_mb_use_best_found(ac, e4b);
2381 ext4_unlock_group(ac->ac_sb, group);
2392 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac,
2395 struct super_block *sb = ac->ac_sb;
2402 BUG_ON(ac->ac_2order <= 0);
2403 for (i = ac->ac_2order; i < MB_NUM_ORDERS(sb); i++) {
2414 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0,
2417 ext4_mark_group_bitmap_corrupted(ac->ac_sb,
2422 ac->ac_found++;
2423 ac->ac_cX_found[ac->ac_criteria]++;
2425 ac->ac_b_ex.fe_len = 1 << i;
2426 ac->ac_b_ex.fe_start = k << i;
2427 ac->ac_b_ex.fe_group = e4b->bd_group;
2429 ext4_mb_use_best_found(ac, e4b);
2431 BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len);
2446 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac,
2449 struct super_block *sb = ac->ac_sb;
2461 while (free && ac->ac_status == AC_STATUS_CONTINUE) {
2479 if (!ext4_mb_cr_expensive(ac->ac_criteria)) {
2490 if (freelen < ac->ac_g_ex.fe_len) {
2497 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);
2515 ext4_mb_measure_extent(ac, &ex, e4b);
2521 ext4_mb_check_limits(ac, e4b, 1);
2529 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac,
2532 struct super_block *sb = ac->ac_sb;
2556 ac->ac_found++;
2557 ac->ac_cX_found[ac->ac_criteria]++;
2559 ac->ac_b_ex = ex;
2560 ext4_mb_use_best_found(ac, e4b);
2573 static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
2577 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));
2578 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
2595 BUG_ON(ac->ac_2order == 0);
2598 if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&
2603 if (free < ac->ac_g_ex.fe_len)
2606 if (ac->ac_2order >= MB_NUM_ORDERS(ac->ac_sb))
2609 if (grp->bb_largest_free_order < ac->ac_2order)
2615 if ((free / fragments) >= ac->ac_g_ex.fe_len)
2619 if (free >= ac->ac_g_ex.fe_len)
2642 static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac,
2645 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
2646 struct super_block *sb = ac->ac_sb;
2648 bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK;
2655 atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]);
2668 if (cr < CR_ANY_FREE && free < ac->ac_g_ex.fe_len)
2707 ret = ext4_mb_good_group(ac, group, cr);
2791 ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
2802 sb = ac->ac_sb;
2806 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))
2809 BUG_ON(ac->ac_status == AC_STATUS_FOUND);
2812 err = ext4_mb_find_by_goal(ac, &e4b);
2813 if (err || ac->ac_status == AC_STATUS_FOUND)
2816 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
2820 * ac->ac_2order is set only if the fe_len is a power of 2
2821 * if ac->ac_2order is set we also set criteria to CR_POWER2_ALIGNED
2824 i = fls(ac->ac_g_ex.fe_len);
2825 ac->ac_2order = 0;
2834 if (is_power_of_2(ac->ac_g_ex.fe_len))
2835 ac->ac_2order = array_index_nospec(i - 1,
2840 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
2843 ac->ac_g_ex.fe_group = sbi->s_mb_last_group;
2844 ac->ac_g_ex.fe_start = sbi->s_mb_last_start;
2853 if (ac->ac_2order)
2856 for (; cr < EXT4_MB_NUM_CRS && ac->ac_status == AC_STATUS_CONTINUE; cr++) {
2857 ac->ac_criteria = cr;
2862 group = ac->ac_g_ex.fe_group;
2863 ac->ac_groups_linear_remaining = sbi->s_mb_max_linear_groups;
2867 ext4_mb_choose_next_group(ac, &new_cr, &group, ngroups)) {
2896 ret = ext4_mb_good_group_nolock(ac, group, cr);
2913 ret = ext4_mb_good_group(ac, group, cr);
2920 ac->ac_groups_scanned++;
2922 ext4_mb_simple_scan_group(ac, &e4b);
2926 !(ac->ac_g_ex.fe_len %
2928 ext4_mb_scan_aligned(ac, &e4b);
2930 ext4_mb_complex_scan_group(ac, &e4b);
2935 if (ac->ac_status != AC_STATUS_CONTINUE)
2942 if (i == ngroups && ac->ac_criteria == CR_BEST_AVAIL_LEN)
2945 ac->ac_g_ex.fe_len = ac->ac_orig_goal_len;
2948 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&
2949 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
2954 ext4_mb_try_best_found(ac, &e4b);
2955 if (ac->ac_status != AC_STATUS_FOUND) {
2963 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start,
2964 ac->ac_b_ex.fe_len, lost);
2966 ac->ac_b_ex.fe_group = 0;
2967 ac->ac_b_ex.fe_start = 0;
2968 ac->ac_b_ex.fe_len = 0;
2969 ac->ac_status = AC_STATUS_CONTINUE;
2970 ac->ac_flags |= EXT4_MB_HINT_FIRST;
2976 if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND)
2977 atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]);
2979 if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)
2983 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status,
2984 ac->ac_flags, cr, err);
3982 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3986 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
3997 BUG_ON(ac->ac_status != AC_STATUS_FOUND);
3998 BUG_ON(ac->ac_b_ex.fe_len <= 0);
4000 sb = ac->ac_sb;
4003 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group);
4015 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);
4019 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,
4027 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
4029 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
4030 if (!ext4_inode_block_valid(ac->ac_inode, block, len)) {
4037 ext4_lock_group(sb, ac->ac_b_ex.fe_group);
4038 mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,
4039 ac->ac_b_ex.fe_len);
4040 ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
4047 ext4_lock_group(sb, ac->ac_b_ex.fe_group);
4051 for (i = 0; i < ac->ac_b_ex.fe_len; i++) {
4052 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,
4057 mb_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,
4058 ac->ac_b_ex.fe_len);
4064 ac->ac_b_ex.fe_group, gdp));
4066 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len;
4069 ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp);
4071 ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
4072 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);
4076 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))
4083 ac->ac_b_ex.fe_group);
4084 atomic64_sub(ac->ac_b_ex.fe_len,
4220 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac)
4222 struct super_block *sb = ac->ac_sb;
4223 struct ext4_locality_group *lg = ac->ac_lg;
4226 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;
4227 mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len);
4249 ext4_mb_pa_assert_overlap(struct ext4_allocation_context *ac,
4252 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4253 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
4276 * Given an allocation context "ac" and a range "start", "end", check
4281 * ac allocation context
4286 ext4_mb_pa_adjust_overlap(struct ext4_allocation_context *ac,
4289 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
4290 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4308 iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical,
4318 BUG_ON(!(ac->ac_o_ex.fe_logical >= tmp_pa_end ||
4319 ac->ac_o_ex.fe_logical < tmp_pa_start));
4328 if (tmp_pa->pa_lstart < ac->ac_o_ex.fe_logical) {
4394 BUG_ON(left_pa_end > ac->ac_o_ex.fe_logical);
4399 BUG_ON(right_pa_start <= ac->ac_o_ex.fe_logical);
4415 ext4_mb_pa_assert_overlap(ac, new_start, new_end);
4426 ext4_mb_normalize_request(struct ext4_allocation_context *ac,
4429 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4438 if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
4442 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
4447 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)
4450 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {
4451 ext4_mb_normalize_group_request(ac);
4455 bsbits = ac->ac_sb->s_blocksize_bits;
4459 size = extent_logical_end(sbi, &ac->ac_o_ex);
4461 if (size < i_size_read(ac->ac_inode))
4462 size = i_size_read(ac->ac_inode);
4489 start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
4493 start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
4496 } else if (NRL_CHECK_SIZE(EXT4_C2B(sbi, ac->ac_o_ex.fe_len),
4498 start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
4502 start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits;
4504 ac->ac_o_ex.fe_len) << bsbits;
4515 start = max(start, rounddown(ac->ac_o_ex.fe_logical,
4516 (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb)));
4534 if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb))
4535 size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb);
4539 ext4_mb_pa_adjust_overlap(ac, &start, &end);
4547 * ac->ac_o_ex.fe_logical & fe_len should always lie within "start" and
4558 if (start + size <= ac->ac_o_ex.fe_logical ||
4559 start > ac->ac_o_ex.fe_logical) {
4560 ext4_msg(ac->ac_sb, KERN_ERR,
4563 (unsigned long) ac->ac_o_ex.fe_logical);
4566 BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));
4572 ac->ac_g_ex.fe_logical = start;
4573 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);
4574 ac->ac_orig_goal_len = ac->ac_g_ex.fe_len;
4581 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,
4582 &ac->ac_g_ex.fe_group,
4583 &ac->ac_g_ex.fe_start);
4584 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
4589 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,
4590 &ac->ac_g_ex.fe_group,
4591 &ac->ac_g_ex.fe_start);
4592 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
4595 mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size,
4599 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac)
4601 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4603 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) {
4605 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);
4606 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)
4609 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);
4611 atomic_add(ac->ac_cX_found[i], &sbi->s_bal_cX_ex_scanned[i]);
4614 atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned);
4615 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&
4616 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)
4619 if (ac->ac_f_ex.fe_len == ac->ac_orig_goal_len)
4622 if (ac->ac_found > sbi->s_mb_max_to_scan)
4626 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)
4627 trace_ext4_mballoc_alloc(ac);
4629 trace_ext4_mballoc_prealloc(ac);
4636 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
4638 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac)
4640 struct ext4_prealloc_space *pa = ac->ac_pa;
4645 if (ac->ac_f_ex.fe_len == 0)
4647 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);
4656 ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group);
4657 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,
4658 ac->ac_f_ex.fe_len);
4659 ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group);
4665 pa->pa_free += ac->ac_b_ex.fe_len;
4673 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac,
4676 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4682 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);
4684 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));
4686 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,
4687 &ac->ac_b_ex.fe_start);
4688 ac->ac_b_ex.fe_len = len;
4689 ac->ac_status = AC_STATUS_FOUND;
4690 ac->ac_pa = pa;
4695 BUG_ON(ac->ac_b_ex.fe_len <= 0);
4698 mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa);
4704 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac,
4707 unsigned int len = ac->ac_o_ex.fe_len;
4709 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,
4710 &ac->ac_b_ex.fe_group,
4711 &ac->ac_b_ex.fe_start);
4712 ac->ac_b_ex.fe_len = len;
4713 ac->ac_status = AC_STATUS_FOUND;
4714 ac->ac_pa = pa;
4722 mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n",
4759 ext4_mb_pa_goal_check(struct ext4_allocation_context *ac,
4762 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4765 if (likely(!(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY)))
4775 (ac->ac_g_ex.fe_logical - pa->pa_lstart);
4776 if (ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex) != start)
4779 if (ac->ac_g_ex.fe_len > pa->pa_len -
4780 EXT4_B2C(sbi, ac->ac_g_ex.fe_logical - pa->pa_lstart))
4790 ext4_mb_use_preallocated(struct ext4_allocation_context *ac)
4792 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4794 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
4801 if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
4824 iter = ext4_mb_pa_rb_next_iter(ac->ac_o_ex.fe_logical,
4835 if (tmp_pa->pa_lstart > ac->ac_o_ex.fe_logical) {
4852 BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical));
4884 BUG_ON(!(tmp_pa && tmp_pa->pa_lstart <= ac->ac_o_ex.fe_logical));
4892 if (ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, tmp_pa)) {
4898 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&
4909 if (tmp_pa->pa_free && likely(ext4_mb_pa_goal_check(ac, tmp_pa))) {
4911 ext4_mb_use_inode_pa(ac, tmp_pa);
4950 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))
4954 lg = ac->ac_lg;
4957 order = fls(ac->ac_o_ex.fe_len) - 1;
4962 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);
4973 tmp_pa->pa_free >= ac->ac_o_ex.fe_len) {
4983 ext4_mb_use_group_pa(ac, cpa);
5073 static void ext4_mb_put_pa(struct ext4_allocation_context *ac,
5078 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
5165 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac)
5167 struct super_block *sb = ac->ac_sb;
5174 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);
5175 BUG_ON(ac->ac_status != AC_STATUS_FOUND);
5176 BUG_ON(!S_ISREG(ac->ac_inode->i_mode));
5177 BUG_ON(ac->ac_pa == NULL);
5179 pa = ac->ac_pa;
5181 if (ac->ac_b_ex.fe_len < ac->ac_orig_goal_len) {
5183 .fe_logical = ac->ac_g_ex.fe_logical,
5184 .fe_len = ac->ac_orig_goal_len,
5187 loff_t o_ex_end = extent_logical_end(sbi, &ac->ac_o_ex);
5197 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);
5198 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);
5211 ex.fe_len = ac->ac_b_ex.fe_len;
5214 if (ac->ac_o_ex.fe_logical >= ex.fe_logical)
5217 ex.fe_logical = ac->ac_g_ex.fe_logical;
5221 ex.fe_logical = ac->ac_o_ex.fe_logical;
5223 ac->ac_b_ex.fe_logical = ex.fe_logical;
5225 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);
5229 pa->pa_lstart = ac->ac_b_ex.fe_logical;
5230 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
5231 pa->pa_len = ac->ac_b_ex.fe_len;
5240 trace_ext4_mb_new_inode_pa(ac, pa);
5243 ext4_mb_use_inode_pa(ac, pa);
5245 ei = EXT4_I(ac->ac_inode);
5246 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
5251 pa->pa_inode = ac->ac_inode;
5265 ext4_mb_new_group_pa(struct ext4_allocation_context *ac)
5267 struct super_block *sb = ac->ac_sb;
5273 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);
5274 BUG_ON(ac->ac_status != AC_STATUS_FOUND);
5275 BUG_ON(!S_ISREG(ac->ac_inode->i_mode));
5276 BUG_ON(ac->ac_pa == NULL);
5278 pa = ac->ac_pa;
5280 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
5282 pa->pa_len = ac->ac_b_ex.fe_len;
5292 trace_ext4_mb_new_group_pa(ac, pa);
5294 ext4_mb_use_group_pa(ac, pa);
5297 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
5300 lg = ac->ac_lg;
5314 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac)
5316 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)
5317 ext4_mb_new_group_pa(ac);
5319 ext4_mb_new_inode_pa(ac);
5638 static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac)
5647 ac->ac_pa = pa;
5651 static void ext4_mb_pa_put_free(struct ext4_allocation_context *ac)
5653 struct ext4_prealloc_space *pa = ac->ac_pa;
5656 ac->ac_pa = NULL;
5702 static void ext4_mb_show_ac(struct ext4_allocation_context *ac)
5704 struct super_block *sb = ac->ac_sb;
5712 ac->ac_status, ac->ac_flags);
5716 (unsigned long)ac->ac_o_ex.fe_group,
5717 (unsigned long)ac->ac_o_ex.fe_start,
5718 (unsigned long)ac->ac_o_ex.fe_len,
5719 (unsigned long)ac->ac_o_ex.fe_logical,
5720 (unsigned long)ac->ac_g_ex.fe_group,
5721 (unsigned long)ac->ac_g_ex.fe_start,
5722 (unsigned long)ac->ac_g_ex.fe_len,
5723 (unsigned long)ac->ac_g_ex.fe_logical,
5724 (unsigned long)ac->ac_b_ex.fe_group,
5725 (unsigned long)ac->ac_b_ex.fe_start,
5726 (unsigned long)ac->ac_b_ex.fe_len,
5727 (unsigned long)ac->ac_b_ex.fe_logical,
5728 (int)ac->ac_criteria);
5729 mb_debug(sb, "%u found", ac->ac_found);
5730 mb_debug(sb, "used pa: %s, ", ac->ac_pa ? "yes" : "no");
5731 if (ac->ac_pa)
5732 mb_debug(sb, "pa_type %s\n", ac->ac_pa->pa_type == MB_GROUP_PA ?
5740 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac)
5742 ext4_mb_show_pa(ac->ac_sb);
5753 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac)
5755 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
5756 int bsbits = ac->ac_sb->s_blocksize_bits;
5760 if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
5763 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
5768 size = extent_logical_end(sbi, &ac->ac_o_ex);
5769 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1)
5774 !inode_is_open_for_write(ac->ac_inode))
5784 ac->ac_flags |= EXT4_MB_STREAM_ALLOC;
5786 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC;
5790 BUG_ON(ac->ac_lg != NULL);
5796 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups);
5799 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;
5802 mutex_lock(&ac->ac_lg->lg_mutex);
5806 ext4_mb_initialize_context(struct ext4_allocation_context *ac,
5832 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical);
5833 ac->ac_status = AC_STATUS_CONTINUE;
5834 ac->ac_sb = sb;
5835 ac->ac_inode = ar->inode;
5836 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical;
5837 ac->ac_o_ex.fe_group = group;
5838 ac->ac_o_ex.fe_start = block;
5839 ac->ac_o_ex.fe_len = len;
5840 ac->ac_g_ex = ac->ac_o_ex;
5841 ac->ac_orig_goal_len = ac->ac_g_ex.fe_len;
5842 ac->ac_flags = ar->flags;
5846 ext4_mb_group_or_file(ac);
5848 mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, "
5851 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order,
5941 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac)
5944 struct super_block *sb = ac->ac_sb;
5945 struct ext4_locality_group *lg = ac->ac_lg;
5946 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa;
5989 static int ext4_mb_release_context(struct ext4_allocation_context *ac)
5991 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
5992 struct ext4_prealloc_space *pa = ac->ac_pa;
5997 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
5998 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
5999 pa->pa_free -= ac->ac_b_ex.fe_len;
6000 pa->pa_len -= ac->ac_b_ex.fe_len;
6013 ext4_mb_add_n_trim(ac);
6017 ext4_mb_put_pa(ac, ac->ac_sb, pa);
6019 if (ac->ac_bitmap_page)
6020 put_page(ac->ac_bitmap_page);
6021 if (ac->ac_buddy_page)
6022 put_page(ac->ac_buddy_page);
6023 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)
6024 mutex_unlock(&ac->ac_lg->lg_mutex);
6025 ext4_mb_collect_stats(ac);
6057 struct ext4_allocation_context *ac, u64 *seq)
6063 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);
6069 if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) {
6070 ac->ac_flags |= EXT4_MB_STRICT_CHECK;
6155 struct ext4_allocation_context *ac = NULL;
6213 ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS);
6214 if (!ac) {
6220 ext4_mb_initialize_context(ac, ar);
6222 ac->ac_op = EXT4_MB_HISTORY_PREALLOC;
6224 if (!ext4_mb_use_preallocated(ac)) {
6225 ac->ac_op = EXT4_MB_HISTORY_ALLOC;
6226 ext4_mb_normalize_request(ac, ar);
6228 *errp = ext4_mb_pa_alloc(ac);
6233 *errp = ext4_mb_regular_allocator(ac);
6237 * ac->ac_status == AC_STATUS_FOUND.
6238 * And error from above mean ac->ac_status != AC_STATUS_FOUND
6242 ext4_mb_pa_put_free(ac);
6243 ext4_discard_allocated_blocks(ac);
6246 if (ac->ac_status == AC_STATUS_FOUND &&
6247 ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len)
6248 ext4_mb_pa_put_free(ac);
6250 if (likely(ac->ac_status == AC_STATUS_FOUND)) {
6251 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs);
6253 ext4_discard_allocated_blocks(ac);
6256 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
6257 ar->len = ac->ac_b_ex.fe_len;
6261 ext4_mb_discard_preallocations_should_retry(sb, ac, &seq))
6267 ext4_mb_pa_put_free(ac);
6273 ac->ac_b_ex.fe_len = 0;
6275 ext4_mb_show_ac(ac);
6277 ext4_mb_release_context(ac);
6278 kmem_cache_free(ext4_ac_cachep, ac);