Lines Matching refs:ac
353 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac);
1704 static void ext4_mb_use_best_found(struct ext4_allocation_context *ac,
1707 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
1710 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);
1711 BUG_ON(ac->ac_status == AC_STATUS_FOUND);
1713 ac->ac_b_ex.fe_len = min(ac->ac_b_ex.fe_len, ac->ac_g_ex.fe_len);
1714 ac->ac_b_ex.fe_logical = ac->ac_g_ex.fe_logical;
1715 ret = mb_mark_used(e4b, &ac->ac_b_ex);
1719 ac->ac_f_ex = ac->ac_b_ex;
1721 ac->ac_status = AC_STATUS_FOUND;
1722 ac->ac_tail = ret & 0xffff;
1723 ac->ac_buddy = ret >> 16;
1732 ac->ac_bitmap_page = e4b->bd_bitmap_page;
1733 get_page(ac->ac_bitmap_page);
1734 ac->ac_buddy_page = e4b->bd_buddy_page;
1735 get_page(ac->ac_buddy_page);
1737 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
1739 sbi->s_mb_last_group = ac->ac_f_ex.fe_group;
1740 sbi->s_mb_last_start = ac->ac_f_ex.fe_start;
1748 if (ac->ac_o_ex.fe_len < ac->ac_b_ex.fe_len)
1749 ext4_mb_new_preallocation(ac);
1753 static void ext4_mb_check_limits(struct ext4_allocation_context *ac,
1757 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
1758 struct ext4_free_extent *bex = &ac->ac_b_ex;
1759 struct ext4_free_extent *gex = &ac->ac_g_ex;
1763 if (ac->ac_status == AC_STATUS_FOUND)
1768 if (ac->ac_found > sbi->s_mb_max_to_scan &&
1769 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
1770 ac->ac_status = AC_STATUS_BREAK;
1780 if ((finish_group || ac->ac_found > sbi->s_mb_min_to_scan)
1787 ext4_mb_use_best_found(ac, e4b);
1803 static void ext4_mb_measure_extent(struct ext4_allocation_context *ac,
1807 struct ext4_free_extent *bex = &ac->ac_b_ex;
1808 struct ext4_free_extent *gex = &ac->ac_g_ex;
1811 BUG_ON(ex->fe_len > EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));
1812 BUG_ON(ex->fe_start >= EXT4_CLUSTERS_PER_GROUP(ac->ac_sb));
1813 BUG_ON(ac->ac_status != AC_STATUS_CONTINUE);
1815 ac->ac_found++;
1820 if (unlikely(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
1822 ext4_mb_use_best_found(ac, e4b);
1831 ext4_mb_use_best_found(ac, e4b);
1859 ext4_mb_check_limits(ac, e4b, 0);
1863 int ext4_mb_try_best_found(struct ext4_allocation_context *ac,
1866 struct ext4_free_extent ex = ac->ac_b_ex;
1872 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
1876 ext4_lock_group(ac->ac_sb, group);
1883 ac->ac_b_ex = ex;
1884 ext4_mb_use_best_found(ac, e4b);
1888 ext4_unlock_group(ac->ac_sb, group);
1895 int ext4_mb_find_by_goal(struct ext4_allocation_context *ac,
1898 ext4_group_t group = ac->ac_g_ex.fe_group;
1901 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
1902 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
1907 if (!(ac->ac_flags & (EXT4_MB_HINT_TRY_GOAL | EXT4_MB_HINT_GOAL_ONLY)))
1912 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
1916 ext4_lock_group(ac->ac_sb, group);
1920 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,
1921 ac->ac_g_ex.fe_len, &ex);
1924 if (max >= ac->ac_g_ex.fe_len && ac->ac_g_ex.fe_len == sbi->s_stripe) {
1927 start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) +
1931 ac->ac_found++;
1932 ac->ac_b_ex = ex;
1933 ext4_mb_use_best_found(ac, e4b);
1935 } else if (max >= ac->ac_g_ex.fe_len) {
1937 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
1938 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
1939 ac->ac_found++;
1940 ac->ac_b_ex = ex;
1941 ext4_mb_use_best_found(ac, e4b);
1942 } else if (max > 0 && (ac->ac_flags & EXT4_MB_HINT_MERGE)) {
1946 BUG_ON(ex.fe_group != ac->ac_g_ex.fe_group);
1947 BUG_ON(ex.fe_start != ac->ac_g_ex.fe_start);
1948 ac->ac_found++;
1949 ac->ac_b_ex = ex;
1950 ext4_mb_use_best_found(ac, e4b);
1953 ext4_unlock_group(ac->ac_sb, group);
1964 void ext4_mb_simple_scan_group(struct ext4_allocation_context *ac,
1967 struct super_block *sb = ac->ac_sb;
1974 BUG_ON(ac->ac_2order <= 0);
1975 for (i = ac->ac_2order; i <= sb->s_blocksize_bits + 1; i++) {
1984 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0,
1987 ext4_mark_group_bitmap_corrupted(ac->ac_sb,
1992 ac->ac_found++;
1994 ac->ac_b_ex.fe_len = 1 << i;
1995 ac->ac_b_ex.fe_start = k << i;
1996 ac->ac_b_ex.fe_group = e4b->bd_group;
1998 ext4_mb_use_best_found(ac, e4b);
2000 BUG_ON(ac->ac_f_ex.fe_len != ac->ac_g_ex.fe_len);
2015 void ext4_mb_complex_scan_group(struct ext4_allocation_context *ac,
2018 struct super_block *sb = ac->ac_sb;
2030 while (free && ac->ac_status == AC_STATUS_CONTINUE) {
2048 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);
2066 ext4_mb_measure_extent(ac, &ex, e4b);
2072 ext4_mb_check_limits(ac, e4b, 1);
2080 void ext4_mb_scan_aligned(struct ext4_allocation_context *ac,
2083 struct super_block *sb = ac->ac_sb;
2105 ac->ac_found++;
2107 ac->ac_b_ex = ex;
2108 ext4_mb_use_best_found(ac, e4b);
2121 static bool ext4_mb_good_group(struct ext4_allocation_context *ac,
2125 int flex_size = ext4_flex_bg_size(EXT4_SB(ac->ac_sb));
2126 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
2143 BUG_ON(ac->ac_2order == 0);
2146 if ((ac->ac_flags & EXT4_MB_HINT_DATA) &&
2151 if (free < ac->ac_g_ex.fe_len)
2154 if (ac->ac_2order > ac->ac_sb->s_blocksize_bits+1)
2157 if (grp->bb_largest_free_order < ac->ac_2order)
2162 if ((free / fragments) >= ac->ac_g_ex.fe_len)
2166 if (free >= ac->ac_g_ex.fe_len)
2183 static int ext4_mb_good_group_nolock(struct ext4_allocation_context *ac,
2186 struct ext4_group_info *grp = ext4_get_group_info(ac->ac_sb, group);
2187 struct super_block *sb = ac->ac_sb;
2189 bool should_lock = ac->ac_flags & EXT4_MB_STRICT_CHECK;
2196 atomic64_inc(&sbi->s_bal_cX_groups_considered[ac->ac_criteria]);
2202 if (cr <= 2 && free < ac->ac_g_ex.fe_len)
2236 ret = ext4_mb_good_group(ac, group, cr);
2322 ext4_mb_regular_allocator(struct ext4_allocation_context *ac)
2333 sb = ac->ac_sb;
2337 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)))
2340 BUG_ON(ac->ac_status == AC_STATUS_FOUND);
2343 err = ext4_mb_find_by_goal(ac, &e4b);
2344 if (err || ac->ac_status == AC_STATUS_FOUND)
2347 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
2351 * ac->ac_2order is set only if the fe_len is a power of 2
2352 * if ac->ac_2order is set we also set criteria to 0 so that we
2355 i = fls(ac->ac_g_ex.fe_len);
2356 ac->ac_2order = 0;
2368 if ((ac->ac_g_ex.fe_len & (~(1 << (i - 1)))) == 0)
2369 ac->ac_2order = array_index_nospec(i - 1,
2374 if (ac->ac_flags & EXT4_MB_STREAM_ALLOC) {
2377 ac->ac_g_ex.fe_group = sbi->s_mb_last_group;
2378 ac->ac_g_ex.fe_start = sbi->s_mb_last_start;
2383 cr = ac->ac_2order ? 0 : 1;
2389 for (; cr < 4 && ac->ac_status == AC_STATUS_CONTINUE; cr++) {
2390 ac->ac_criteria = cr;
2395 group = ac->ac_g_ex.fe_group;
2432 ret = ext4_mb_good_group_nolock(ac, group, cr);
2449 ret = ext4_mb_good_group(ac, group, cr);
2456 ac->ac_groups_scanned++;
2458 ext4_mb_simple_scan_group(ac, &e4b);
2460 !(ac->ac_g_ex.fe_len % sbi->s_stripe))
2461 ext4_mb_scan_aligned(ac, &e4b);
2463 ext4_mb_complex_scan_group(ac, &e4b);
2468 if (ac->ac_status != AC_STATUS_CONTINUE)
2476 if (ac->ac_b_ex.fe_len > 0 && ac->ac_status != AC_STATUS_FOUND &&
2477 !(ac->ac_flags & EXT4_MB_HINT_FIRST)) {
2482 ext4_mb_try_best_found(ac, &e4b);
2483 if (ac->ac_status != AC_STATUS_FOUND) {
2491 ac->ac_b_ex.fe_group, ac->ac_b_ex.fe_start,
2492 ac->ac_b_ex.fe_len, lost);
2494 ac->ac_b_ex.fe_group = 0;
2495 ac->ac_b_ex.fe_start = 0;
2496 ac->ac_b_ex.fe_len = 0;
2497 ac->ac_status = AC_STATUS_CONTINUE;
2498 ac->ac_flags |= EXT4_MB_HINT_FIRST;
2504 if (sbi->s_mb_stats && ac->ac_status == AC_STATUS_FOUND)
2505 atomic64_inc(&sbi->s_bal_cX_hits[ac->ac_criteria]);
2507 if (!err && ac->ac_status != AC_STATUS_FOUND && first_err)
2511 ac->ac_b_ex.fe_len, ac->ac_o_ex.fe_len, ac->ac_status,
2512 ac->ac_flags, cr, err);
3289 * Check quota and mark chosen space (ac->ac_b_ex) non-free in bitmaps
3293 ext4_mb_mark_diskspace_used(struct ext4_allocation_context *ac,
3304 BUG_ON(ac->ac_status != AC_STATUS_FOUND);
3305 BUG_ON(ac->ac_b_ex.fe_len <= 0);
3307 sb = ac->ac_sb;
3310 bitmap_bh = ext4_read_block_bitmap(sb, ac->ac_b_ex.fe_group);
3323 gdp = ext4_get_group_desc(sb, ac->ac_b_ex.fe_group, &gdp_bh);
3327 ext4_debug("using block group %u(%d)\n", ac->ac_b_ex.fe_group,
3335 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
3337 len = EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
3338 if (!ext4_inode_block_valid(ac->ac_inode, block, len)) {
3345 ext4_lock_group(sb, ac->ac_b_ex.fe_group);
3346 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,
3347 ac->ac_b_ex.fe_len);
3348 ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
3355 ext4_lock_group(sb, ac->ac_b_ex.fe_group);
3359 for (i = 0; i < ac->ac_b_ex.fe_len; i++) {
3360 BUG_ON(mb_test_bit(ac->ac_b_ex.fe_start + i,
3365 ext4_set_bits(bitmap_bh->b_data, ac->ac_b_ex.fe_start,
3366 ac->ac_b_ex.fe_len);
3372 ac->ac_b_ex.fe_group, gdp));
3374 len = ext4_free_group_clusters(sb, gdp) - ac->ac_b_ex.fe_len;
3376 ext4_block_bitmap_csum_set(sb, ac->ac_b_ex.fe_group, gdp, bitmap_bh);
3377 ext4_group_desc_csum_set(sb, ac->ac_b_ex.fe_group, gdp);
3379 ext4_unlock_group(sb, ac->ac_b_ex.fe_group);
3380 percpu_counter_sub(&sbi->s_freeclusters_counter, ac->ac_b_ex.fe_len);
3384 if (!(ac->ac_flags & EXT4_MB_DELALLOC_RESERVED))
3391 ac->ac_b_ex.fe_group);
3392 atomic64_sub(ac->ac_b_ex.fe_len,
3520 static void ext4_mb_normalize_group_request(struct ext4_allocation_context *ac)
3522 struct super_block *sb = ac->ac_sb;
3523 struct ext4_locality_group *lg = ac->ac_lg;
3526 ac->ac_g_ex.fe_len = EXT4_SB(sb)->s_mb_group_prealloc;
3527 mb_debug(sb, "goal %u blocks for locality group\n", ac->ac_g_ex.fe_len);
3535 ext4_mb_normalize_request(struct ext4_allocation_context *ac,
3538 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
3544 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
3549 if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
3553 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
3558 if (ac->ac_flags & EXT4_MB_HINT_NOPREALLOC)
3561 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC) {
3562 ext4_mb_normalize_group_request(ac);
3566 bsbits = ac->ac_sb->s_blocksize_bits;
3570 size = extent_logical_end(sbi, &ac->ac_o_ex);
3572 if (size < i_size_read(ac->ac_inode))
3573 size = i_size_read(ac->ac_inode);
3600 start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
3604 start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
3607 } else if (NRL_CHECK_SIZE(ac->ac_o_ex.fe_len,
3609 start_off = ((loff_t)ac->ac_o_ex.fe_logical >>
3613 start_off = (loff_t) ac->ac_o_ex.fe_logical << bsbits;
3614 size = (loff_t) EXT4_C2B(EXT4_SB(ac->ac_sb),
3615 ac->ac_o_ex.fe_len) << bsbits;
3626 start = max(start, rounddown(ac->ac_o_ex.fe_logical,
3627 (ext4_lblk_t)EXT4_BLOCKS_PER_GROUP(ac->ac_sb)));
3645 if (size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb))
3646 size = EXT4_BLOCKS_PER_GROUP(ac->ac_sb);
3663 pa_end = pa_logical_end(EXT4_SB(ac->ac_sb), pa);
3666 BUG_ON(!(ac->ac_o_ex.fe_logical >= pa_end ||
3667 ac->ac_o_ex.fe_logical < pa->pa_lstart));
3677 if (pa_end <= ac->ac_o_ex.fe_logical) {
3680 } else if (pa->pa_lstart > ac->ac_o_ex.fe_logical) {
3696 pa_end = pa_logical_end(EXT4_SB(ac->ac_sb), pa);
3703 if (start + size <= ac->ac_o_ex.fe_logical &&
3704 start > ac->ac_o_ex.fe_logical) {
3705 ext4_msg(ac->ac_sb, KERN_ERR,
3708 (unsigned long) ac->ac_o_ex.fe_logical);
3711 BUG_ON(size <= 0 || size > EXT4_BLOCKS_PER_GROUP(ac->ac_sb));
3717 ac->ac_g_ex.fe_logical = start;
3718 ac->ac_g_ex.fe_len = EXT4_NUM_B2C(sbi, size);
3725 ext4_get_group_no_and_offset(ac->ac_sb, ar->pright - size,
3726 &ac->ac_g_ex.fe_group,
3727 &ac->ac_g_ex.fe_start);
3728 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
3733 ext4_get_group_no_and_offset(ac->ac_sb, ar->pleft + 1,
3734 &ac->ac_g_ex.fe_group,
3735 &ac->ac_g_ex.fe_start);
3736 ac->ac_flags |= EXT4_MB_HINT_TRY_GOAL;
3739 mb_debug(ac->ac_sb, "goal: %lld(was %lld) blocks at %u\n", size,
3743 static void ext4_mb_collect_stats(struct ext4_allocation_context *ac)
3745 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
3747 if (sbi->s_mb_stats && ac->ac_g_ex.fe_len >= 1) {
3749 atomic_add(ac->ac_b_ex.fe_len, &sbi->s_bal_allocated);
3750 if (ac->ac_b_ex.fe_len >= ac->ac_o_ex.fe_len)
3752 atomic_add(ac->ac_found, &sbi->s_bal_ex_scanned);
3753 atomic_add(ac->ac_groups_scanned, &sbi->s_bal_groups_scanned);
3754 if (ac->ac_g_ex.fe_start == ac->ac_b_ex.fe_start &&
3755 ac->ac_g_ex.fe_group == ac->ac_b_ex.fe_group)
3757 if (ac->ac_found > sbi->s_mb_max_to_scan)
3761 if (ac->ac_op == EXT4_MB_HISTORY_ALLOC)
3762 trace_ext4_mballoc_alloc(ac);
3764 trace_ext4_mballoc_prealloc(ac);
3771 * zeroed out ac->ac_b_ex.fe_len, so group_pa->pa_free is not changed.
3773 static void ext4_discard_allocated_blocks(struct ext4_allocation_context *ac)
3775 struct ext4_prealloc_space *pa = ac->ac_pa;
3780 if (ac->ac_f_ex.fe_len == 0)
3782 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);
3792 ext4_lock_group(ac->ac_sb, ac->ac_f_ex.fe_group);
3793 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,
3794 ac->ac_f_ex.fe_len);
3795 ext4_unlock_group(ac->ac_sb, ac->ac_f_ex.fe_group);
3800 pa->pa_free += ac->ac_b_ex.fe_len;
3806 static void ext4_mb_use_inode_pa(struct ext4_allocation_context *ac,
3809 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
3815 start = pa->pa_pstart + (ac->ac_o_ex.fe_logical - pa->pa_lstart);
3817 start + EXT4_C2B(sbi, ac->ac_o_ex.fe_len));
3819 ext4_get_group_no_and_offset(ac->ac_sb, start, &ac->ac_b_ex.fe_group,
3820 &ac->ac_b_ex.fe_start);
3821 ac->ac_b_ex.fe_len = len;
3822 ac->ac_status = AC_STATUS_FOUND;
3823 ac->ac_pa = pa;
3828 BUG_ON(ac->ac_b_ex.fe_len <= 0);
3831 mb_debug(ac->ac_sb, "use %llu/%d from inode pa %p\n", start, len, pa);
3837 static void ext4_mb_use_group_pa(struct ext4_allocation_context *ac,
3840 unsigned int len = ac->ac_o_ex.fe_len;
3842 ext4_get_group_no_and_offset(ac->ac_sb, pa->pa_pstart,
3843 &ac->ac_b_ex.fe_group,
3844 &ac->ac_b_ex.fe_start);
3845 ac->ac_b_ex.fe_len = len;
3846 ac->ac_status = AC_STATUS_FOUND;
3847 ac->ac_pa = pa;
3855 mb_debug(ac->ac_sb, "use %u/%u from group pa %p\n",
3892 ext4_mb_use_preallocated(struct ext4_allocation_context *ac)
3894 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
3896 struct ext4_inode_info *ei = EXT4_I(ac->ac_inode);
3902 if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
3911 if (ac->ac_o_ex.fe_logical < pa->pa_lstart ||
3912 ac->ac_o_ex.fe_logical >= pa_logical_end(sbi, pa))
3916 if (!(ext4_test_inode_flag(ac->ac_inode, EXT4_INODE_EXTENTS)) &&
3925 ext4_mb_use_inode_pa(ac, pa);
3927 ac->ac_criteria = 10;
3936 if (!(ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC))
3940 lg = ac->ac_lg;
3943 order = fls(ac->ac_o_ex.fe_len) - 1;
3948 goal_block = ext4_grp_offs_to_block(ac->ac_sb, &ac->ac_g_ex);
3959 pa->pa_free >= ac->ac_o_ex.fe_len) {
3969 ext4_mb_use_group_pa(ac, cpa);
3970 ac->ac_criteria = 20;
4080 static void ext4_mb_put_pa(struct ext4_allocation_context *ac,
4140 ext4_mb_new_inode_pa(struct ext4_allocation_context *ac)
4142 struct super_block *sb = ac->ac_sb;
4149 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);
4150 BUG_ON(ac->ac_status != AC_STATUS_FOUND);
4151 BUG_ON(!S_ISREG(ac->ac_inode->i_mode));
4152 BUG_ON(ac->ac_pa == NULL);
4154 pa = ac->ac_pa;
4156 if (ac->ac_b_ex.fe_len < ac->ac_g_ex.fe_len) {
4158 .fe_logical = ac->ac_g_ex.fe_logical,
4159 .fe_len = ac->ac_g_ex.fe_len,
4166 BUG_ON(ac->ac_g_ex.fe_logical > ac->ac_o_ex.fe_logical);
4167 BUG_ON(ac->ac_g_ex.fe_len < ac->ac_o_ex.fe_len);
4180 ex.fe_len = ac->ac_b_ex.fe_len;
4183 if (ac->ac_o_ex.fe_logical >= ex.fe_logical)
4186 ex.fe_logical = ac->ac_g_ex.fe_logical;
4187 if (ac->ac_o_ex.fe_logical < extent_logical_end(sbi, &ex))
4190 ex.fe_logical = ac->ac_o_ex.fe_logical;
4192 ac->ac_b_ex.fe_logical = ex.fe_logical;
4194 BUG_ON(ac->ac_o_ex.fe_logical < ac->ac_b_ex.fe_logical);
4195 BUG_ON(ac->ac_o_ex.fe_len > ac->ac_b_ex.fe_len);
4201 ac->ac_f_ex = ac->ac_b_ex;
4203 pa->pa_lstart = ac->ac_b_ex.fe_logical;
4204 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
4205 pa->pa_len = ac->ac_b_ex.fe_len;
4215 trace_ext4_mb_new_inode_pa(ac, pa);
4217 ext4_mb_use_inode_pa(ac, pa);
4220 ei = EXT4_I(ac->ac_inode);
4221 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
4226 pa->pa_inode = ac->ac_inode;
4240 ext4_mb_new_group_pa(struct ext4_allocation_context *ac)
4242 struct super_block *sb = ac->ac_sb;
4248 BUG_ON(ac->ac_o_ex.fe_len >= ac->ac_b_ex.fe_len);
4249 BUG_ON(ac->ac_status != AC_STATUS_FOUND);
4250 BUG_ON(!S_ISREG(ac->ac_inode->i_mode));
4251 BUG_ON(ac->ac_pa == NULL);
4253 pa = ac->ac_pa;
4257 ac->ac_f_ex = ac->ac_b_ex;
4259 pa->pa_pstart = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
4261 pa->pa_len = ac->ac_b_ex.fe_len;
4271 trace_ext4_mb_new_group_pa(ac, pa);
4273 ext4_mb_use_group_pa(ac, pa);
4276 grp = ext4_get_group_info(sb, ac->ac_b_ex.fe_group);
4279 lg = ac->ac_lg;
4293 static void ext4_mb_new_preallocation(struct ext4_allocation_context *ac)
4295 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)
4296 ext4_mb_new_group_pa(ac);
4298 ext4_mb_new_inode_pa(ac);
4608 static int ext4_mb_pa_alloc(struct ext4_allocation_context *ac)
4617 ac->ac_pa = pa;
4621 static void ext4_mb_pa_free(struct ext4_allocation_context *ac)
4623 struct ext4_prealloc_space *pa = ac->ac_pa;
4626 ac->ac_pa = NULL;
4666 static void ext4_mb_show_ac(struct ext4_allocation_context *ac)
4668 struct super_block *sb = ac->ac_sb;
4676 ac->ac_status, ac->ac_flags);
4680 (unsigned long)ac->ac_o_ex.fe_group,
4681 (unsigned long)ac->ac_o_ex.fe_start,
4682 (unsigned long)ac->ac_o_ex.fe_len,
4683 (unsigned long)ac->ac_o_ex.fe_logical,
4684 (unsigned long)ac->ac_g_ex.fe_group,
4685 (unsigned long)ac->ac_g_ex.fe_start,
4686 (unsigned long)ac->ac_g_ex.fe_len,
4687 (unsigned long)ac->ac_g_ex.fe_logical,
4688 (unsigned long)ac->ac_b_ex.fe_group,
4689 (unsigned long)ac->ac_b_ex.fe_start,
4690 (unsigned long)ac->ac_b_ex.fe_len,
4691 (unsigned long)ac->ac_b_ex.fe_logical,
4692 (int)ac->ac_criteria);
4693 mb_debug(sb, "%u found", ac->ac_found);
4701 static inline void ext4_mb_show_ac(struct ext4_allocation_context *ac)
4703 ext4_mb_show_pa(ac->ac_sb);
4715 static void ext4_mb_group_or_file(struct ext4_allocation_context *ac)
4717 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4718 int bsbits = ac->ac_sb->s_blocksize_bits;
4721 if (!(ac->ac_flags & EXT4_MB_HINT_DATA))
4724 if (unlikely(ac->ac_flags & EXT4_MB_HINT_GOAL_ONLY))
4727 size = extent_logical_end(sbi, &ac->ac_o_ex);
4728 isize = (i_size_read(ac->ac_inode) + ac->ac_sb->s_blocksize - 1)
4732 !inode_is_open_for_write(ac->ac_inode)) {
4733 ac->ac_flags |= EXT4_MB_HINT_NOPREALLOC;
4738 ac->ac_flags |= EXT4_MB_STREAM_ALLOC;
4745 ac->ac_flags |= EXT4_MB_STREAM_ALLOC;
4749 BUG_ON(ac->ac_lg != NULL);
4755 ac->ac_lg = raw_cpu_ptr(sbi->s_locality_groups);
4758 ac->ac_flags |= EXT4_MB_HINT_GROUP_ALLOC;
4761 mutex_lock(&ac->ac_lg->lg_mutex);
4765 ext4_mb_initialize_context(struct ext4_allocation_context *ac,
4791 ac->ac_b_ex.fe_logical = EXT4_LBLK_CMASK(sbi, ar->logical);
4792 ac->ac_status = AC_STATUS_CONTINUE;
4793 ac->ac_sb = sb;
4794 ac->ac_inode = ar->inode;
4795 ac->ac_o_ex.fe_logical = ac->ac_b_ex.fe_logical;
4796 ac->ac_o_ex.fe_group = group;
4797 ac->ac_o_ex.fe_start = block;
4798 ac->ac_o_ex.fe_len = len;
4799 ac->ac_g_ex = ac->ac_o_ex;
4800 ac->ac_flags = ar->flags;
4804 ext4_mb_group_or_file(ac);
4806 mb_debug(sb, "init ac: %u blocks @ %u, goal %u, flags 0x%x, 2^%d, "
4809 (unsigned) ar->goal, ac->ac_flags, ac->ac_2order,
4903 static void ext4_mb_add_n_trim(struct ext4_allocation_context *ac)
4906 struct super_block *sb = ac->ac_sb;
4907 struct ext4_locality_group *lg = ac->ac_lg;
4908 struct ext4_prealloc_space *tmp_pa, *pa = ac->ac_pa;
4971 static int ext4_mb_release_context(struct ext4_allocation_context *ac)
4973 struct inode *inode = ac->ac_inode;
4975 struct ext4_sb_info *sbi = EXT4_SB(ac->ac_sb);
4976 struct ext4_prealloc_space *pa = ac->ac_pa;
4981 pa->pa_pstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
4982 pa->pa_lstart += EXT4_C2B(sbi, ac->ac_b_ex.fe_len);
4983 pa->pa_free -= ac->ac_b_ex.fe_len;
4984 pa->pa_len -= ac->ac_b_ex.fe_len;
4997 ext4_mb_add_n_trim(ac);
5011 ext4_mb_put_pa(ac, ac->ac_sb, pa);
5013 if (ac->ac_bitmap_page)
5014 put_page(ac->ac_bitmap_page);
5015 if (ac->ac_buddy_page)
5016 put_page(ac->ac_buddy_page);
5017 if (ac->ac_flags & EXT4_MB_HINT_GROUP_ALLOC)
5018 mutex_unlock(&ac->ac_lg->lg_mutex);
5019 ext4_mb_collect_stats(ac);
5052 struct ext4_allocation_context *ac, u64 *seq)
5058 freed = ext4_mb_discard_preallocations(sb, ac->ac_o_ex.fe_len);
5064 if (!(ac->ac_flags & EXT4_MB_STRICT_CHECK) || seq_retry != *seq) {
5065 ac->ac_flags |= EXT4_MB_STRICT_CHECK;
5086 struct ext4_allocation_context *ac = NULL;
5144 ac = kmem_cache_zalloc(ext4_ac_cachep, GFP_NOFS);
5145 if (!ac) {
5151 *errp = ext4_mb_initialize_context(ac, ar);
5157 ac->ac_op = EXT4_MB_HISTORY_PREALLOC;
5159 if (!ext4_mb_use_preallocated(ac)) {
5160 ac->ac_op = EXT4_MB_HISTORY_ALLOC;
5161 ext4_mb_normalize_request(ac, ar);
5163 *errp = ext4_mb_pa_alloc(ac);
5168 *errp = ext4_mb_regular_allocator(ac);
5172 * ac->ac_status == AC_STATUS_FOUND.
5173 * And error from above mean ac->ac_status != AC_STATUS_FOUND
5177 ext4_mb_pa_free(ac);
5178 ext4_discard_allocated_blocks(ac);
5181 if (ac->ac_status == AC_STATUS_FOUND &&
5182 ac->ac_o_ex.fe_len >= ac->ac_f_ex.fe_len)
5183 ext4_mb_pa_free(ac);
5185 if (likely(ac->ac_status == AC_STATUS_FOUND)) {
5186 *errp = ext4_mb_mark_diskspace_used(ac, handle, reserv_clstrs);
5188 ext4_discard_allocated_blocks(ac);
5191 block = ext4_grp_offs_to_block(sb, &ac->ac_b_ex);
5192 ar->len = ac->ac_b_ex.fe_len;
5196 ext4_mb_discard_preallocations_should_retry(sb, ac, &seq))
5202 ext4_mb_pa_free(ac);
5208 ac->ac_b_ex.fe_len = 0;
5210 ext4_mb_show_ac(ac);
5212 ext4_mb_release_context(ac);
5214 if (ac)
5215 kmem_cache_free(ext4_ac_cachep, ac);