Lines Matching refs:e4b
426 struct ext4_buddy *e4b, ext4_grpblk_t start,
526 static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max)
530 BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);
533 if (order > e4b->bd_blkbits + 1) {
540 *max = 1 << (e4b->bd_blkbits + 3);
541 return e4b->bd_bitmap;
544 bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];
545 *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];
551 static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b,
555 struct super_block *sb = e4b->bd_sb;
557 if (unlikely(e4b->bd_info->bb_bitmap == NULL))
559 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
561 if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {
564 blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
566 ext4_grp_locked_error(sb, e4b->bd_group,
572 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
575 mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);
579 static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count)
583 if (unlikely(e4b->bd_info->bb_bitmap == NULL))
585 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
587 BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));
588 mb_set_bit(first + i, e4b->bd_info->bb_bitmap);
592 static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
594 if (unlikely(e4b->bd_info->bb_bitmap == NULL))
596 if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {
599 b1 = (unsigned char *) e4b->bd_info->bb_bitmap;
601 for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {
603 ext4_msg(e4b->bd_sb, KERN_ERR,
607 e4b->bd_group, i, i * 8, b1[i], b2[i]);
641 struct ext4_buddy *e4b, int first, int count)
645 static inline void mb_mark_used_double(struct ext4_buddy *e4b,
650 static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
679 static int __mb_check_buddy(struct ext4_buddy *e4b, char *file,
682 struct super_block *sb = e4b->bd_sb;
683 int order = e4b->bd_blkbits + 1;
697 if (e4b->bd_info->bb_check_counter++ % 10)
701 buddy = mb_find_buddy(e4b, order, &max);
703 buddy2 = mb_find_buddy(e4b, order - 1, &max2);
727 !mb_test_bit(k, e4b->bd_bitmap));
731 MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);
736 buddy = mb_find_buddy(e4b, 0, &max);
739 MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);
748 for (j = 0; j < e4b->bd_blkbits + 1; j++) {
749 buddy2 = mb_find_buddy(e4b, j, &max2);
755 MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));
756 MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);
758 grp = ext4_get_group_info(sb, e4b->bd_group);
766 MB_CHECK_ASSERT(groupnr == e4b->bd_group);
773 #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \
776 #define mb_check_buddy(e4b)
1235 static void mb_regenerate_buddy(struct ext4_buddy *e4b)
1241 while ((buddy = mb_find_buddy(e4b, order++, &count)))
1244 e4b->bd_info->bb_fragments = 0;
1245 memset(e4b->bd_info->bb_counters, 0,
1246 sizeof(*e4b->bd_info->bb_counters) *
1247 (e4b->bd_sb->s_blocksize_bits + 2));
1249 ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy,
1250 e4b->bd_bitmap, e4b->bd_group, e4b->bd_info);
1441 * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap
1442 * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
1445 ext4_group_t group, struct ext4_buddy *e4b, gfp_t gfp)
1452 e4b->bd_buddy_page = NULL;
1453 e4b->bd_bitmap_page = NULL;
1468 e4b->bd_bitmap_page = page;
1469 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
1482 e4b->bd_buddy_page = page;
1486 static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b)
1488 if (e4b->bd_bitmap_page) {
1489 unlock_page(e4b->bd_bitmap_page);
1490 put_page(e4b->bd_bitmap_page);
1492 if (e4b->bd_buddy_page) {
1493 unlock_page(e4b->bd_buddy_page);
1494 put_page(e4b->bd_buddy_page);
1508 struct ext4_buddy e4b;
1527 ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b, gfp);
1536 page = e4b.bd_bitmap_page;
1545 if (e4b.bd_buddy_page == NULL) {
1555 page = e4b.bd_buddy_page;
1556 ret = ext4_mb_init_cache(page, e4b.bd_bitmap, gfp);
1564 ext4_mb_put_buddy_page_lock(&e4b);
1575 struct ext4_buddy *e4b, gfp_t gfp)
1595 e4b->bd_blkbits = sb->s_blocksize_bits;
1596 e4b->bd_info = grp;
1597 e4b->bd_sb = sb;
1598 e4b->bd_group = group;
1599 e4b->bd_buddy_page = NULL;
1600 e4b->bd_bitmap_page = NULL;
1650 mb_cmp_bitmaps(e4b, page_address(page) +
1666 e4b->bd_bitmap_page = page;
1667 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
1687 ret = ext4_mb_init_cache(page, e4b->bd_bitmap,
1707 e4b->bd_buddy_page = page;
1708 e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);
1715 if (e4b->bd_bitmap_page)
1716 put_page(e4b->bd_bitmap_page);
1718 e4b->bd_buddy = NULL;
1719 e4b->bd_bitmap = NULL;
1724 struct ext4_buddy *e4b)
1726 return ext4_mb_load_buddy_gfp(sb, group, e4b, GFP_NOFS);
1729 static void ext4_mb_unload_buddy(struct ext4_buddy *e4b)
1731 if (e4b->bd_bitmap_page)
1732 put_page(e4b->bd_bitmap_page);
1733 if (e4b->bd_buddy_page)
1734 put_page(e4b->bd_buddy_page);
1738 static int mb_find_order_for_block(struct ext4_buddy *e4b, int block)
1743 BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);
1744 BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));
1746 while (order <= e4b->bd_blkbits + 1) {
1747 bb = mb_find_buddy(e4b, order, &max);
1834 static void mb_buddy_mark_free(struct ext4_buddy *e4b, int first, int last)
1838 void *buddy = mb_find_buddy(e4b, order, &max);
1873 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1);
1875 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1);
1880 buddy2 = mb_find_buddy(e4b, order, &max);
1883 e4b->bd_info->bb_counters[order - 1] += last - first + 1;
1892 static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b,
1899 struct super_block *sb = e4b->bd_sb;
1904 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
1906 if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))
1909 mb_check_buddy(e4b);
1910 mb_free_blocks_double(inode, e4b, first, count);
1916 left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap);
1917 block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count);
1919 right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap);
1930 mb_regenerate_buddy(e4b);
1934 blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
1936 ext4_grp_locked_error(sb, e4b->bd_group,
1940 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
1946 e4b->bd_info->bb_free += count;
1947 if (first < e4b->bd_info->bb_first_free)
1948 e4b->bd_info->bb_first_free = first;
1952 e4b->bd_info->bb_fragments--;
1954 e4b->bd_info->bb_fragments++;
1964 e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1;
1968 e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1;
1972 mb_buddy_mark_free(e4b, first >> 1, last >> 1);
1974 mb_set_largest_free_order(sb, e4b->bd_info);
1975 mb_update_avg_fragment_size(sb, e4b->bd_info);
1977 mb_check_buddy(e4b);
1980 static int mb_find_extent(struct ext4_buddy *e4b, int block,
1987 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
1990 buddy = mb_find_buddy(e4b, 0, &max);
2001 order = mb_find_order_for_block(e4b, block);
2006 ex->fe_group = e4b->bd_group;
2014 mb_find_buddy(e4b, order, &max)) {
2020 if (mb_test_bit(next, e4b->bd_bitmap))
2023 order = mb_find_order_for_block(e4b, next);
2029 if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) {
2032 ext4_grp_locked_error(e4b->bd_sb, e4b->bd_group, 0, 0,
2044 static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex)
2057 BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));
2058 BUG_ON(e4b->bd_group != ex->fe_group);
2059 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
2060 mb_check_buddy(e4b);
2061 mb_mark_used_double(e4b, start, len);
2064 e4b->bd_info->bb_free -= len;
2065 if (e4b->bd_info->bb_first_free == start)
2066 e4b->bd_info->bb_first_free += len;
2070 mlen = !mb_test_bit(start - 1, e4b->bd_bitmap);
2071 if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])
2072 max = !mb_test_bit(start + len, e4b->bd_bitmap);
2074 e4b->bd_info->bb_fragments++;
2076 e4b->bd_info->bb_fragments--;
2081 ord = mb_find_order_for_block(e4b, start);
2087 buddy = mb_find_buddy(e4b, ord, &max);
2092 e4b->bd_info->bb_counters[ord]--;
2105 buddy = mb_find_buddy(e4b, ord, &max);
2107 e4b->bd_info->bb_counters[ord]--;
2111 buddy = mb_find_buddy(e4b, ord, &max);
2114 e4b->bd_info->bb_counters[ord]++;
2115 e4b->bd_info->bb_counters[ord]++;
2118 mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info);
2120 mb_update_avg_fragment_size(e4b->bd_sb, e4b->bd_info);
2121 mb_set_bits(e4b->bd_bitmap, ex->fe_start, len0);
2122 mb_check_buddy(e4b);
2131 struct ext4_buddy *e4b)
2136 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);
2141 ret = mb_mark_used(e4b, &ac->ac_b_ex);
2158 ac->ac_bitmap_page = e4b->bd_bitmap_page;
2160 ac->ac_buddy_page = e4b->bd_buddy_page;
2180 struct ext4_buddy *e4b,
2205 ext4_mb_use_best_found(ac, e4b);
2234 struct ext4_buddy *e4b)
2252 ext4_mb_use_best_found(ac, e4b);
2261 ext4_mb_use_best_found(ac, e4b);
2289 ext4_mb_check_limits(ac, e4b, 0);
2294 struct ext4_buddy *e4b)
2302 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
2307 if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))
2310 max = mb_find_extent(e4b, ex.fe_start, ex.fe_len, &ex);
2314 ext4_mb_use_best_found(ac, e4b);
2319 ext4_mb_unload_buddy(e4b);
2324 struct ext4_buddy *e4b)
2340 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
2345 if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))
2348 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,
2361 ext4_mb_use_best_found(ac, e4b);
2369 ext4_mb_use_best_found(ac, e4b);
2378 ext4_mb_use_best_found(ac, e4b);
2382 ext4_mb_unload_buddy(e4b);
2393 struct ext4_buddy *e4b)
2396 struct ext4_group_info *grp = e4b->bd_info;
2407 buddy = mb_find_buddy(e4b, i, &max);
2414 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0,
2418 e4b->bd_group,
2427 ac->ac_b_ex.fe_group = e4b->bd_group;
2429 ext4_mb_use_best_found(ac, e4b);
2447 struct ext4_buddy *e4b)
2450 void *bitmap = e4b->bd_bitmap;
2455 free = e4b->bd_info->bb_free;
2459 i = e4b->bd_info->bb_first_free;
2470 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
2474 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
2497 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);
2501 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
2505 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
2515 ext4_mb_measure_extent(ac, &ex, e4b);
2521 ext4_mb_check_limits(ac, e4b, 1);
2530 struct ext4_buddy *e4b)
2534 void *bitmap = e4b->bd_bitmap;
2544 first_group_block = ext4_group_first_block_no(sb, e4b->bd_group);
2554 max = mb_find_extent(e4b, i, stripe, &ex);
2560 ext4_mb_use_best_found(ac, e4b);
2799 struct ext4_buddy e4b;
2812 err = ext4_mb_find_by_goal(ac, &e4b);
2903 err = ext4_mb_load_buddy(sb, group, &e4b);
2916 ext4_mb_unload_buddy(&e4b);
2922 ext4_mb_simple_scan_group(ac, &e4b);
2928 ext4_mb_scan_aligned(ac, &e4b);
2930 ext4_mb_complex_scan_group(ac, &e4b);
2933 ext4_mb_unload_buddy(&e4b);
2954 ext4_mb_try_best_found(ac, &e4b);
3021 struct ext4_buddy e4b;
3045 err = ext4_mb_load_buddy(sb, group, &e4b);
3056 ext4_mb_unload_buddy(&e4b);
3531 struct ext4_buddy e4b;
3551 ext4_mb_unload_buddy(&e4b);
3553 err = ext4_mb_load_buddy(sb, grp, &e4b);
3564 ext4_try_to_trim_range(sb, &e4b, fd->efd_start_cluster,
3572 ext4_mb_unload_buddy(&e4b);
3856 struct ext4_buddy e4b;
3863 err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);
3871 db = e4b.bd_info;
3877 mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count);
3892 put_page(e4b.bd_buddy_page);
3893 put_page(e4b.bd_bitmap_page);
3896 ext4_mb_unload_buddy(&e4b);
4641 struct ext4_buddy e4b;
4647 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);
4657 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,
4660 ext4_mb_unload_buddy(&e4b);
5331 ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh,
5334 struct super_block *sb = e4b->bd_sb;
5346 BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
5363 mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);
5367 ext4_msg(e4b->bd_sb, KERN_CRIT,
5385 ext4_mb_release_group_pa(struct ext4_buddy *e4b,
5388 struct super_block *sb = e4b->bd_sb;
5395 if (unlikely(group != e4b->bd_group && pa->pa_len != 0)) {
5397 e4b->bd_group, group, pa->pa_pstart);
5400 mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);
5424 struct ext4_buddy e4b;
5444 err = ext4_mb_load_buddy(sb, group, &e4b);
5499 ext4_mb_release_group_pa(&e4b, pa);
5502 ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
5508 ext4_mb_unload_buddy(&e4b);
5533 struct ext4_buddy e4b;
5608 err = ext4_mb_load_buddy_gfp(sb, group, &e4b,
5621 ext4_mb_unload_buddy(&e4b);
5627 ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
5630 ext4_mb_unload_buddy(&e4b);
5863 struct ext4_buddy e4b;
5914 err = ext4_mb_load_buddy_gfp(sb, group, &e4b,
5923 ext4_mb_release_group_pa(&e4b, pa);
5926 ext4_mb_unload_buddy(&e4b);
6324 ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b,
6327 ext4_group_t group = e4b->bd_group;
6331 struct ext4_group_info *db = e4b->bd_info;
6332 struct super_block *sb = e4b->bd_sb;
6338 BUG_ON(e4b->bd_bitmap_page == NULL);
6339 BUG_ON(e4b->bd_buddy_page == NULL);
6350 get_page(e4b->bd_buddy_page);
6351 get_page(e4b->bd_bitmap_page);
6458 struct ext4_buddy e4b;
6539 err = ext4_mb_load_buddy_gfp(sb, block_group, &e4b,
6567 ext4_mb_free_metadata(handle, &e4b, new_entry);
6582 EXT4_MB_GRP_CLEAR_TRIMMED(e4b.bd_info);
6586 mb_free_blocks(inode, &e4b, bit, count_clusters);
6614 ext4_mb_unload_buddy(&e4b);
6760 struct ext4_buddy e4b;
6832 err = ext4_mb_load_buddy(sb, block_group, &e4b);
6843 mb_free_blocks(NULL, &e4b, bit, cluster_count);
6860 ext4_mb_unload_buddy(&e4b);
6883 * @e4b: ext4 buddy for the group
6890 int start, int count, struct ext4_buddy *e4b)
6895 ext4_group_t group = e4b->bd_group;
6910 mb_mark_used(e4b, &ex);
6914 mb_free_blocks(NULL, e4b, start, ex.fe_len);
6939 struct ext4_buddy *e4b, ext4_grpblk_t start,
6941 __acquires(ext4_group_lock_ptr(sb, e4b->bd_group))
6942 __releases(ext4_group_lock_ptr(sb, e4b->bd_group))
6948 last = ext4_last_grp_cluster(sb, e4b->bd_group);
6949 bitmap = e4b->bd_bitmap;
6953 start = max(e4b->bd_info->bb_first_free, start);
6967 int ret = ext4_trim_extent(sb, start, next - start, e4b);
6980 ext4_unlock_group(sb, e4b->bd_group);
6982 ext4_lock_group(sb, e4b->bd_group);
6985 if ((e4b->bd_info->bb_free - free_count) < minblocks)
6990 EXT4_MB_GRP_SET_TRIMMED(e4b->bd_info);
7013 struct ext4_buddy e4b;
7018 ret = ext4_mb_load_buddy(sb, group, &e4b);
7027 if (!EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) ||
7029 ret = ext4_try_to_trim_range(sb, &e4b, start, max, minblocks);
7034 ext4_mb_unload_buddy(&e4b);
7156 struct ext4_buddy e4b;
7159 error = ext4_mb_load_buddy(sb, group, &e4b);
7162 bitmap = e4b.bd_bitmap;
7166 start = max(e4b.bd_info->bb_first_free, start);
7187 ext4_mb_unload_buddy(&e4b);