Lines Matching refs:e4b
452 static void *mb_find_buddy(struct ext4_buddy *e4b, int order, int *max)
456 BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);
459 if (order > e4b->bd_blkbits + 1) {
466 *max = 1 << (e4b->bd_blkbits + 3);
467 return e4b->bd_bitmap;
470 bb = e4b->bd_buddy + EXT4_SB(e4b->bd_sb)->s_mb_offsets[order];
471 *max = EXT4_SB(e4b->bd_sb)->s_mb_maxs[order];
477 static void mb_free_blocks_double(struct inode *inode, struct ext4_buddy *e4b,
481 struct super_block *sb = e4b->bd_sb;
483 if (unlikely(e4b->bd_info->bb_bitmap == NULL))
485 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
487 if (!mb_test_bit(first + i, e4b->bd_info->bb_bitmap)) {
490 blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
492 ext4_grp_locked_error(sb, e4b->bd_group,
498 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
501 mb_clear_bit(first + i, e4b->bd_info->bb_bitmap);
505 static void mb_mark_used_double(struct ext4_buddy *e4b, int first, int count)
509 if (unlikely(e4b->bd_info->bb_bitmap == NULL))
511 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
513 BUG_ON(mb_test_bit(first + i, e4b->bd_info->bb_bitmap));
514 mb_set_bit(first + i, e4b->bd_info->bb_bitmap);
518 static void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
520 if (unlikely(e4b->bd_info->bb_bitmap == NULL))
522 if (memcmp(e4b->bd_info->bb_bitmap, bitmap, e4b->bd_sb->s_blocksize)) {
525 b1 = (unsigned char *) e4b->bd_info->bb_bitmap;
527 for (i = 0; i < e4b->bd_sb->s_blocksize; i++) {
529 ext4_msg(e4b->bd_sb, KERN_ERR,
533 e4b->bd_group, i, i * 8, b1[i], b2[i]);
567 struct ext4_buddy *e4b, int first, int count)
571 static inline void mb_mark_used_double(struct ext4_buddy *e4b,
576 static inline void mb_cmp_bitmaps(struct ext4_buddy *e4b, void *bitmap)
605 static int __mb_check_buddy(struct ext4_buddy *e4b, char *file,
608 struct super_block *sb = e4b->bd_sb;
609 int order = e4b->bd_blkbits + 1;
623 if (e4b->bd_info->bb_check_counter++ % 10)
627 buddy = mb_find_buddy(e4b, order, &max);
629 buddy2 = mb_find_buddy(e4b, order - 1, &max2);
656 !mb_test_bit(k, e4b->bd_bitmap));
660 MB_CHECK_ASSERT(e4b->bd_info->bb_counters[order] == count);
665 buddy = mb_find_buddy(e4b, 0, &max);
668 MB_CHECK_ASSERT(i >= e4b->bd_info->bb_first_free);
677 for (j = 0; j < e4b->bd_blkbits + 1; j++) {
678 buddy2 = mb_find_buddy(e4b, j, &max2);
684 MB_CHECK_ASSERT(!EXT4_MB_GRP_NEED_INIT(e4b->bd_info));
685 MB_CHECK_ASSERT(e4b->bd_info->bb_fragments == fragments);
687 grp = ext4_get_group_info(sb, e4b->bd_group);
695 MB_CHECK_ASSERT(groupnr == e4b->bd_group);
702 #define mb_check_buddy(e4b) __mb_check_buddy(e4b, \
705 #define mb_check_buddy(e4b)
826 static void mb_regenerate_buddy(struct ext4_buddy *e4b)
832 while ((buddy = mb_find_buddy(e4b, order++, &count)))
835 e4b->bd_info->bb_fragments = 0;
836 memset(e4b->bd_info->bb_counters, 0,
837 sizeof(*e4b->bd_info->bb_counters) *
838 (e4b->bd_sb->s_blocksize_bits + 2));
840 ext4_mb_generate_buddy(e4b->bd_sb, e4b->bd_buddy,
841 e4b->bd_bitmap, e4b->bd_group, e4b->bd_info);
1034 * Return locked buddy and bitmap pages on e4b struct. If buddy and bitmap
1035 * are on the same page e4b->bd_buddy_page is NULL and return value is 0.
1038 ext4_group_t group, struct ext4_buddy *e4b, gfp_t gfp)
1045 e4b->bd_buddy_page = NULL;
1046 e4b->bd_bitmap_page = NULL;
1061 e4b->bd_bitmap_page = page;
1062 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
1075 e4b->bd_buddy_page = page;
1079 static void ext4_mb_put_buddy_page_lock(struct ext4_buddy *e4b)
1081 if (e4b->bd_bitmap_page) {
1082 unlock_page(e4b->bd_bitmap_page);
1083 put_page(e4b->bd_bitmap_page);
1085 if (e4b->bd_buddy_page) {
1086 unlock_page(e4b->bd_buddy_page);
1087 put_page(e4b->bd_buddy_page);
1101 struct ext4_buddy e4b;
1120 ret = ext4_mb_get_buddy_page_lock(sb, group, &e4b, gfp);
1129 page = e4b.bd_bitmap_page;
1138 if (e4b.bd_buddy_page == NULL) {
1148 page = e4b.bd_buddy_page;
1149 ret = ext4_mb_init_cache(page, e4b.bd_bitmap, gfp);
1157 ext4_mb_put_buddy_page_lock(&e4b);
1168 struct ext4_buddy *e4b, gfp_t gfp)
1188 e4b->bd_blkbits = sb->s_blocksize_bits;
1189 e4b->bd_info = grp;
1190 e4b->bd_sb = sb;
1191 e4b->bd_group = group;
1192 e4b->bd_buddy_page = NULL;
1193 e4b->bd_bitmap_page = NULL;
1237 mb_cmp_bitmaps(e4b, page_address(page) +
1253 e4b->bd_bitmap_page = page;
1254 e4b->bd_bitmap = page_address(page) + (poff * sb->s_blocksize);
1268 ret = ext4_mb_init_cache(page, e4b->bd_bitmap,
1288 e4b->bd_buddy_page = page;
1289 e4b->bd_buddy = page_address(page) + (poff * sb->s_blocksize);
1296 if (e4b->bd_bitmap_page)
1297 put_page(e4b->bd_bitmap_page);
1298 if (e4b->bd_buddy_page)
1299 put_page(e4b->bd_buddy_page);
1300 e4b->bd_buddy = NULL;
1301 e4b->bd_bitmap = NULL;
1306 struct ext4_buddy *e4b)
1308 return ext4_mb_load_buddy_gfp(sb, group, e4b, GFP_NOFS);
1311 static void ext4_mb_unload_buddy(struct ext4_buddy *e4b)
1313 if (e4b->bd_bitmap_page)
1314 put_page(e4b->bd_bitmap_page);
1315 if (e4b->bd_buddy_page)
1316 put_page(e4b->bd_buddy_page);
1320 static int mb_find_order_for_block(struct ext4_buddy *e4b, int block)
1323 int bb_incr = 1 << (e4b->bd_blkbits - 1);
1326 BUG_ON(e4b->bd_bitmap == e4b->bd_buddy);
1327 BUG_ON(block >= (1 << (e4b->bd_blkbits + 3)));
1329 bb = e4b->bd_buddy;
1330 while (order <= e4b->bd_blkbits + 1) {
1420 static void mb_buddy_mark_free(struct ext4_buddy *e4b, int first, int last)
1424 void *buddy = mb_find_buddy(e4b, order, &max);
1459 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&first, buddy, -1);
1461 e4b->bd_info->bb_counters[order] += mb_buddy_adjust_border(&last, buddy, 1);
1466 if (first == last || !(buddy2 = mb_find_buddy(e4b, order, &max))) {
1468 e4b->bd_info->bb_counters[order - 1] += last - first + 1;
1477 static void mb_free_blocks(struct inode *inode, struct ext4_buddy *e4b,
1484 struct super_block *sb = e4b->bd_sb;
1489 assert_spin_locked(ext4_group_lock_ptr(sb, e4b->bd_group));
1491 if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))
1494 mb_check_buddy(e4b);
1495 mb_free_blocks_double(inode, e4b, first, count);
1498 e4b->bd_info->bb_free += count;
1499 if (first < e4b->bd_info->bb_first_free)
1500 e4b->bd_info->bb_first_free = first;
1506 left_is_free = !mb_test_bit(first - 1, e4b->bd_bitmap);
1507 block = mb_test_and_clear_bits(e4b->bd_bitmap, first, count);
1509 right_is_free = !mb_test_bit(last + 1, e4b->bd_bitmap);
1515 blocknr = ext4_group_first_block_no(sb, e4b->bd_group);
1518 ext4_grp_locked_error(sb, e4b->bd_group,
1524 sb, e4b->bd_group,
1527 mb_regenerate_buddy(e4b);
1534 e4b->bd_info->bb_fragments--;
1536 e4b->bd_info->bb_fragments++;
1546 e4b->bd_info->bb_counters[0] += left_is_free ? -1 : 1;
1550 e4b->bd_info->bb_counters[0] += right_is_free ? -1 : 1;
1554 mb_buddy_mark_free(e4b, first >> 1, last >> 1);
1557 mb_set_largest_free_order(sb, e4b->bd_info);
1558 mb_check_buddy(e4b);
1561 static int mb_find_extent(struct ext4_buddy *e4b, int block,
1568 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
1571 buddy = mb_find_buddy(e4b, 0, &max);
1582 order = mb_find_order_for_block(e4b, block);
1587 ex->fe_group = e4b->bd_group;
1595 mb_find_buddy(e4b, order, &max)) {
1601 if (mb_test_bit(next, e4b->bd_bitmap))
1604 order = mb_find_order_for_block(e4b, next);
1610 if (ex->fe_start + ex->fe_len > EXT4_CLUSTERS_PER_GROUP(e4b->bd_sb)) {
1613 ext4_grp_locked_error(e4b->bd_sb, e4b->bd_group, 0, 0,
1625 static int mb_mark_used(struct ext4_buddy *e4b, struct ext4_free_extent *ex)
1637 BUG_ON(start + len > (e4b->bd_sb->s_blocksize << 3));
1638 BUG_ON(e4b->bd_group != ex->fe_group);
1639 assert_spin_locked(ext4_group_lock_ptr(e4b->bd_sb, e4b->bd_group));
1640 mb_check_buddy(e4b);
1641 mb_mark_used_double(e4b, start, len);
1644 e4b->bd_info->bb_free -= len;
1645 if (e4b->bd_info->bb_first_free == start)
1646 e4b->bd_info->bb_first_free += len;
1650 mlen = !mb_test_bit(start - 1, e4b->bd_bitmap);
1651 if (start + len < EXT4_SB(e4b->bd_sb)->s_mb_maxs[0])
1652 max = !mb_test_bit(start + len, e4b->bd_bitmap);
1654 e4b->bd_info->bb_fragments++;
1656 e4b->bd_info->bb_fragments--;
1660 ord = mb_find_order_for_block(e4b, start);
1665 buddy = mb_find_buddy(e4b, ord, &max);
1668 e4b->bd_info->bb_counters[ord]--;
1681 buddy = mb_find_buddy(e4b, ord, &max);
1683 e4b->bd_info->bb_counters[ord]--;
1687 buddy = mb_find_buddy(e4b, ord, &max);
1690 e4b->bd_info->bb_counters[ord]++;
1691 e4b->bd_info->bb_counters[ord]++;
1693 mb_set_largest_free_order(e4b->bd_sb, e4b->bd_info);
1695 ext4_set_bits(e4b->bd_bitmap, ex->fe_start, len0);
1696 mb_check_buddy(e4b);
1705 struct ext4_buddy *e4b)
1710 BUG_ON(ac->ac_b_ex.fe_group != e4b->bd_group);
1715 ret = mb_mark_used(e4b, &ac->ac_b_ex);
1732 ac->ac_bitmap_page = e4b->bd_bitmap_page;
1734 ac->ac_buddy_page = e4b->bd_buddy_page;
1754 struct ext4_buddy *e4b,
1781 && bex->fe_group == e4b->bd_group) {
1785 max = mb_find_extent(e4b, bex->fe_start, gex->fe_len, &ex);
1787 ext4_mb_use_best_found(ac, e4b);
1805 struct ext4_buddy *e4b)
1822 ext4_mb_use_best_found(ac, e4b);
1831 ext4_mb_use_best_found(ac, e4b);
1859 ext4_mb_check_limits(ac, e4b, 0);
1864 struct ext4_buddy *e4b)
1872 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
1877 if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))
1880 max = mb_find_extent(e4b, ex.fe_start, ex.fe_len, &ex);
1884 ext4_mb_use_best_found(ac, e4b);
1889 ext4_mb_unload_buddy(e4b);
1896 struct ext4_buddy *e4b)
1912 err = ext4_mb_load_buddy(ac->ac_sb, group, e4b);
1917 if (unlikely(EXT4_MB_GRP_BBITMAP_CORRUPT(e4b->bd_info)))
1920 max = mb_find_extent(e4b, ac->ac_g_ex.fe_start,
1927 start = ext4_group_first_block_no(ac->ac_sb, e4b->bd_group) +
1933 ext4_mb_use_best_found(ac, e4b);
1941 ext4_mb_use_best_found(ac, e4b);
1950 ext4_mb_use_best_found(ac, e4b);
1954 ext4_mb_unload_buddy(e4b);
1965 struct ext4_buddy *e4b)
1968 struct ext4_group_info *grp = e4b->bd_info;
1979 buddy = mb_find_buddy(e4b, i, &max);
1984 ext4_grp_locked_error(ac->ac_sb, e4b->bd_group, 0, 0,
1988 e4b->bd_group,
1996 ac->ac_b_ex.fe_group = e4b->bd_group;
1998 ext4_mb_use_best_found(ac, e4b);
2016 struct ext4_buddy *e4b)
2019 void *bitmap = e4b->bd_bitmap;
2024 free = e4b->bd_info->bb_free;
2028 i = e4b->bd_info->bb_first_free;
2039 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
2043 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
2048 mb_find_extent(e4b, i, ac->ac_g_ex.fe_len, &ex);
2052 ext4_grp_locked_error(sb, e4b->bd_group, 0, 0,
2056 ext4_mark_group_bitmap_corrupted(sb, e4b->bd_group,
2066 ext4_mb_measure_extent(ac, &ex, e4b);
2072 ext4_mb_check_limits(ac, e4b, 1);
2081 struct ext4_buddy *e4b)
2085 void *bitmap = e4b->bd_bitmap;
2095 first_group_block = ext4_group_first_block_no(sb, e4b->bd_group);
2103 max = mb_find_extent(e4b, i, sbi->s_stripe, &ex);
2108 ext4_mb_use_best_found(ac, e4b);
2330 struct ext4_buddy e4b;
2343 err = ext4_mb_find_by_goal(ac, &e4b);
2439 err = ext4_mb_load_buddy(sb, group, &e4b);
2452 ext4_mb_unload_buddy(&e4b);
2458 ext4_mb_simple_scan_group(ac, &e4b);
2461 ext4_mb_scan_aligned(ac, &e4b);
2463 ext4_mb_complex_scan_group(ac, &e4b);
2466 ext4_mb_unload_buddy(&e4b);
2482 ext4_mb_try_best_found(ac, &e4b);
2549 struct ext4_buddy e4b;
2573 err = ext4_mb_load_buddy(sb, group, &e4b);
2584 ext4_mb_unload_buddy(&e4b);
3147 struct ext4_buddy e4b;
3154 err = ext4_mb_load_buddy(sb, entry->efd_group, &e4b);
3162 db = e4b.bd_info;
3169 mb_free_blocks(NULL, &e4b, entry->efd_start_cluster, entry->efd_count);
3184 put_page(e4b.bd_buddy_page);
3185 put_page(e4b.bd_bitmap_page);
3189 ext4_mb_unload_buddy(&e4b);
3776 struct ext4_buddy e4b;
3782 err = ext4_mb_load_buddy(ac->ac_sb, ac->ac_f_ex.fe_group, &e4b);
3793 mb_free_blocks(ac->ac_inode, &e4b, ac->ac_f_ex.fe_start,
3796 ext4_mb_unload_buddy(&e4b);
4310 ext4_mb_release_inode_pa(struct ext4_buddy *e4b, struct buffer_head *bitmap_bh,
4313 struct super_block *sb = e4b->bd_sb;
4325 BUG_ON(group != e4b->bd_group && pa->pa_len != 0);
4342 mb_free_blocks(pa->pa_inode, e4b, bit, next - bit);
4346 ext4_msg(e4b->bd_sb, KERN_CRIT,
4364 ext4_mb_release_group_pa(struct ext4_buddy *e4b,
4367 struct super_block *sb = e4b->bd_sb;
4374 if (unlikely(group != e4b->bd_group && pa->pa_len != 0)) {
4376 e4b->bd_group, group, pa->pa_pstart);
4379 mb_free_blocks(pa->pa_inode, e4b, bit, pa->pa_len);
4403 struct ext4_buddy e4b;
4422 err = ext4_mb_load_buddy(sb, group, &e4b);
4469 ext4_mb_release_group_pa(&e4b, pa);
4471 ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
4478 ext4_mb_unload_buddy(&e4b);
4503 struct ext4_buddy e4b;
4578 err = ext4_mb_load_buddy_gfp(sb, group, &e4b,
4591 ext4_mb_unload_buddy(&e4b);
4597 ext4_mb_release_inode_pa(&e4b, bitmap_bh, pa);
4600 ext4_mb_unload_buddy(&e4b);
4823 struct ext4_buddy e4b;
4876 err = ext4_mb_load_buddy_gfp(sb, group, &e4b,
4885 ext4_mb_release_group_pa(&e4b, pa);
4888 ext4_mb_unload_buddy(&e4b);
5260 ext4_mb_free_metadata(handle_t *handle, struct ext4_buddy *e4b,
5263 ext4_group_t group = e4b->bd_group;
5267 struct ext4_group_info *db = e4b->bd_info;
5268 struct super_block *sb = e4b->bd_sb;
5274 BUG_ON(e4b->bd_bitmap_page == NULL);
5275 BUG_ON(e4b->bd_buddy_page == NULL);
5286 get_page(e4b->bd_buddy_page);
5287 get_page(e4b->bd_bitmap_page);
5458 struct ext4_buddy e4b;
5538 err = ext4_mb_load_buddy_gfp(sb, block_group, &e4b,
5566 ext4_mb_free_metadata(handle, &e4b, new_entry);
5581 EXT4_MB_GRP_CLEAR_TRIMMED(e4b.bd_info);
5585 mb_free_blocks(inode, &e4b, bit, count_clusters);
5613 ext4_mb_unload_buddy(&e4b);
5761 struct ext4_buddy e4b;
5832 err = ext4_mb_load_buddy(sb, block_group, &e4b);
5843 mb_free_blocks(NULL, &e4b, bit, cluster_count);
5860 ext4_mb_unload_buddy(&e4b);
5883 * @e4b: ext4 buddy for the group
5890 int start, int count, struct ext4_buddy *e4b)
5895 ext4_group_t group = e4b->bd_group;
5910 mb_mark_used(e4b, &ex);
5914 mb_free_blocks(NULL, e4b, start, ex.fe_len);
5939 struct ext4_buddy *e4b, ext4_grpblk_t start,
5946 last = ext4_last_grp_cluster(sb, e4b->bd_group);
5947 bitmap = e4b->bd_bitmap;
5951 start = max(e4b->bd_info->bb_first_free, start);
5965 int ret = ext4_trim_extent(sb, start, next - start, e4b);
5978 ext4_unlock_group(sb, e4b->bd_group);
5980 ext4_lock_group(sb, e4b->bd_group);
5983 if ((e4b->bd_info->bb_free - free_count) < minblocks)
5988 EXT4_MB_GRP_SET_TRIMMED(e4b->bd_info);
6016 struct ext4_buddy e4b;
6021 ret = ext4_mb_load_buddy(sb, group, &e4b);
6030 if (!EXT4_MB_GRP_WAS_TRIMMED(e4b.bd_info) ||
6032 ret = ext4_try_to_trim_range(sb, &e4b, start, max, minblocks);
6037 ext4_mb_unload_buddy(&e4b);
6159 struct ext4_buddy e4b;
6162 error = ext4_mb_load_buddy(sb, group, &e4b);
6165 bitmap = e4b.bd_bitmap;
6169 start = max(e4b.bd_info->bb_first_free, start);
6190 ext4_mb_unload_buddy(&e4b);