Lines Matching refs:bitmap
3 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
5 * bitmap_create - sets up the bitmap structure
6 * bitmap_destroy - destroys the bitmap structure
9 * - added disk storage for bitmap
10 * - changes to allow various bitmap chunk sizes
33 #include "md-bitmap.h"
35 static inline char *bmname(struct bitmap *bitmap)
37 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
50 static int md_bitmap_checkpage(struct bitmap_counts *bitmap,
52 __releases(bitmap->lock)
53 __acquires(bitmap->lock)
57 WARN_ON_ONCE(page >= bitmap->pages);
58 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
61 if (bitmap->bp[page].map) /* page is already allocated, just return */
69 spin_unlock_irq(&bitmap->lock);
84 spin_lock_irq(&bitmap->lock);
87 pr_debug("md/bitmap: map page allocation failed, hijacking\n");
93 if (!bitmap->bp[page].map)
94 bitmap->bp[page].hijacked = 1;
95 } else if (bitmap->bp[page].map ||
96 bitmap->bp[page].hijacked) {
103 bitmap->bp[page].map = mappage;
104 bitmap->missing_pages--;
112 static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
116 if (bitmap->bp[page].count) /* page is still busy */
121 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
122 bitmap->bp[page].hijacked = 0;
123 bitmap->bp[page].map = NULL;
126 ptr = bitmap->bp[page].map;
127 bitmap->bp[page].map = NULL;
128 bitmap->missing_pages++;
134 * bitmap file handling - read and write the bitmap file and its superblock
141 /* IO operations when bitmap is stored near all superblocks */
224 static int __write_sb_page(struct md_rdev *rdev, struct bitmap *bitmap,
228 struct mddev *mddev = bitmap->mddev;
229 struct bitmap_storage *store = &bitmap->storage;
260 /* bitmap runs in to metadata */
264 /* data runs in to bitmap */
270 /* bitmap runs in to data */
280 static void write_sb_page(struct bitmap *bitmap, unsigned long pg_index,
283 struct mddev *mddev = bitmap->mddev;
289 if (__write_sb_page(rdev, bitmap, pg_index, page) < 0) {
290 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
297 static void md_bitmap_file_kick(struct bitmap *bitmap);
300 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
305 atomic_inc(&bitmap->pending_writes);
313 wait_event(bitmap->write_wait,
314 atomic_read(&bitmap->pending_writes) == 0);
319 struct bitmap *bitmap = bh->b_private;
322 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
323 if (atomic_dec_and_test(&bitmap->pending_writes))
324 wake_up(&bitmap->write_wait);
352 struct bitmap *bitmap, unsigned long count, struct page *page)
360 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
391 bh->b_private = bitmap;
392 atomic_inc(&bitmap->pending_writes);
401 wait_event(bitmap->write_wait,
402 atomic_read(&bitmap->pending_writes)==0);
403 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
407 pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
414 static void write_file_page(struct bitmap *bitmap, struct page *page, int wait)
418 struct bitmap *bitmap, unsigned long count, struct page *page)
429 * bitmap file superblock operations
435 static void filemap_write_page(struct bitmap *bitmap, unsigned long pg_index,
438 struct bitmap_storage *store = &bitmap->storage;
441 if (mddev_is_clustered(bitmap->mddev)) {
442 /* go to node bitmap area starting point */
447 write_file_page(bitmap, page, wait);
449 write_sb_page(bitmap, pg_index, page, wait);
453 * md_bitmap_wait_writes() should be called before writing any bitmap
457 static void md_bitmap_wait_writes(struct bitmap *bitmap)
459 if (bitmap->storage.file)
460 wait_event(bitmap->write_wait,
461 atomic_read(&bitmap->pending_writes)==0);
466 * which is safe. The relevant bitmap blocks will
470 md_super_wait(bitmap->mddev);
475 void md_bitmap_update_sb(struct bitmap *bitmap)
479 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
481 if (bitmap->mddev->bitmap_info.external)
483 if (!bitmap->storage.sb_page) /* no superblock */
485 sb = kmap_atomic(bitmap->storage.sb_page);
486 sb->events = cpu_to_le64(bitmap->mddev->events);
487 if (bitmap->mddev->events < bitmap->events_cleared)
489 bitmap->events_cleared = bitmap->mddev->events;
490 sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
493 * a bitmap write error occurred but the later writes succeeded.
495 sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR));
497 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
498 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
500 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
501 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
502 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
503 sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
507 if (bitmap->storage.file)
508 write_file_page(bitmap, bitmap->storage.sb_page, 1);
510 write_sb_page(bitmap, bitmap->storage.sb_index,
511 bitmap->storage.sb_page, 1);
515 /* print out the bitmap file superblock */
516 void md_bitmap_print_sb(struct bitmap *bitmap)
520 if (!bitmap || !bitmap->storage.sb_page)
522 sb = kmap_atomic(bitmap->storage.sb_page);
523 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
546 * @bitmap
549 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
550 * This function verifies 'bitmap_info' and populates the on-disk bitmap
555 static int md_bitmap_new_disk_sb(struct bitmap *bitmap)
560 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
561 if (bitmap->storage.sb_page == NULL)
563 bitmap->storage.sb_index = 0;
565 sb = kmap_atomic(bitmap->storage.sb_page);
570 chunksize = bitmap->mddev->bitmap_info.chunksize;
574 pr_warn("bitmap chunksize not a power of 2\n");
579 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
585 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
591 write_behind = bitmap->mddev->bitmap_info.max_write_behind;
595 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
597 /* keep the array size field of the bitmap superblock up to date */
598 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
600 memcpy(sb->uuid, bitmap->mddev->uuid, 16);
602 set_bit(BITMAP_STALE, &bitmap->flags);
603 sb->state = cpu_to_le32(bitmap->flags);
604 bitmap->events_cleared = bitmap->mddev->events;
605 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
606 bitmap->mddev->bitmap_info.nodes = 0;
613 /* read the superblock from the bitmap file and initialize some bitmap fields */
614 static int md_bitmap_read_sb(struct bitmap *bitmap)
626 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
630 set_bit(BITMAP_STALE, &bitmap->flags);
638 bitmap->storage.sb_page = sb_page;
642 if (bitmap->cluster_slot >= 0) {
643 sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
646 (bitmap->mddev->bitmap_info.chunksize >> 9));
651 offset = bitmap->cluster_slot * (bm_blocks << 3);
653 bitmap->cluster_slot, offset);
656 if (bitmap->storage.file) {
657 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
660 err = read_file_page(bitmap->storage.file, 0,
661 bitmap, bytes, sb_page);
663 err = read_sb_page(bitmap->mddev, offset, sb_page, 0,
677 /* verify that the bitmap-specific fields are valid */
684 reason = "bitmap chunksize too small";
686 reason = "bitmap chunksize not a power of 2";
692 pr_warn("%s: invalid bitmap file superblock: %s\n",
693 bmname(bitmap), reason);
698 * Setup nodes/clustername only if bitmap version is
703 strscpy(bitmap->mddev->bitmap_info.cluster_name,
707 /* keep the array size field of the bitmap superblock up to date */
708 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
710 if (bitmap->mddev->persistent) {
713 * bitmap's UUID and event counter to the mddev's
715 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
716 pr_warn("%s: bitmap superblock UUID mismatch\n",
717 bmname(bitmap));
721 if (!nodes && (events < bitmap->mddev->events)) {
722 pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
723 bmname(bitmap), events,
724 (unsigned long long) bitmap->mddev->events);
725 set_bit(BITMAP_STALE, &bitmap->flags);
730 bitmap->flags |= le32_to_cpu(sb->state);
732 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
733 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
738 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
740 bitmap->mddev->bitmap_info.chunksize = chunksize;
741 err = md_setup_cluster(bitmap->mddev, nodes);
744 bmname(bitmap), err);
747 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
753 if (test_bit(BITMAP_STALE, &bitmap->flags))
754 bitmap->events_cleared = bitmap->mddev->events;
755 bitmap->mddev->bitmap_info.chunksize = chunksize;
756 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
757 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
758 bitmap->mddev->bitmap_info.nodes = nodes;
759 if (bitmap->mddev->bitmap_info.space == 0 ||
760 bitmap->mddev->bitmap_info.space > sectors_reserved)
761 bitmap->mddev->bitmap_info.space = sectors_reserved;
763 md_bitmap_print_sb(bitmap);
764 if (bitmap->cluster_slot < 0)
765 md_cluster_stop(bitmap->mddev);
771 * general bitmap file operations
775 * on-disk bitmap:
777 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
889 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
893 static void md_bitmap_file_kick(struct bitmap *bitmap)
895 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
896 md_bitmap_update_sb(bitmap);
898 if (bitmap->storage.file) {
899 pr_warn("%s: kicking failed bitmap file %pD4 from array!\n",
900 bmname(bitmap), bitmap->storage.file);
903 pr_warn("%s: disabling internal bitmap due to errors\n",
904 bmname(bitmap));
915 static inline void set_page_attr(struct bitmap *bitmap, int pnum,
918 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
921 static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
924 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
927 static inline int test_page_attr(struct bitmap *bitmap, int pnum,
930 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
933 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
937 bitmap->storage.filemap_attr);
941 * to set (and eventually sync) a particular bit in the bitmap file
946 static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
951 unsigned long chunk = block >> bitmap->counts.chunkshift;
952 struct bitmap_storage *store = &bitmap->storage;
957 if (mddev_is_clustered(bitmap->mddev))
958 node_offset = bitmap->cluster_slot * store->file_pages;
960 page = filemap_get_page(&bitmap->storage, chunk);
963 bit = file_page_offset(&bitmap->storage, chunk);
967 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
974 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_DIRTY);
977 static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
982 unsigned long chunk = block >> bitmap->counts.chunkshift;
983 struct bitmap_storage *store = &bitmap->storage;
988 if (mddev_is_clustered(bitmap->mddev))
989 node_offset = bitmap->cluster_slot * store->file_pages;
991 page = filemap_get_page(&bitmap->storage, chunk);
994 bit = file_page_offset(&bitmap->storage, chunk);
996 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1001 if (!test_page_attr(bitmap, index - node_offset, BITMAP_PAGE_NEEDWRITE)) {
1002 set_page_attr(bitmap, index - node_offset, BITMAP_PAGE_PENDING);
1003 bitmap->allclean = 0;
1007 static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
1012 unsigned long chunk = block >> bitmap->counts.chunkshift;
1015 page = filemap_get_page(&bitmap->storage, chunk);
1018 bit = file_page_offset(&bitmap->storage, chunk);
1020 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1030 * sync the dirty pages of the bitmap file to disk */
1031 void md_bitmap_unplug(struct bitmap *bitmap)
1037 if (!md_bitmap_enabled(bitmap))
1042 for (i = 0; i < bitmap->storage.file_pages; i++) {
1043 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1044 need_write = test_and_clear_page_attr(bitmap, i,
1048 md_bitmap_wait_writes(bitmap);
1049 if (bitmap->mddev->queue)
1050 blk_add_trace_msg(bitmap->mddev->queue,
1053 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
1054 filemap_write_page(bitmap, i, false);
1059 md_bitmap_wait_writes(bitmap);
1061 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1062 md_bitmap_file_kick(bitmap);
1068 struct bitmap *bitmap;
1077 md_bitmap_unplug(unplug_work->bitmap);
1081 void md_bitmap_unplug_async(struct bitmap *bitmap)
1087 unplug_work.bitmap = bitmap;
1095 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1098 * Initialize the in-memory bitmap from the on-disk bitmap and set up the memory
1099 * mapping of the bitmap file.
1101 * Special case: If there's no bitmap file, or if the bitmap file had been
1106 * This is used when reading an out-of-date bitmap.
1108 static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1110 bool outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1111 struct mddev *mddev = bitmap->mddev;
1112 unsigned long chunks = bitmap->counts.chunks;
1113 struct bitmap_storage *store = &bitmap->storage;
1121 /* No permanent bitmap - fill with '1s'. */
1126 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1128 md_bitmap_set_memory_bits(bitmap,
1129 (sector_t)i << bitmap->counts.chunkshift,
1136 pr_warn("%s: bitmap file too short %lu < %lu\n",
1137 bmname(bitmap),
1145 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1158 ret = read_file_page(file, i, bitmap, count, page);
1167 pr_warn("%s: bitmap file is out of date, doing full recovery\n",
1168 bmname(bitmap));
1179 * If the bitmap is out of date, dirty the whole page
1186 filemap_write_page(bitmap, i, true);
1187 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags)) {
1195 struct page *page = filemap_get_page(&bitmap->storage, i);
1196 unsigned long bit = file_page_offset(&bitmap->storage, i);
1201 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1209 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1211 md_bitmap_set_memory_bits(bitmap,
1212 (sector_t)i << bitmap->counts.chunkshift,
1218 pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
1219 bmname(bitmap), store->file_pages,
1225 pr_warn("%s: bitmap initialisation failed: %d\n",
1226 bmname(bitmap), ret);
1230 void md_bitmap_write_all(struct bitmap *bitmap)
1232 /* We don't actually write all bitmap blocks here,
1237 if (!bitmap || !bitmap->storage.filemap)
1239 if (bitmap->storage.file)
1243 for (i = 0; i < bitmap->storage.file_pages; i++)
1244 set_page_attr(bitmap, i,
1246 bitmap->allclean = 0;
1249 static void md_bitmap_count_page(struct bitmap_counts *bitmap,
1252 sector_t chunk = offset >> bitmap->chunkshift;
1254 bitmap->bp[page].count += inc;
1255 md_bitmap_checkfree(bitmap, page);
1258 static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1260 sector_t chunk = offset >> bitmap->chunkshift;
1262 struct bitmap_page *bp = &bitmap->bp[page];
1268 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1291 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1296 struct bitmap *bitmap;
1306 bitmap = mddev->bitmap;
1307 if (bitmap == NULL) {
1311 if (time_before(jiffies, bitmap->daemon_lastrun
1315 bitmap->daemon_lastrun = jiffies;
1316 if (bitmap->allclean) {
1320 bitmap->allclean = 1;
1322 if (bitmap->mddev->queue)
1323 blk_add_trace_msg(bitmap->mddev->queue,
1330 for (j = 0; j < bitmap->storage.file_pages; j++)
1331 if (test_and_clear_page_attr(bitmap, j,
1333 set_page_attr(bitmap, j,
1336 if (bitmap->need_sync &&
1341 bitmap->need_sync = 0;
1342 if (bitmap->storage.filemap) {
1343 sb = kmap_atomic(bitmap->storage.sb_page);
1345 cpu_to_le64(bitmap->events_cleared);
1347 set_page_attr(bitmap, 0,
1351 /* Now look at the bitmap counters and if any are '2' or '1',
1354 counts = &bitmap->counts;
1375 if (*bmc == 1 && !bitmap->need_sync) {
1379 md_bitmap_file_clear_bit(bitmap, block);
1383 bitmap->allclean = 0;
1388 md_bitmap_wait_writes(bitmap);
1398 j < bitmap->storage.file_pages
1399 && !test_bit(BITMAP_STALE, &bitmap->flags);
1401 if (test_page_attr(bitmap, j,
1405 if (bitmap->storage.filemap &&
1406 test_and_clear_page_attr(bitmap, j,
1408 filemap_write_page(bitmap, j, false);
1412 if (bitmap->allclean == 0)
1417 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1420 __releases(bitmap->lock)
1421 __acquires(bitmap->lock)
1427 sector_t chunk = offset >> bitmap->chunkshift;
1433 if (page >= bitmap->pages) {
1441 err = md_bitmap_checkpage(bitmap, page, create, 0);
1443 if (bitmap->bp[page].hijacked ||
1444 bitmap->bp[page].map == NULL)
1445 csize = ((sector_t)1) << (bitmap->chunkshift +
1448 csize = ((sector_t)1) << bitmap->chunkshift;
1456 if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1461 &bitmap->bp[page].map)[hi];
1464 &(bitmap->bp[page].map[pageoff]);
1467 int md_bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1469 if (!bitmap)
1474 atomic_inc(&bitmap->behind_writes);
1475 bw = atomic_read(&bitmap->behind_writes);
1476 if (bw > bitmap->behind_writes_used)
1477 bitmap->behind_writes_used = bw;
1480 bw, bitmap->mddev->bitmap_info.max_write_behind);
1487 spin_lock_irq(&bitmap->counts.lock);
1488 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1490 spin_unlock_irq(&bitmap->counts.lock);
1500 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1502 spin_unlock_irq(&bitmap->counts.lock);
1504 finish_wait(&bitmap->overflow_wait, &__wait);
1510 md_bitmap_file_set_bit(bitmap, offset);
1511 md_bitmap_count_page(&bitmap->counts, offset, 1);
1519 spin_unlock_irq(&bitmap->counts.lock);
1531 void md_bitmap_endwrite(struct bitmap *bitmap, sector_t offset,
1534 if (!bitmap)
1537 if (atomic_dec_and_test(&bitmap->behind_writes))
1538 wake_up(&bitmap->behind_wait);
1540 atomic_read(&bitmap->behind_writes),
1541 bitmap->mddev->bitmap_info.max_write_behind);
1549 spin_lock_irqsave(&bitmap->counts.lock, flags);
1550 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1552 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1556 if (success && !bitmap->mddev->degraded &&
1557 bitmap->events_cleared < bitmap->mddev->events) {
1558 bitmap->events_cleared = bitmap->mddev->events;
1559 bitmap->need_sync = 1;
1560 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1567 wake_up(&bitmap->overflow_wait);
1571 md_bitmap_set_pending(&bitmap->counts, offset);
1572 bitmap->allclean = 0;
1574 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1584 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1589 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1591 return 1; /* always resync if no bitmap */
1593 spin_lock_irq(&bitmap->counts.lock);
1594 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1608 spin_unlock_irq(&bitmap->counts.lock);
1612 int md_bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1627 rv |= __bitmap_start_sync(bitmap, offset,
1636 void md_bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1641 if (bitmap == NULL) {
1645 spin_lock_irqsave(&bitmap->counts.lock, flags);
1646 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1657 md_bitmap_set_pending(&bitmap->counts, offset);
1658 bitmap->allclean = 0;
1663 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1667 void md_bitmap_close_sync(struct bitmap *bitmap)
1669 /* Sync has finished, and any bitmap chunks that weren't synced
1675 if (!bitmap)
1677 while (sector < bitmap->mddev->resync_max_sectors) {
1678 md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1684 void md_bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
1689 if (!bitmap)
1692 bitmap->last_end_sync = jiffies;
1695 if (!force && time_before(jiffies, (bitmap->last_end_sync
1696 + bitmap->mddev->bitmap_info.daemon_sleep)))
1698 wait_event(bitmap->mddev->recovery_wait,
1699 atomic_read(&bitmap->mddev->recovery_active) == 0);
1701 bitmap->mddev->curr_resync_completed = sector;
1702 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
1703 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1705 while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1706 md_bitmap_end_sync(bitmap, s, &blocks, 0);
1709 bitmap->last_end_sync = jiffies;
1710 sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed);
1718 struct bitmap *bitmap = mddev->bitmap;
1722 md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1728 md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1735 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1744 spin_lock_irq(&bitmap->counts.lock);
1745 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1747 spin_unlock_irq(&bitmap->counts.lock);
1752 md_bitmap_count_page(&bitmap->counts, offset, 1);
1753 md_bitmap_set_pending(&bitmap->counts, offset);
1754 bitmap->allclean = 0;
1758 spin_unlock_irq(&bitmap->counts.lock);
1761 /* dirty the memory and file bits for bitmap chunks "s" to "e" */
1762 void md_bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1767 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1768 md_bitmap_set_memory_bits(bitmap, sec, 1);
1769 md_bitmap_file_set_bit(bitmap, sec);
1770 if (sec < bitmap->mddev->recovery_cp)
1775 bitmap->mddev->recovery_cp = sec;
1784 struct bitmap *bitmap = mddev->bitmap;
1787 if (!bitmap) /* there was no bitmap */
1794 bitmap->daemon_lastrun -= sleep;
1796 bitmap->daemon_lastrun -= sleep;
1798 bitmap->daemon_lastrun -= sleep;
1802 md_bitmap_update_sb(bitmap);
1808 void md_bitmap_free(struct bitmap *bitmap)
1813 if (!bitmap) /* there was no bitmap */
1816 if (bitmap->sysfs_can_clear)
1817 sysfs_put(bitmap->sysfs_can_clear);
1819 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
1820 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
1821 md_cluster_stop(bitmap->mddev);
1824 wait_event(bitmap->write_wait,
1825 atomic_read(&bitmap->pending_writes) == 0);
1827 /* release the bitmap file */
1828 md_bitmap_file_unmap(&bitmap->storage);
1830 bp = bitmap->counts.bp;
1831 pages = bitmap->counts.pages;
1840 kfree(bitmap);
1846 struct bitmap *bitmap = mddev->bitmap;
1849 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
1853 wait_event(bitmap->behind_wait,
1854 atomic_read(&bitmap->behind_writes) == 0);
1860 struct bitmap *bitmap = mddev->bitmap;
1862 if (!bitmap) /* there was no bitmap */
1871 mddev->bitmap = NULL; /* disconnect from the md device */
1876 md_bitmap_free(bitmap);
1880 * initialize the bitmap structure
1882 * once mddev->bitmap is set
1884 struct bitmap *md_bitmap_create(struct mddev *mddev, int slot)
1886 struct bitmap *bitmap;
1897 pr_notice("md/raid:%s: array with journal cannot have bitmap\n",
1902 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1903 if (!bitmap)
1906 spin_lock_init(&bitmap->counts.lock);
1907 atomic_set(&bitmap->pending_writes, 0);
1908 init_waitqueue_head(&bitmap->write_wait);
1909 init_waitqueue_head(&bitmap->overflow_wait);
1910 init_waitqueue_head(&bitmap->behind_wait);
1912 bitmap->mddev = mddev;
1913 bitmap->cluster_slot = slot;
1916 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1918 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1921 bitmap->sysfs_can_clear = NULL;
1923 bitmap->storage.file = file;
1932 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1936 * instructing us to create a new on-disk bitmap instance.
1939 err = md_bitmap_new_disk_sb(bitmap);
1941 err = md_bitmap_read_sb(bitmap);
1953 bitmap->daemon_lastrun = jiffies;
1954 err = md_bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1958 pr_debug("created bitmap (%lu pages) for device %s\n",
1959 bitmap->counts.pages, bmname(bitmap));
1961 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1965 return bitmap;
1967 md_bitmap_free(bitmap);
1976 struct bitmap *bitmap = mddev->bitmap;
1979 if (!bitmap)
1988 /* Clear out old bitmap info first: Either there is none, or we
1995 md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1998 md_bitmap_close_sync(bitmap);
2001 || bitmap->events_cleared == mddev->events)
2007 err = md_bitmap_init_from_disk(bitmap, start);
2012 clear_bit(BITMAP_STALE, &bitmap->flags);
2015 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
2020 md_bitmap_update_sb(bitmap);
2022 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
2029 /* caller need to free returned bitmap with md_bitmap_free() */
2030 struct bitmap *get_bitmap_from_slot(struct mddev *mddev, int slot)
2033 struct bitmap *bitmap;
2035 bitmap = md_bitmap_create(mddev, slot);
2036 if (IS_ERR(bitmap)) {
2037 rv = PTR_ERR(bitmap);
2041 rv = md_bitmap_init_from_disk(bitmap, 0);
2043 md_bitmap_free(bitmap);
2047 return bitmap;
2051 /* Loads the bitmap associated with slot and copies the resync information
2052 * to our bitmap
2060 struct bitmap *bitmap;
2062 bitmap = get_bitmap_from_slot(mddev, slot);
2063 if (IS_ERR(bitmap)) {
2064 pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
2068 counts = &bitmap->counts;
2071 if (md_bitmap_file_test_bit(bitmap, block)) {
2075 md_bitmap_file_clear_bit(bitmap, block);
2076 md_bitmap_set_memory_bits(mddev->bitmap, block, 1);
2077 md_bitmap_file_set_bit(mddev->bitmap, block);
2082 md_bitmap_update_sb(bitmap);
2085 for (i = 0; i < bitmap->storage.file_pages; i++)
2086 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING))
2087 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
2088 md_bitmap_unplug(bitmap);
2090 md_bitmap_unplug(mddev->bitmap);
2093 md_bitmap_free(bitmap);
2100 void md_bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
2105 if (!bitmap)
2108 counts = &bitmap->counts;
2110 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
2111 seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
2117 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
2119 if (bitmap->storage.file) {
2121 seq_file_path(seq, bitmap->storage.file, " \t\n");
2127 int md_bitmap_resize(struct bitmap *bitmap, sector_t blocks,
2132 * Then quiesce, copy bits, replace bitmap, and re-start
2134 * This function is called both to set up the initial bitmap
2135 * and to resize the bitmap while the array is active.
2150 if (bitmap->storage.file && !init) {
2151 pr_info("md: cannot resize file-based bitmap\n");
2160 long space = bitmap->mddev->bitmap_info.space;
2166 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
2167 if (!bitmap->mddev->bitmap_info.external)
2170 bitmap->mddev->bitmap_info.space = space;
2172 chunkshift = bitmap->counts.chunkshift;
2179 if (!bitmap->mddev->bitmap_info.external)
2188 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
2190 !bitmap->mddev->bitmap_info.external,
2191 mddev_is_clustered(bitmap->mddev)
2192 ? bitmap->cluster_slot : 0);
2208 bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2210 store.file = bitmap->storage.file;
2211 bitmap->storage.file = NULL;
2213 if (store.sb_page && bitmap->storage.sb_page)
2215 page_address(bitmap->storage.sb_page),
2217 spin_lock_irq(&bitmap->counts.lock);
2218 md_bitmap_file_unmap(&bitmap->storage);
2219 bitmap->storage = store;
2221 old_counts = bitmap->counts;
2222 bitmap->counts.bp = new_bp;
2223 bitmap->counts.pages = pages;
2224 bitmap->counts.missing_pages = pages;
2225 bitmap->counts.chunkshift = chunkshift;
2226 bitmap->counts.chunks = chunks;
2227 bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift +
2233 /* For cluster raid, need to pre-allocate bitmap */
2234 if (mddev_is_clustered(bitmap->mddev)) {
2237 ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1);
2248 bitmap->counts.bp = old_counts.bp;
2249 bitmap->counts.pages = old_counts.pages;
2250 bitmap->counts.missing_pages = old_counts.pages;
2251 bitmap->counts.chunkshift = old_counts.chunkshift;
2252 bitmap->counts.chunks = old_counts.chunks;
2253 bitmap->mddev->bitmap_info.chunksize =
2256 pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
2259 bitmap->counts.bp[page].count += 1;
2271 bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2280 md_bitmap_file_set_bit(bitmap, block);
2284 md_bitmap_count_page(&bitmap->counts, block, 1);
2285 md_bitmap_set_pending(&bitmap->counts, block);
2295 if (bitmap->counts.bp != old_counts.bp) {
2307 bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2314 md_bitmap_count_page(&bitmap->counts, block, 1);
2315 md_bitmap_set_pending(&bitmap->counts, block);
2320 for (i = 0; i < bitmap->storage.file_pages; i++)
2321 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
2323 spin_unlock_irq(&bitmap->counts.lock);
2326 md_bitmap_unplug(bitmap);
2327 bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2368 if (mddev->bitmap || mddev->bitmap_info.file ||
2370 /* bitmap already configured. Only option is to clear it */
2387 /* No bitmap, OK to set a location */
2414 struct bitmap *bitmap;
2415 bitmap = md_bitmap_create(mddev, -1);
2417 if (IS_ERR(bitmap))
2418 rv = PTR_ERR(bitmap);
2420 mddev->bitmap = bitmap;
2435 /* Ensure new bitmap info is stored in
2452 /* 'bitmap/space' is the space available at 'location' for the
2453 * bitmap. This allows the kernel to know when it is safe to
2454 * resize the bitmap to match a resized array.
2475 if (mddev->bitmap &&
2476 sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2584 md_bitmap_update_sb(mddev->bitmap);
2602 /* Can only be changed when no bitmap is active */
2605 if (mddev->bitmap)
2633 if (mddev->bitmap ||
2654 if (mddev->bitmap)
2655 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2665 if (mddev->bitmap == NULL)
2668 mddev->bitmap->need_sync = 1;
2672 mddev->bitmap->need_sync = 0;
2686 if (mddev->bitmap == NULL)
2690 mddev->bitmap->behind_writes_used);
2698 if (mddev->bitmap)
2699 mddev->bitmap->behind_writes_used = 0;
2719 .name = "bitmap",