Lines Matching refs:bitmap
3 * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
5 * bitmap_create - sets up the bitmap structure
6 * bitmap_destroy - destroys the bitmap structure
9 * - added disk storage for bitmap
10 * - changes to allow various bitmap chunk sizes
33 #include "md-bitmap.h"
35 static inline char *bmname(struct bitmap *bitmap)
37 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
50 static int md_bitmap_checkpage(struct bitmap_counts *bitmap,
52 __releases(bitmap->lock)
53 __acquires(bitmap->lock)
57 WARN_ON_ONCE(page >= bitmap->pages);
58 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
61 if (bitmap->bp[page].map) /* page is already allocated, just return */
69 spin_unlock_irq(&bitmap->lock);
84 spin_lock_irq(&bitmap->lock);
87 pr_debug("md/bitmap: map page allocation failed, hijacking\n");
93 if (!bitmap->bp[page].map)
94 bitmap->bp[page].hijacked = 1;
95 } else if (bitmap->bp[page].map ||
96 bitmap->bp[page].hijacked) {
103 bitmap->bp[page].map = mappage;
104 bitmap->missing_pages--;
112 static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
116 if (bitmap->bp[page].count) /* page is still busy */
121 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
122 bitmap->bp[page].hijacked = 0;
123 bitmap->bp[page].map = NULL;
126 ptr = bitmap->bp[page].map;
127 bitmap->bp[page].map = NULL;
128 bitmap->missing_pages++;
134 * bitmap file handling - read and write the bitmap file and its superblock
141 /* IO operations when bitmap is stored near all superblocks */
205 static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
209 struct mddev *mddev = bitmap->mddev;
210 struct bitmap_storage *store = &bitmap->storage;
245 /* bitmap runs in to metadata */
249 /* data runs in to bitmap */
257 /* bitmap runs in to data */
277 static void md_bitmap_file_kick(struct bitmap *bitmap);
281 static void write_page(struct bitmap *bitmap, struct page *page, int wait)
285 if (bitmap->storage.file == NULL) {
286 switch (write_sb_page(bitmap, page, wait)) {
288 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
295 atomic_inc(&bitmap->pending_writes);
303 wait_event(bitmap->write_wait,
304 atomic_read(&bitmap->pending_writes)==0);
306 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
307 md_bitmap_file_kick(bitmap);
312 struct bitmap *bitmap = bh->b_private;
315 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
316 if (atomic_dec_and_test(&bitmap->pending_writes))
317 wake_up(&bitmap->write_wait);
345 struct bitmap *bitmap,
355 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
386 bh->b_private = bitmap;
387 atomic_inc(&bitmap->pending_writes);
397 wait_event(bitmap->write_wait,
398 atomic_read(&bitmap->pending_writes)==0);
399 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
403 pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
411 * bitmap file superblock operations
415 * md_bitmap_wait_writes() should be called before writing any bitmap
419 static void md_bitmap_wait_writes(struct bitmap *bitmap)
421 if (bitmap->storage.file)
422 wait_event(bitmap->write_wait,
423 atomic_read(&bitmap->pending_writes)==0);
428 * which is safe. The relevant bitmap blocks will
432 md_super_wait(bitmap->mddev);
437 void md_bitmap_update_sb(struct bitmap *bitmap)
441 if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
443 if (bitmap->mddev->bitmap_info.external)
445 if (!bitmap->storage.sb_page) /* no superblock */
447 sb = kmap_atomic(bitmap->storage.sb_page);
448 sb->events = cpu_to_le64(bitmap->mddev->events);
449 if (bitmap->mddev->events < bitmap->events_cleared)
451 bitmap->events_cleared = bitmap->mddev->events;
452 sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
455 * a bitmap write error occurred but the later writes succeeded.
457 sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR));
459 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
460 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
462 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
463 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
464 sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
465 sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
468 write_page(bitmap, bitmap->storage.sb_page, 1);
472 /* print out the bitmap file superblock */
473 void md_bitmap_print_sb(struct bitmap *bitmap)
477 if (!bitmap || !bitmap->storage.sb_page)
479 sb = kmap_atomic(bitmap->storage.sb_page);
480 pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
503 * @bitmap
506 * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
507 * This function verifies 'bitmap_info' and populates the on-disk bitmap
512 static int md_bitmap_new_disk_sb(struct bitmap *bitmap)
517 bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
518 if (bitmap->storage.sb_page == NULL)
520 bitmap->storage.sb_page->index = 0;
522 sb = kmap_atomic(bitmap->storage.sb_page);
527 chunksize = bitmap->mddev->bitmap_info.chunksize;
531 pr_warn("bitmap chunksize not a power of 2\n");
536 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
542 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
548 write_behind = bitmap->mddev->bitmap_info.max_write_behind;
552 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
554 /* keep the array size field of the bitmap superblock up to date */
555 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
557 memcpy(sb->uuid, bitmap->mddev->uuid, 16);
559 set_bit(BITMAP_STALE, &bitmap->flags);
560 sb->state = cpu_to_le32(bitmap->flags);
561 bitmap->events_cleared = bitmap->mddev->events;
562 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
563 bitmap->mddev->bitmap_info.nodes = 0;
570 /* read the superblock from the bitmap file and initialize some bitmap fields */
571 static int md_bitmap_read_sb(struct bitmap *bitmap)
581 loff_t offset = bitmap->mddev->bitmap_info.offset;
583 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
587 set_bit(BITMAP_STALE, &bitmap->flags);
595 bitmap->storage.sb_page = sb_page;
599 if (bitmap->cluster_slot >= 0) {
600 sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
603 (bitmap->mddev->bitmap_info.chunksize >> 9));
608 offset = bitmap->mddev->bitmap_info.offset + (bitmap->cluster_slot * (bm_blocks << 3));
610 bitmap->cluster_slot, offset);
613 if (bitmap->storage.file) {
614 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
617 err = read_page(bitmap->storage.file, 0,
618 bitmap, bytes, sb_page);
620 err = read_sb_page(bitmap->mddev,
636 /* verify that the bitmap-specific fields are valid */
643 reason = "bitmap chunksize too small";
645 reason = "bitmap chunksize not a power of 2";
651 pr_warn("%s: invalid bitmap file superblock: %s\n",
652 bmname(bitmap), reason);
657 * Setup nodes/clustername only if bitmap version is
662 strlcpy(bitmap->mddev->bitmap_info.cluster_name,
666 /* keep the array size field of the bitmap superblock up to date */
667 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
669 if (bitmap->mddev->persistent) {
672 * bitmap's UUID and event counter to the mddev's
674 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
675 pr_warn("%s: bitmap superblock UUID mismatch\n",
676 bmname(bitmap));
680 if (!nodes && (events < bitmap->mddev->events)) {
681 pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
682 bmname(bitmap), events,
683 (unsigned long long) bitmap->mddev->events);
684 set_bit(BITMAP_STALE, &bitmap->flags);
689 bitmap->flags |= le32_to_cpu(sb->state);
691 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
692 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
693 strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
698 if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
700 bitmap->mddev->bitmap_info.chunksize = chunksize;
701 err = md_setup_cluster(bitmap->mddev, nodes);
704 bmname(bitmap), err);
707 bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
713 if (test_bit(BITMAP_STALE, &bitmap->flags))
714 bitmap->events_cleared = bitmap->mddev->events;
715 bitmap->mddev->bitmap_info.chunksize = chunksize;
716 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
717 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
718 bitmap->mddev->bitmap_info.nodes = nodes;
719 if (bitmap->mddev->bitmap_info.space == 0 ||
720 bitmap->mddev->bitmap_info.space > sectors_reserved)
721 bitmap->mddev->bitmap_info.space = sectors_reserved;
723 md_bitmap_print_sb(bitmap);
724 if (bitmap->cluster_slot < 0)
725 md_cluster_stop(bitmap->mddev);
731 * general bitmap file operations
735 * on-disk bitmap:
737 * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
854 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
858 static void md_bitmap_file_kick(struct bitmap *bitmap)
862 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
863 md_bitmap_update_sb(bitmap);
865 if (bitmap->storage.file) {
868 ptr = file_path(bitmap->storage.file,
871 pr_warn("%s: kicking failed bitmap file %s from array!\n",
872 bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
876 pr_warn("%s: disabling internal bitmap due to errors\n",
877 bmname(bitmap));
888 static inline void set_page_attr(struct bitmap *bitmap, int pnum,
891 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
894 static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
897 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
900 static inline int test_page_attr(struct bitmap *bitmap, int pnum,
903 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
906 static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
910 bitmap->storage.filemap_attr);
914 * to set (and eventually sync) a particular bit in the bitmap file
919 static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
924 unsigned long chunk = block >> bitmap->counts.chunkshift;
925 struct bitmap_storage *store = &bitmap->storage;
928 if (mddev_is_clustered(bitmap->mddev))
929 node_offset = bitmap->cluster_slot * store->file_pages;
931 page = filemap_get_page(&bitmap->storage, chunk);
934 bit = file_page_offset(&bitmap->storage, chunk);
938 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
945 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_DIRTY);
948 static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
953 unsigned long chunk = block >> bitmap->counts.chunkshift;
954 struct bitmap_storage *store = &bitmap->storage;
957 if (mddev_is_clustered(bitmap->mddev))
958 node_offset = bitmap->cluster_slot * store->file_pages;
960 page = filemap_get_page(&bitmap->storage, chunk);
963 bit = file_page_offset(&bitmap->storage, chunk);
965 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
970 if (!test_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_NEEDWRITE)) {
971 set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_PENDING);
972 bitmap->allclean = 0;
976 static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
981 unsigned long chunk = block >> bitmap->counts.chunkshift;
984 page = filemap_get_page(&bitmap->storage, chunk);
987 bit = file_page_offset(&bitmap->storage, chunk);
989 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1000 * sync the dirty pages of the bitmap file to disk */
1001 void md_bitmap_unplug(struct bitmap *bitmap)
1007 if (!bitmap || !bitmap->storage.filemap ||
1008 test_bit(BITMAP_STALE, &bitmap->flags))
1013 for (i = 0; i < bitmap->storage.file_pages; i++) {
1014 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1015 need_write = test_and_clear_page_attr(bitmap, i,
1019 md_bitmap_wait_writes(bitmap);
1020 if (bitmap->mddev->queue)
1021 blk_add_trace_msg(bitmap->mddev->queue,
1024 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
1025 write_page(bitmap, bitmap->storage.filemap[i], 0);
1030 md_bitmap_wait_writes(bitmap);
1032 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1033 md_bitmap_file_kick(bitmap);
1037 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1039 * the in-memory bitmap from the on-disk bitmap -- also, sets up the
1040 * memory mapping of the bitmap file
1042 * if there's no bitmap file, or if the bitmap file had been
1047 * This is used when reading an out-of-date bitmap...
1049 static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1059 struct bitmap_storage *store = &bitmap->storage;
1061 chunks = bitmap->counts.chunks;
1064 if (!file && !bitmap->mddev->bitmap_info.offset) {
1065 /* No permanent bitmap - fill with '1s'. */
1070 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
1072 md_bitmap_set_memory_bits(bitmap,
1073 (sector_t)i << bitmap->counts.chunkshift,
1079 outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
1081 pr_warn("%s: bitmap file is out of date, doing full recovery\n", bmname(bitmap));
1084 pr_warn("%s: bitmap file too short %lu < %lu\n",
1085 bmname(bitmap),
1093 if (!bitmap->mddev->bitmap_info.external)
1096 if (mddev_is_clustered(bitmap->mddev))
1097 node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
1101 index = file_page_index(&bitmap->storage, i);
1102 bit = file_page_offset(&bitmap->storage, i);
1112 ret = read_page(file, index, bitmap,
1116 bitmap->mddev,
1117 bitmap->mddev->bitmap_info.offset,
1128 * if bitmap is out of date, dirty the
1135 write_page(bitmap, page, 1);
1139 &bitmap->flags))
1144 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1151 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1153 md_bitmap_set_memory_bits(bitmap,
1154 (sector_t)i << bitmap->counts.chunkshift,
1161 pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
1162 bmname(bitmap), store->file_pages,
1168 pr_warn("%s: bitmap initialisation failed: %d\n",
1169 bmname(bitmap), ret);
1173 void md_bitmap_write_all(struct bitmap *bitmap)
1175 /* We don't actually write all bitmap blocks here,
1180 if (!bitmap || !bitmap->storage.filemap)
1182 if (bitmap->storage.file)
1186 for (i = 0; i < bitmap->storage.file_pages; i++)
1187 set_page_attr(bitmap, i,
1189 bitmap->allclean = 0;
1192 static void md_bitmap_count_page(struct bitmap_counts *bitmap,
1195 sector_t chunk = offset >> bitmap->chunkshift;
1197 bitmap->bp[page].count += inc;
1198 md_bitmap_checkfree(bitmap, page);
1201 static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1203 sector_t chunk = offset >> bitmap->chunkshift;
1205 struct bitmap_page *bp = &bitmap->bp[page];
1211 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1216 * bitmap daemon -- periodically wakes up to clean bits and flush pages
1222 struct bitmap *bitmap;
1232 bitmap = mddev->bitmap;
1233 if (bitmap == NULL) {
1237 if (time_before(jiffies, bitmap->daemon_lastrun
1241 bitmap->daemon_lastrun = jiffies;
1242 if (bitmap->allclean) {
1246 bitmap->allclean = 1;
1248 if (bitmap->mddev->queue)
1249 blk_add_trace_msg(bitmap->mddev->queue,
1256 for (j = 0; j < bitmap->storage.file_pages; j++)
1257 if (test_and_clear_page_attr(bitmap, j,
1259 set_page_attr(bitmap, j,
1262 if (bitmap->need_sync &&
1267 bitmap->need_sync = 0;
1268 if (bitmap->storage.filemap) {
1269 sb = kmap_atomic(bitmap->storage.sb_page);
1271 cpu_to_le64(bitmap->events_cleared);
1273 set_page_attr(bitmap, 0,
1277 /* Now look at the bitmap counters and if any are '2' or '1',
1280 counts = &bitmap->counts;
1301 if (*bmc == 1 && !bitmap->need_sync) {
1305 md_bitmap_file_clear_bit(bitmap, block);
1309 bitmap->allclean = 0;
1314 md_bitmap_wait_writes(bitmap);
1324 j < bitmap->storage.file_pages
1325 && !test_bit(BITMAP_STALE, &bitmap->flags);
1327 if (test_page_attr(bitmap, j,
1331 if (bitmap->storage.filemap &&
1332 test_and_clear_page_attr(bitmap, j,
1334 write_page(bitmap, bitmap->storage.filemap[j], 0);
1339 if (bitmap->allclean == 0)
1345 static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
1348 __releases(bitmap->lock)
1349 __acquires(bitmap->lock)
1355 sector_t chunk = offset >> bitmap->chunkshift;
1361 if (page >= bitmap->pages) {
1369 err = md_bitmap_checkpage(bitmap, page, create, 0);
1371 if (bitmap->bp[page].hijacked ||
1372 bitmap->bp[page].map == NULL)
1373 csize = ((sector_t)1) << (bitmap->chunkshift +
1376 csize = ((sector_t)1) << bitmap->chunkshift;
1384 if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1389 &bitmap->bp[page].map)[hi];
1392 &(bitmap->bp[page].map[pageoff]);
1395 int md_bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1397 if (!bitmap)
1402 atomic_inc(&bitmap->behind_writes);
1403 bw = atomic_read(&bitmap->behind_writes);
1404 if (bw > bitmap->behind_writes_used)
1405 bitmap->behind_writes_used = bw;
1408 bw, bitmap->mddev->bitmap_info.max_write_behind);
1415 spin_lock_irq(&bitmap->counts.lock);
1416 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1418 spin_unlock_irq(&bitmap->counts.lock);
1428 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1430 spin_unlock_irq(&bitmap->counts.lock);
1432 finish_wait(&bitmap->overflow_wait, &__wait);
1438 md_bitmap_file_set_bit(bitmap, offset);
1439 md_bitmap_count_page(&bitmap->counts, offset, 1);
1447 spin_unlock_irq(&bitmap->counts.lock);
1459 void md_bitmap_endwrite(struct bitmap *bitmap, sector_t offset,
1462 if (!bitmap)
1465 if (atomic_dec_and_test(&bitmap->behind_writes))
1466 wake_up(&bitmap->behind_wait);
1468 atomic_read(&bitmap->behind_writes),
1469 bitmap->mddev->bitmap_info.max_write_behind);
1477 spin_lock_irqsave(&bitmap->counts.lock, flags);
1478 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1480 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1484 if (success && !bitmap->mddev->degraded &&
1485 bitmap->events_cleared < bitmap->mddev->events) {
1486 bitmap->events_cleared = bitmap->mddev->events;
1487 bitmap->need_sync = 1;
1488 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1495 wake_up(&bitmap->overflow_wait);
1499 md_bitmap_set_pending(&bitmap->counts, offset);
1500 bitmap->allclean = 0;
1502 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1512 static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1517 if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
1519 return 1; /* always resync if no bitmap */
1521 spin_lock_irq(&bitmap->counts.lock);
1522 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1536 spin_unlock_irq(&bitmap->counts.lock);
1540 int md_bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1555 rv |= __bitmap_start_sync(bitmap, offset,
1564 void md_bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1569 if (bitmap == NULL) {
1573 spin_lock_irqsave(&bitmap->counts.lock, flags);
1574 bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1585 md_bitmap_set_pending(&bitmap->counts, offset);
1586 bitmap->allclean = 0;
1591 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1595 void md_bitmap_close_sync(struct bitmap *bitmap)
1597 /* Sync has finished, and any bitmap chunks that weren't synced
1603 if (!bitmap)
1605 while (sector < bitmap->mddev->resync_max_sectors) {
1606 md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1612 void md_bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
1617 if (!bitmap)
1620 bitmap->last_end_sync = jiffies;
1623 if (!force && time_before(jiffies, (bitmap->last_end_sync
1624 + bitmap->mddev->bitmap_info.daemon_sleep)))
1626 wait_event(bitmap->mddev->recovery_wait,
1627 atomic_read(&bitmap->mddev->recovery_active) == 0);
1629 bitmap->mddev->curr_resync_completed = sector;
1630 set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
1631 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1633 while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1634 md_bitmap_end_sync(bitmap, s, &blocks, 0);
1637 bitmap->last_end_sync = jiffies;
1638 sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed);
1646 struct bitmap *bitmap = mddev->bitmap;
1650 md_bitmap_end_sync(bitmap, sector, &blocks, 0);
1656 md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1663 static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1672 spin_lock_irq(&bitmap->counts.lock);
1673 bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1675 spin_unlock_irq(&bitmap->counts.lock);
1680 md_bitmap_count_page(&bitmap->counts, offset, 1);
1681 md_bitmap_set_pending(&bitmap->counts, offset);
1682 bitmap->allclean = 0;
1686 spin_unlock_irq(&bitmap->counts.lock);
1689 /* dirty the memory and file bits for bitmap chunks "s" to "e" */
1690 void md_bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1695 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1696 md_bitmap_set_memory_bits(bitmap, sec, 1);
1697 md_bitmap_file_set_bit(bitmap, sec);
1698 if (sec < bitmap->mddev->recovery_cp)
1703 bitmap->mddev->recovery_cp = sec;
1712 struct bitmap *bitmap = mddev->bitmap;
1715 if (!bitmap) /* there was no bitmap */
1722 bitmap->daemon_lastrun -= sleep;
1724 bitmap->daemon_lastrun -= sleep;
1726 bitmap->daemon_lastrun -= sleep;
1730 md_bitmap_update_sb(bitmap);
1736 void md_bitmap_free(struct bitmap *bitmap)
1741 if (!bitmap) /* there was no bitmap */
1744 if (bitmap->sysfs_can_clear)
1745 sysfs_put(bitmap->sysfs_can_clear);
1747 if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
1748 bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
1749 md_cluster_stop(bitmap->mddev);
1752 wait_event(bitmap->write_wait,
1753 atomic_read(&bitmap->pending_writes) == 0);
1755 /* release the bitmap file */
1756 md_bitmap_file_unmap(&bitmap->storage);
1758 bp = bitmap->counts.bp;
1759 pages = bitmap->counts.pages;
1768 kfree(bitmap);
1774 struct bitmap *bitmap = mddev->bitmap;
1777 if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
1781 wait_event(bitmap->behind_wait,
1782 atomic_read(&bitmap->behind_writes) == 0);
1788 struct bitmap *bitmap = mddev->bitmap;
1790 if (!bitmap) /* there was no bitmap */
1799 mddev->bitmap = NULL; /* disconnect from the md device */
1805 md_bitmap_free(bitmap);
1809 * initialize the bitmap structure
1811 * once mddev->bitmap is set
1813 struct bitmap *md_bitmap_create(struct mddev *mddev, int slot)
1815 struct bitmap *bitmap;
1826 pr_notice("md/raid:%s: array with journal cannot have bitmap\n",
1831 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1832 if (!bitmap)
1835 spin_lock_init(&bitmap->counts.lock);
1836 atomic_set(&bitmap->pending_writes, 0);
1837 init_waitqueue_head(&bitmap->write_wait);
1838 init_waitqueue_head(&bitmap->overflow_wait);
1839 init_waitqueue_head(&bitmap->behind_wait);
1841 bitmap->mddev = mddev;
1842 bitmap->cluster_slot = slot;
1845 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
1847 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
1850 bitmap->sysfs_can_clear = NULL;
1852 bitmap->storage.file = file;
1861 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1865 * instructing us to create a new on-disk bitmap instance.
1868 err = md_bitmap_new_disk_sb(bitmap);
1870 err = md_bitmap_read_sb(bitmap);
1882 bitmap->daemon_lastrun = jiffies;
1883 err = md_bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1887 pr_debug("created bitmap (%lu pages) for device %s\n",
1888 bitmap->counts.pages, bmname(bitmap));
1890 err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1894 return bitmap;
1896 md_bitmap_free(bitmap);
1905 struct bitmap *bitmap = mddev->bitmap;
1908 if (!bitmap)
1917 /* Clear out old bitmap info first: Either there is none, or we
1924 md_bitmap_start_sync(bitmap, sector, &blocks, 0);
1927 md_bitmap_close_sync(bitmap);
1930 || bitmap->events_cleared == mddev->events)
1936 err = md_bitmap_init_from_disk(bitmap, start);
1941 clear_bit(BITMAP_STALE, &bitmap->flags);
1944 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1949 md_bitmap_update_sb(bitmap);
1951 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1958 /* caller need to free returned bitmap with md_bitmap_free() */
1959 struct bitmap *get_bitmap_from_slot(struct mddev *mddev, int slot)
1962 struct bitmap *bitmap;
1964 bitmap = md_bitmap_create(mddev, slot);
1965 if (IS_ERR(bitmap)) {
1966 rv = PTR_ERR(bitmap);
1970 rv = md_bitmap_init_from_disk(bitmap, 0);
1972 md_bitmap_free(bitmap);
1976 return bitmap;
1980 /* Loads the bitmap associated with slot and copies the resync information
1981 * to our bitmap
1989 struct bitmap *bitmap;
1991 bitmap = get_bitmap_from_slot(mddev, slot);
1992 if (IS_ERR(bitmap)) {
1993 pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
1997 counts = &bitmap->counts;
2000 if (md_bitmap_file_test_bit(bitmap, block)) {
2004 md_bitmap_file_clear_bit(bitmap, block);
2005 md_bitmap_set_memory_bits(mddev->bitmap, block, 1);
2006 md_bitmap_file_set_bit(mddev->bitmap, block);
2011 md_bitmap_update_sb(bitmap);
2014 for (i = 0; i < bitmap->storage.file_pages; i++)
2015 if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING))
2016 set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
2017 md_bitmap_unplug(bitmap);
2019 md_bitmap_unplug(mddev->bitmap);
2022 md_bitmap_free(bitmap);
2029 void md_bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
2034 if (!bitmap)
2037 counts = &bitmap->counts;
2039 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
2040 seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
2046 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
2048 if (bitmap->storage.file) {
2050 seq_file_path(seq, bitmap->storage.file, " \t\n");
2056 int md_bitmap_resize(struct bitmap *bitmap, sector_t blocks,
2061 * Then quiesce, copy bits, replace bitmap, and re-start
2063 * This function is called both to set up the initial bitmap
2064 * and to resize the bitmap while the array is active.
2079 if (bitmap->storage.file && !init) {
2080 pr_info("md: cannot resize file-based bitmap\n");
2089 long space = bitmap->mddev->bitmap_info.space;
2095 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
2096 if (!bitmap->mddev->bitmap_info.external)
2099 bitmap->mddev->bitmap_info.space = space;
2101 chunkshift = bitmap->counts.chunkshift;
2108 if (!bitmap->mddev->bitmap_info.external)
2117 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
2119 !bitmap->mddev->bitmap_info.external,
2120 mddev_is_clustered(bitmap->mddev)
2121 ? bitmap->cluster_slot : 0);
2137 bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
2139 store.file = bitmap->storage.file;
2140 bitmap->storage.file = NULL;
2142 if (store.sb_page && bitmap->storage.sb_page)
2144 page_address(bitmap->storage.sb_page),
2146 spin_lock_irq(&bitmap->counts.lock);
2147 md_bitmap_file_unmap(&bitmap->storage);
2148 bitmap->storage = store;
2150 old_counts = bitmap->counts;
2151 bitmap->counts.bp = new_bp;
2152 bitmap->counts.pages = pages;
2153 bitmap->counts.missing_pages = pages;
2154 bitmap->counts.chunkshift = chunkshift;
2155 bitmap->counts.chunks = chunks;
2156 bitmap->mddev->bitmap_info.chunksize = 1UL << (chunkshift +
2162 /* For cluster raid, need to pre-allocate bitmap */
2163 if (mddev_is_clustered(bitmap->mddev)) {
2166 ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1);
2177 bitmap->counts.bp = old_counts.bp;
2178 bitmap->counts.pages = old_counts.pages;
2179 bitmap->counts.missing_pages = old_counts.pages;
2180 bitmap->counts.chunkshift = old_counts.chunkshift;
2181 bitmap->counts.chunks = old_counts.chunks;
2182 bitmap->mddev->bitmap_info.chunksize =
2185 pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
2188 bitmap->counts.bp[page].count += 1;
2200 bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2209 md_bitmap_file_set_bit(bitmap, block);
2213 md_bitmap_count_page(&bitmap->counts, block, 1);
2214 md_bitmap_set_pending(&bitmap->counts, block);
2224 if (bitmap->counts.bp != old_counts.bp) {
2236 bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
2243 md_bitmap_count_page(&bitmap->counts, block, 1);
2244 md_bitmap_set_pending(&bitmap->counts, block);
2249 for (i = 0; i < bitmap->storage.file_pages; i++)
2250 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
2252 spin_unlock_irq(&bitmap->counts.lock);
2255 md_bitmap_unplug(bitmap);
2256 bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
2297 if (mddev->bitmap || mddev->bitmap_info.file ||
2299 /* bitmap already configured. Only option is to clear it */
2316 /* No bitmap, OK to set a location */
2343 struct bitmap *bitmap;
2344 bitmap = md_bitmap_create(mddev, -1);
2346 if (IS_ERR(bitmap))
2347 rv = PTR_ERR(bitmap);
2349 mddev->bitmap = bitmap;
2364 /* Ensure new bitmap info is stored in
2381 /* 'bitmap/space' is the space available at 'location' for the
2382 * bitmap. This allows the kernel to know when it is safe to
2383 * resize the bitmap to match a resized array.
2404 if (mddev->bitmap &&
2405 sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2454 * the bitmap is all clean and we don't need to
2519 md_bitmap_update_sb(mddev->bitmap);
2537 /* Can only be changed when no bitmap is active */
2540 if (mddev->bitmap)
2568 if (mddev->bitmap ||
2589 if (mddev->bitmap)
2590 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2600 if (mddev->bitmap == NULL)
2603 mddev->bitmap->need_sync = 1;
2607 mddev->bitmap->need_sync = 0;
2621 if (mddev->bitmap == NULL)
2625 mddev->bitmap->behind_writes_used);
2633 if (mddev->bitmap)
2634 mddev->bitmap->behind_writes_used = 0;
2654 .name = "bitmap",