Lines Matching defs:class
143 * determined). NOTE: all those class sizes must be set as multiple of
204 * Size of objects stored in this class. Must be multiple
282 unsigned int class:CLASS_BITS + 1;
470 /* Protected by class->lock */
517 *class_idx = zspage->class;
524 zspage->class = class_idx;
530 * class maintains a list of zspages where each zspage is divided
533 * size class which has chunk size big enough to hold the give size.
547 static inline void zs_stat_inc(struct size_class *class,
550 class->stats.objs[type] += cnt;
554 static inline void zs_stat_dec(struct size_class *class,
557 class->stats.objs[type] -= cnt;
561 static inline unsigned long zs_stat_get(struct size_class *class,
564 return class->stats.objs[type];
584 static unsigned long zs_can_compact(struct size_class *class);
590 struct size_class *class;
599 "class", "size", "almost_full", "almost_empty",
604 class = pool->size_class[i];
606 if (class->index != i)
609 spin_lock(&class->lock);
610 class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL);
611 class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY);
612 obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
613 obj_used = zs_stat_get(class, OBJ_USED);
614 freeable = zs_can_compact(class);
615 spin_unlock(&class->lock);
617 objs_per_zspage = class->objs_per_zspage;
619 class->pages_per_zspage;
623 i, class->size, class_almost_full, class_almost_empty,
625 class->pages_per_zspage, freeable);
683 * For each size class, zspages are divided into different groups
689 static enum fullness_group get_fullness_group(struct size_class *class,
696 objs_per_zspage = class->objs_per_zspage;
711 * Each size class maintains various freelists and zspages are assigned
714 * identified by <class, fullness_group>.
716 static void insert_zspage(struct size_class *class,
722 zs_stat_inc(class, fullness, 1);
723 head = list_first_entry_or_null(&class->fullness_list[fullness],
735 list_add(&zspage->list, &class->fullness_list[fullness]);
740 * by <class, fullness_group>.
742 static void remove_zspage(struct size_class *class,
746 VM_BUG_ON(list_empty(&class->fullness_list[fullness]));
750 zs_stat_dec(class, fullness, 1);
754 * Each size class maintains zspages in different fullness groups depending
762 static enum fullness_group fix_fullness_group(struct size_class *class,
769 newfg = get_fullness_group(class, zspage);
774 remove_zspage(class, zspage, currfg);
775 insert_zspage(class, zspage, newfg);
786 * to form a zspage for each size class. This is important
793 * For example, for size class of 3/8 * PAGE_SIZE, we should
931 static void __free_zspage(struct zs_pool *pool, struct size_class *class,
940 assert_spin_locked(&class->lock);
958 zs_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage);
959 atomic_long_sub(class->pages_per_zspage,
963 static void free_zspage(struct zs_pool *pool, struct size_class *class,
974 remove_zspage(class, zspage, ZS_EMPTY);
975 __free_zspage(pool, class, zspage);
979 static void init_zspage(struct size_class *class, struct zspage *zspage)
995 while ((off += class->size) < PAGE_SIZE) {
997 link += class->size / sizeof(*link);
1023 static void create_page_chain(struct size_class *class, struct zspage *zspage,
1029 int nr_pages = class->pages_per_zspage;
1046 if (unlikely(class->objs_per_zspage == 1 &&
1047 class->pages_per_zspage == 1))
1057 * Allocate a zspage for the given size class
1060 struct size_class *class,
1074 for (i = 0; i < class->pages_per_zspage; i++) {
1094 create_page_chain(class, zspage, pages);
1095 init_zspage(class, zspage);
1100 static struct zspage *find_get_zspage(struct size_class *class)
1106 zspage = list_first_entry_or_null(&class->fullness_list[i],
1222 static bool zspage_full(struct size_class *class, struct zspage *zspage)
1224 return get_zspage_inuse(zspage) == class->objs_per_zspage;
1258 struct size_class *class;
1281 class = pool->size_class[class_idx];
1282 off = (class->size * obj_idx) & ~PAGE_MASK;
1286 if (off + class->size <= PAGE_SIZE) {
1298 ret = __zs_map_object(area, pages, off, class->size);
1316 struct size_class *class;
1323 class = pool->size_class[class_idx];
1324 off = (class->size * obj_idx) & ~PAGE_MASK;
1327 if (off + class->size <= PAGE_SIZE)
1336 __zs_unmap_object(area, pages, off, class->size);
1350 * The function returns the size of the first huge class - any object of equal
1364 static unsigned long obj_malloc(struct size_class *class,
1378 offset = obj * class->size;
1398 zs_stat_inc(class, OBJ_USED, 1);
1419 struct size_class *class;
1432 class = pool->size_class[get_size_class_index(size)];
1434 spin_lock(&class->lock);
1435 zspage = find_get_zspage(class);
1437 obj = obj_malloc(class, zspage, handle);
1439 fix_fullness_group(class, zspage);
1441 spin_unlock(&class->lock);
1446 spin_unlock(&class->lock);
1448 zspage = alloc_zspage(pool, class, gfp);
1454 spin_lock(&class->lock);
1455 obj = obj_malloc(class, zspage, handle);
1456 newfg = get_fullness_group(class, zspage);
1457 insert_zspage(class, zspage, newfg);
1458 set_zspage_mapping(zspage, class->index, newfg);
1460 atomic_long_add(class->pages_per_zspage,
1462 zs_stat_inc(class, OBJ_ALLOCATED, class->objs_per_zspage);
1466 spin_unlock(&class->lock);
1472 static void obj_free(struct size_class *class, unsigned long obj)
1483 f_offset = (class->size * f_objidx) & ~PAGE_MASK;
1494 zs_stat_dec(class, OBJ_USED, 1);
1504 struct size_class *class;
1519 class = pool->size_class[class_idx];
1521 spin_lock(&class->lock);
1522 obj_free(class, obj);
1523 fullness = fix_fullness_group(class, zspage);
1533 free_zspage(pool, class, zspage);
1536 spin_unlock(&class->lock);
1542 static void zs_object_copy(struct size_class *class, unsigned long dst,
1552 s_size = d_size = class->size;
1557 s_off = (class->size * s_objidx) & ~PAGE_MASK;
1558 d_off = (class->size * d_objidx) & ~PAGE_MASK;
1560 if (s_off + class->size > PAGE_SIZE)
1563 if (d_off + class->size > PAGE_SIZE)
1574 if (written == class->size)
1588 s_size = class->size - written;
1596 d_size = class->size - written;
1609 static unsigned long find_alloced_obj(struct size_class *class,
1619 offset += class->size * index;
1630 offset += class->size;
1652 static int migrate_zspage(struct zs_pool *pool, struct size_class *class,
1663 handle = find_alloced_obj(class, s_page, &obj_idx);
1673 if (zspage_full(class, get_zspage(d_page))) {
1680 free_obj = obj_malloc(class, get_zspage(d_page), handle);
1681 zs_object_copy(class, free_obj, used_obj);
1692 obj_free(class, used_obj);
1702 static struct zspage *isolate_zspage(struct size_class *class, bool source)
1714 zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],
1718 remove_zspage(class, zspage, fg[i]);
1727 * putback_zspage - add @zspage into right class's fullness list
1728 * @class: destination class
1733 static enum fullness_group putback_zspage(struct size_class *class,
1740 fullness = get_fullness_group(class, zspage);
1741 insert_zspage(class, zspage, fullness);
1742 set_zspage_mapping(zspage, class->index, fullness);
1854 struct size_class *class,
1859 fg = putback_zspage(class, zspage);
1879 static void replace_sub_page(struct size_class *class, struct zspage *zspage,
1895 create_page_chain(class, zspage, pages);
1905 struct size_class *class;
1921 * Without class lock, fullness could be stale while class_idx is okay
1923 * fullness again under class lock.
1928 class = pool->size_class[class_idx];
1930 spin_lock(&class->lock);
1932 spin_unlock(&class->lock);
1938 spin_unlock(&class->lock);
1949 remove_zspage(class, zspage, fullness);
1953 spin_unlock(&class->lock);
1962 struct size_class *class;
1991 class = pool->size_class[class_idx];
1994 spin_lock(&class->lock);
2012 pos += class->size;
2023 addr += class->size) {
2039 replace_sub_page(class, zspage, newpage, page);
2055 putback_zspage_deferred(pool, class, zspage);
2071 addr += class->size) {
2081 spin_unlock(&class->lock);
2090 struct size_class *class;
2103 class = pool->size_class[class_idx];
2105 spin_lock(&class->lock);
2112 putback_zspage_deferred(pool, class, zspage);
2115 spin_unlock(&class->lock);
2178 struct size_class *class;
2187 class = pool->size_class[i];
2188 if (class->index != i)
2191 spin_lock(&class->lock);
2192 list_splice_init(&class->fullness_list[ZS_EMPTY], &free_pages);
2193 spin_unlock(&class->lock);
2203 class = pool->size_class[class_idx];
2204 spin_lock(&class->lock);
2206 spin_unlock(&class->lock);
2237 static unsigned long zs_can_compact(struct size_class *class)
2240 unsigned long obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
2241 unsigned long obj_used = zs_stat_get(class, OBJ_USED);
2247 obj_wasted /= class->objs_per_zspage;
2249 return obj_wasted * class->pages_per_zspage;
2253 struct size_class *class)
2260 spin_lock(&class->lock);
2261 while ((src_zspage = isolate_zspage(class, true))) {
2263 if (!zs_can_compact(class))
2269 while ((dst_zspage = isolate_zspage(class, false))) {
2275 if (!migrate_zspage(pool, class, &cc))
2278 putback_zspage(class, dst_zspage);
2285 putback_zspage(class, dst_zspage);
2286 if (putback_zspage(class, src_zspage) == ZS_EMPTY) {
2287 free_zspage(pool, class, src_zspage);
2288 pages_freed += class->pages_per_zspage;
2290 spin_unlock(&class->lock);
2292 spin_lock(&class->lock);
2296 putback_zspage(class, src_zspage);
2298 spin_unlock(&class->lock);
2306 struct size_class *class;
2310 class = pool->size_class[i];
2311 if (!class)
2313 if (class->index != i)
2315 pages_freed += __zs_compact(pool, class);
2350 struct size_class *class;
2356 class = pool->size_class[i];
2357 if (!class)
2359 if (class->index != i)
2362 pages_to_free += zs_can_compact(class);
2424 struct size_class *class;
2436 * class. Any object bigger than or equal to that will
2437 * endup in the huge class.
2446 * size class search - so object may be smaller than
2447 * huge class size, yet it still can end up in the huge
2448 * class because it grows by ZS_HANDLE_SIZE extra bytes
2449 * right before class lookup.
2470 class = kzalloc(sizeof(struct size_class), GFP_KERNEL);
2471 if (!class)
2474 class->size = size;
2475 class->index = i;
2476 class->pages_per_zspage = pages_per_zspage;
2477 class->objs_per_zspage = objs_per_zspage;
2478 spin_lock_init(&class->lock);
2479 pool->size_class[i] = class;
2482 INIT_LIST_HEAD(&class->fullness_list[fullness]);
2484 prev_class = class;
2519 struct size_class *class = pool->size_class[i];
2521 if (!class)
2524 if (class->index != i)
2528 if (!list_empty(&class->fullness_list[fg])) {
2529 pr_info("Freeing non-empty class with size %db, fullness group %d\n",
2530 class->size, fg);
2533 kfree(class);