1// SPDX-License-Identifier: GPL-2.0
2/*
3 * drivers/block/zram/zram_group/group_writeback.c
4 *
5 * Copyright (c) 2020-2022 Huawei Technologies Co., Ltd.
6 */
7
8#include <linux/mm.h>
9#include <linux/memcontrol.h>
10#include <linux/blk_types.h>
11#include <linux/zswapd.h>
12
13#include "../zram_drv.h"
14#include "zram_group.h"
15
16#ifdef CONFIG_HYPERHOLD
17#include "hyperhold.h"
18#endif
19
20#define CHECK(cond, ...) ((cond) || (pr_err(__VA_ARGS__), false))
21#define CHECK_BOUND(var, min, max) \
22	CHECK((var) >= (min) && (var) <= (max), \
23			"%s %u out of bounds %u ~ %u!\n", \
24			#var, (var), (min), (max))
25
26static u16 zram_get_memcg_id(struct zram *zram, u32 index)
27{
28	return (zram->table[index].flags & ZRAM_GRPID_MASK) >> ZRAM_SIZE_SHIFT;
29}
30
31static void zram_set_memcg_id(struct zram *zram, u32 index, u16 gid)
32{
33	unsigned long old = zram->table[index].flags & (~ZRAM_GRPID_MASK);
34
35	zram->table[index].flags = old | ((u64)gid << ZRAM_SIZE_SHIFT);
36}
37
38#ifdef CONFIG_ZRAM_GROUP_WRITEBACK
39static bool obj_can_wb(struct zram *zram, u32 index, u16 gid)
40{
41	/* overwrited obj, just skip */
42	if (zram_get_memcg_id(zram, index) != gid) {
43		pr_debug("obj %u is from group %u instead of group %u.\n",
44				index, zram_get_memcg_id(zram, index), gid);
45		return false;
46	}
47	if (!zgrp_obj_is_isolated(zram->zgrp, index)) {
48		pr_debug("obj %u is not isolated.\n", index);
49		return false;
50	}
51	/* need not to writeback, put back the obj as HOTEST */
52	if (zram_test_flag(zram, index, ZRAM_SAME)) {
53		pr_debug("obj %u is filled with same element.\n", index);
54		goto insert;
55	}
56	if (zram_test_flag(zram, index, ZRAM_WB)) {
57		pr_debug("obj %u is writeback.\n", index);
58		goto insert;
59	}
60	/* obj is needed by a pagefault req, do not writeback it. */
61	if (zram_test_flag(zram, index, ZRAM_FAULT)) {
62		pr_debug("obj %u is needed by a pagefault request.\n", index);
63		goto insert;
64	}
65	/* should never happen */
66	if (zram_test_flag(zram, index, ZRAM_GWB)) {
67		pr_debug("obj %u is group writeback.\n", index);
68		BUG();
69		return false;
70	}
71
72	return true;
73insert:
74	zgrp_obj_insert(zram->zgrp, index, gid);
75
76	return false;
77}
78
79static void copy_obj(struct hpio *hpio, u32 offset, char *obj, u32 size, bool to)
80{
81	u32 page_id, start;
82	char *buf = NULL;
83
84	page_id = offset / PAGE_SIZE;
85	start = offset % PAGE_SIZE;
86	if (size + start <= PAGE_SIZE) {
87		buf = page_to_virt(hyperhold_io_page(hpio, page_id));
88		if (to)
89			memcpy(buf + start, obj, size);
90		else
91			memcpy(obj, buf + start, size);
92
93		return;
94	}
95	buf = page_to_virt(hyperhold_io_page(hpio, page_id));
96	if (to)
97		memcpy(buf + start, obj, PAGE_SIZE - start);
98	else
99		memcpy(obj, buf + start, PAGE_SIZE - start);
100	buf = page_to_virt(hyperhold_io_page(hpio, page_id + 1));
101	if (to)
102		memcpy(buf, obj + PAGE_SIZE - start, size + start - PAGE_SIZE);
103	else
104		memcpy(obj + PAGE_SIZE - start, buf, size + start - PAGE_SIZE);
105}
106
107static u32 move_obj_to_hpio(struct zram *zram, u32 index, u16 gid,
108				struct hpio *hpio, u32 offset)
109{
110	u32 size = 0;
111	unsigned long handle;
112	char *src = NULL;
113	u32 ext_size;
114	u32 eid;
115
116	eid = hyperhold_io_extent(hpio);
117	ext_size = hyperhold_extent_size(eid);
118
119	zram_slot_lock(zram, index);
120	if (!obj_can_wb(zram, index, gid))
121		goto unlock;
122	size = zram_get_obj_size(zram, index);
123	/* no space, put back the obj as COLDEST */
124	if (size + offset > ext_size) {
125		pr_debug("obj %u size is %u, but ext %u only %u space left.\n",
126				index, size, eid, ext_size - offset);
127		zgrp_obj_putback(zram->zgrp, index, gid);
128		size = 0;
129		goto unlock;
130	}
131	handle = zram_get_handle(zram, index);
132	src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO);
133	copy_obj(hpio, offset, src, size, true);
134	zs_unmap_object(zram->mem_pool, handle);
135	zs_free(zram->mem_pool, handle);
136	zram_set_handle(zram, index, hyperhold_address(eid, offset));
137	zram_set_flag(zram, index, ZRAM_GWB);
138	wbgrp_obj_insert(zram->zgrp, index, eid);
139	wbgrp_obj_stats_inc(zram->zgrp, gid, eid, size);
140	zgrp_obj_stats_dec(zram->zgrp, gid, size);
141	pr_debug("move obj %u of group %u to hpio %p of eid %u, size = %u, offset = %u\n",
142		index, gid, hpio, eid, size, offset);
143unlock:
144	zram_slot_unlock(zram, index);
145
146	return size;
147}
148
149static void move_obj_from_hpio(struct zram *zram, int index, struct hpio *hpio)
150{
151	u32 size = 0;
152	unsigned long handle = 0;
153	u32 eid, offset;
154	u64 addr;
155	char *dst = NULL;
156	u16 gid;
157
158	eid = hyperhold_io_extent(hpio);
159retry:
160	zram_slot_lock(zram, index);
161	if (!zram_test_flag(zram, index, ZRAM_GWB))
162		goto unlock;
163	addr = zram_get_handle(zram, index);
164	if (hyperhold_addr_extent(addr) != eid)
165		goto unlock;
166	size = zram_get_obj_size(zram, index);
167	if (handle)
168		goto move;
169	handle = zs_malloc(zram->mem_pool, size, GFP_NOWAIT);
170	if (handle)
171		goto move;
172	zram_slot_unlock(zram, index);
173	handle = zs_malloc(zram->mem_pool, size, GFP_NOIO | __GFP_NOFAIL);
174	if (handle)
175		goto retry;
176	BUG();
177
178	return;
179move:
180	offset = hyperhold_addr_offset(addr);
181	dst = zs_map_object(zram->mem_pool, handle, ZS_MM_WO);
182	copy_obj(hpio, offset, dst, size, false);
183	zs_unmap_object(zram->mem_pool, handle);
184	zram_set_handle(zram, index, handle);
185	zram_clear_flag(zram, index, ZRAM_GWB);
186	gid = zram_get_memcg_id(zram, index);
187	zgrp_obj_insert(zram->zgrp, index, gid);
188	wbgrp_obj_stats_dec(zram->zgrp, gid, eid, size);
189	zgrp_obj_stats_inc(zram->zgrp, gid, size);
190	pr_debug("move obj %u of group %u from hpio %p of eid %u, size = %u, offset = %u\n",
191		index, gid, hpio, eid, size, offset);
192unlock:
193	zram_slot_unlock(zram, index);
194}
195
196
197#define NR_ISOLATE 32
198static bool move_extent_from_hpio(struct zram *zram, struct hpio *hpio)
199{
200	u32 idxs[NR_ISOLATE];
201	u32 eid;
202	u32 nr;
203	int i;
204	bool last = false;
205
206	eid = hyperhold_io_extent(hpio);
207repeat:
208	nr = wbgrp_isolate_objs(zram->zgrp, eid, idxs, NR_ISOLATE, &last);
209	for (i = 0; i < nr; i++)
210		move_obj_from_hpio(zram, idxs[i], hpio);
211	if (last)
212		return true;
213	if (nr)
214		goto repeat;
215
216	return false;
217}
218
219struct hpio_priv {
220	struct zram *zram;
221	u16 gid;
222};
223
224static void write_endio(struct hpio *hpio)
225{
226	struct hpio_priv *priv = hyperhold_io_private(hpio);
227	struct zram *zram = priv->zram;
228	u16 gid = priv->gid;
229	u32 eid = hyperhold_io_extent(hpio);
230
231	if (hyperhold_io_success(hpio))
232		goto out;
233	if (move_extent_from_hpio(zram, hpio)) {
234		zgrp_ext_delete(zram->zgrp, eid, gid);
235		hyperhold_should_free_extent(eid);
236	}
237out:
238	hyperhold_io_complete(hpio);
239	hyperhold_io_put(hpio);
240	kfree(priv);
241}
242
243static u32 collect_objs(struct zram *zram, u16 gid, struct hpio *hpio, u32 ext_size)
244{
245	u32 offset = 0;
246	u32 last_offset;
247	u32 nr;
248	u32 idxs[NR_ISOLATE];
249	int i;
250
251more:
252	last_offset = offset;
253	nr = zgrp_isolate_objs(zram->zgrp, gid, idxs, NR_ISOLATE, NULL);
254	for (i = 0; i < nr; i++)
255		offset += move_obj_to_hpio(zram, idxs[i], gid, hpio, offset);
256	pr_debug("%u data attached, offset = %u.\n", offset - last_offset, offset);
257	if (offset < ext_size && offset != last_offset)
258		goto more;
259
260	return offset;
261}
262
263static u64 write_one_extent(struct zram *zram, u16 gid)
264{
265	int eid;
266	struct hpio *hpio = NULL;
267	struct hpio_priv *priv = NULL;
268	u32 size = 0;
269	int ret;
270
271	priv = kmalloc(sizeof(struct hpio_priv), GFP_NOIO);
272	if (!priv)
273		return 0;
274	priv->gid = gid;
275	priv->zram = zram;
276	eid = hyperhold_alloc_extent();
277	if (eid < 0)
278		goto err;
279	hpio = hyperhold_io_get(eid, GFP_NOIO, REQ_OP_WRITE);
280	if (!hpio)
281		goto free_extent;
282
283	zgrp_get_ext(zram->zgrp, eid);
284	size = collect_objs(zram, gid, hpio, hyperhold_extent_size(eid));
285	if (size == 0) {
286		pr_err("group %u has no data in zram.\n", gid);
287		zgrp_put_ext(zram->zgrp, eid);
288		goto put_hpio;
289	}
290	zgrp_ext_insert(zram->zgrp, eid, gid);
291	if (zgrp_put_ext(zram->zgrp, eid)) {
292		zgrp_ext_delete(zram->zgrp, eid, gid);
293		hyperhold_should_free_extent(eid);
294	}
295
296	ret = hyperhold_write_async(hpio, write_endio, priv);
297	if (ret)
298		goto move_back;
299
300	return size;
301move_back:
302	if (move_extent_from_hpio(zram, hpio)) {
303		zgrp_ext_delete(zram->zgrp, eid, gid);
304		hyperhold_should_free_extent(eid);
305	}
306	eid = -EINVAL;
307put_hpio:
308	hyperhold_io_put(hpio);
309free_extent:
310	if (eid >= 0)
311		hyperhold_free_extent(eid);
312err:
313	kfree(priv);
314
315	return 0;
316}
317
318static void read_endio(struct hpio *hpio)
319{
320	struct hpio_priv *priv = hyperhold_io_private(hpio);
321	struct zram *zram = priv->zram;
322	u16 gid = priv->gid;
323	u32 eid = hyperhold_io_extent(hpio);
324
325	if (!hyperhold_io_success(hpio)) {
326		BUG();
327		goto out;
328	}
329	if (move_extent_from_hpio(zram, hpio)) {
330		zgrp_ext_delete(zram->zgrp, eid, gid);
331		hyperhold_should_free_extent(eid);
332	}
333out:
334	hyperhold_io_complete(hpio);
335	hyperhold_io_put(hpio);
336	kfree(priv);
337}
338
339static u64 read_one_extent(struct zram *zram, u32 eid, u16 gid)
340{
341	struct hpio *hpio = NULL;
342	u32 ext_size = 0;
343	int ret;
344	struct hpio_priv *priv = NULL;
345
346	priv = kmalloc(sizeof(struct hpio_priv), GFP_NOIO);
347	if (!priv)
348		goto err;
349	priv->gid = gid;
350	priv->zram = zram;
351	hpio = hyperhold_io_get(eid, GFP_NOIO, REQ_OP_READ);
352	if (!hpio)
353		goto err;
354	ext_size = hyperhold_extent_size(eid);
355	ret = hyperhold_read_async(hpio, read_endio, priv);
356	if (ret)
357		goto err;
358
359	return ext_size;
360err:
361	hyperhold_io_put(hpio);
362	kfree(priv);
363
364	return 0;
365}
366
367static void sync_read_endio(struct hpio *hpio)
368{
369	hyperhold_io_complete(hpio);
370}
371
372static int read_one_obj_sync(struct zram *zram, u32 index)
373{
374	struct hpio *hpio = NULL;
375	int ret;
376	u32 eid;
377	u16 gid;
378	u32 size;
379
380	if (!zram_test_flag(zram, index, ZRAM_GWB))
381		return 0;
382
383	pr_debug("read obj %u.\n", index);
384
385	gid = zram_get_memcg_id(zram, index);
386	eid = hyperhold_addr_extent(zram_get_handle(zram, index));
387	size = zram_get_obj_size(zram, index);
388	wbgrp_fault_stats_inc(zram->zgrp, gid, eid, size);
389check:
390	if (!zram_test_flag(zram, index, ZRAM_GWB))
391		return 0;
392	if (!zram_test_flag(zram, index, ZRAM_FAULT))
393		goto read;
394	zram_slot_unlock(zram, index);
395	wait_event(zram->zgrp->wbgrp.fault_wq, !zram_test_flag(zram, index, ZRAM_FAULT));
396	zram_slot_lock(zram, index);
397	goto check;
398read:
399	zram_set_flag(zram, index, ZRAM_FAULT);
400	zram_slot_unlock(zram, index);
401
402	hpio = hyperhold_io_get(eid, GFP_NOIO, REQ_OP_READ);
403	if (!hpio) {
404		ret = -ENOMEM;
405		goto out;
406	}
407	ret = hyperhold_read_async(hpio, sync_read_endio, NULL);
408	/* io submit error */
409	if (ret && ret != -EAGAIN)
410		goto out;
411
412	hyperhold_io_wait(hpio);
413
414	/* if not reset to zero, will return err sometimes and cause SIG_BUS error */
415	ret = 0;
416
417	/* get a write io, data is ready, copy the pages even write failed */
418	if (op_is_write(hyperhold_io_operate(hpio)))
419		goto move;
420	/* read io failed, return -EIO */
421	if (!hyperhold_io_success(hpio)) {
422		ret = -EIO;
423		goto out;
424	}
425	/* success, copy the data and free extent */
426move:
427	if (move_extent_from_hpio(zram, hpio)) {
428		zgrp_ext_delete(zram->zgrp, eid, gid);
429		hyperhold_should_free_extent(eid);
430	}
431	move_obj_from_hpio(zram, index, hpio);
432out:
433	hyperhold_io_put(hpio);
434	zram_slot_lock(zram, index);
435	zram_clear_flag(zram, index, ZRAM_FAULT);
436	wake_up(&zram->zgrp->wbgrp.fault_wq);
437
438	return ret;
439}
440
441u64 read_group_objs(struct zram *zram, u16 gid, u64 req_size)
442{
443	u32 eid;
444	u64 read_size = 0;
445	u32 nr;
446
447	if (!(zram->zgrp)) {
448		pr_debug("zram group is not enable!\n");
449		return 0;
450	}
451	if (!CHECK_BOUND(gid, 1, zram->zgrp->nr_grp - 1))
452		return 0;
453
454	pr_debug("read %llu data of group %u.\n", req_size, gid);
455
456	while (!req_size || req_size > read_size) {
457		nr = zgrp_isolate_exts(zram->zgrp, gid, &eid, 1, NULL);
458		if (!nr)
459			break;
460		read_size += read_one_extent(zram, eid, gid);
461	}
462
463	return read_size;
464}
465
466u64 write_group_objs(struct zram *zram, u16 gid, u64 req_size)
467{
468	u64 write_size = 0;
469	u64 size = 0;
470
471	if (!(zram->zgrp)) {
472		pr_debug("zram group is not enable!\n");
473		return 0;
474	}
475	if (!CHECK(zram->zgrp->wbgrp.enable, "zram group writeback is not enable!\n"))
476		return 0;
477	if (!CHECK_BOUND(gid, 1, zram->zgrp->nr_grp - 1))
478		return 0;
479
480	pr_debug("write %llu data of group %u.\n", req_size, gid);
481
482	while (!req_size || req_size > write_size) {
483		size = write_one_extent(zram, gid);
484		if (!size)
485			break;
486		write_size += size;
487	}
488
489	atomic64_add(write_size, &zram->zgrp->stats[0].write_size);
490	atomic64_add(write_size, &zram->zgrp->stats[gid].write_size);
491	return write_size;
492}
493#endif
494
495#ifdef CONFIG_ZRAM_GROUP_DEBUG
496#include <linux/random.h>
497#define ZGRP_TEST_MAX_GRP 101
498#endif
499
500int zram_group_fault_obj(struct zram *zram, u32 index)
501{
502	u16 gid;
503	u32 size;
504
505	if (!(zram->zgrp)) {
506		pr_debug("zram group is not enable!\n");
507		return 0;
508	}
509	if (!CHECK_BOUND(index, 0, zram->zgrp->nr_obj - 1))
510		return 0;
511
512	gid = zram_get_memcg_id(zram, index);
513	size = zram_get_obj_size(zram, index);
514	zgrp_fault_stats_inc(zram->zgrp, gid, size);
515#ifdef CONFIG_ZRAM_GROUP_WRITEBACK
516	return read_one_obj_sync(zram, index);
517#else
518	return 0;
519#endif
520}
521
522void zram_group_track_obj(struct zram *zram, u32 index, struct mem_cgroup *memcg)
523{
524	u16 gid;
525
526	if (!(zram->zgrp)) {
527		pr_debug("zram group is not enable!\n");
528		return;
529	}
530	if (!CHECK_BOUND(index, 0, zram->zgrp->nr_obj - 1))
531		return;
532	if (!CHECK(memcg || !memcg->id.id, "obj %u has no memcg!\n", index))
533		return;
534	gid = zram_get_memcg_id(zram, index);
535	if (!CHECK(!gid, "obj %u has gid %u.\n", index, gid))
536		BUG();
537
538	gid = memcg->id.id;
539	zram_set_memcg_id(zram, index, gid);
540	zgrp_obj_insert(zram->zgrp, index, gid);
541	zgrp_obj_stats_inc(zram->zgrp, gid, zram_get_obj_size(zram, index));
542}
543
544void zram_group_untrack_obj(struct zram *zram, u32 index)
545{
546	u16 gid;
547	u32 size;
548
549	if (!(zram->zgrp)) {
550		pr_debug("zram group is not enable!\n");
551		return;
552	}
553	if (!CHECK_BOUND(index, 0, zram->zgrp->nr_obj - 1))
554		return;
555
556#ifdef CONFIG_ZRAM_GROUP_WRITEBACK
557check:
558	if (!zram_test_flag(zram, index, ZRAM_FAULT))
559		goto clear;
560	zram_slot_unlock(zram, index);
561	wait_event(zram->zgrp->wbgrp.fault_wq, !zram_test_flag(zram, index, ZRAM_FAULT));
562	zram_slot_lock(zram, index);
563	goto check;
564clear:
565#endif
566	gid = zram_get_memcg_id(zram, index);
567	size = zram_get_obj_size(zram, index);
568	if (!gid)
569		return;
570#ifdef CONFIG_ZRAM_GROUP_WRITEBACK
571	if (zram_test_flag(zram, index, ZRAM_GWB)) {
572		u32 eid = hyperhold_addr_extent(zram_get_handle(zram, index));
573
574		if (wbgrp_obj_delete(zram->zgrp, index, eid)) {
575			zgrp_ext_delete(zram->zgrp, eid, gid);
576			hyperhold_should_free_extent(eid);
577		}
578		zram_clear_flag(zram, index, ZRAM_GWB);
579		zram_set_memcg_id(zram, index, 0);
580		wbgrp_obj_stats_dec(zram->zgrp, gid, eid, size);
581		zram_set_handle(zram, index, 0);
582		return;
583	}
584#endif
585	zgrp_obj_delete(zram->zgrp, index, gid);
586	zram_set_memcg_id(zram, index, 0);
587	zgrp_obj_stats_dec(zram->zgrp, gid, size);
588}
589
590#ifdef CONFIG_ZRAM_GROUP_DEBUG
591void group_debug(struct zram *zram, u32 op, u32 index, u32 gid)
592{
593	if (op == 0)
594		zram_group_dump(zram->zgrp, gid, index);
595
596#ifdef CONFIG_ZRAM_GROUP_WRITEBACK
597	if (op == 22)
598		read_group_objs(zram, gid, index);
599	if (op == 23)
600		write_group_objs(zram, gid, index);
601	if (op == 20) {
602		if (index)
603			zram_group_apply_writeback(zram->zgrp, hyperhold_nr_extent());
604		else
605			zram_group_remove_writeback(zram->zgrp);
606	}
607#endif
608}
609#endif
610
611static u64 group_obj_stats(struct zram *zram, u16 gid, int type)
612{
613	if (!(zram->zgrp)) {
614		pr_debug("zram group is not enable!\n");
615		return 0;
616	}
617	if (!CHECK_BOUND(gid, 0, zram->zgrp->nr_grp - 1))
618		return 0;
619
620	if (type == CACHE_SIZE)
621		return atomic64_read(&zram->zgrp->stats[gid].zram_size);
622	else if (type == CACHE_PAGE)
623		return atomic_read(&zram->zgrp->stats[gid].zram_pages);
624	else if (type == CACHE_FAULT)
625		return atomic64_read(&zram->zgrp->stats[gid].zram_fault);
626#ifdef CONFIG_ZRAM_GROUP_WRITEBACK
627	else if (type == SWAP_SIZE)
628		return atomic64_read(&zram->zgrp->stats[gid].wb_size);
629	else if (type == SWAP_PAGE)
630		return atomic_read(&zram->zgrp->stats[gid].wb_pages);
631	else if (type == READ_SIZE)
632		return atomic64_read(&zram->zgrp->stats[gid].read_size);
633	else if (type == WRITE_SIZE)
634		return atomic64_read(&zram->zgrp->stats[gid].write_size);
635	else if (type == SWAP_FAULT)
636		return atomic64_read(&zram->zgrp->stats[gid].wb_fault);
637	BUG();
638#endif
639
640	return 0;
641}
642
643#ifdef CONFIG_ZRAM_GROUP_WRITEBACK
644static u64 zram_group_read(u16 gid, u64 req_size, void *priv)
645{
646	if (!CHECK(priv, "priv is NULL!\n"))
647		return 0;
648
649	return read_group_objs((struct zram *)priv, gid, req_size);
650}
651
652static u64 zram_group_write(u16 gid, u64 req_size, void *priv)
653{
654	if (!CHECK(priv, "priv is NULL!\n"))
655		return 0;
656
657	return write_group_objs((struct zram *)priv, gid, req_size);
658}
659#else
660static u64 zram_group_read(u16 gid, u64 req_size, void *priv)
661{
662	return 0;
663}
664static u64 zram_group_write(u16 gid, u64 req_size, void *priv)
665{
666	return 0;
667}
668#endif
669
670
671static u64 zram_group_data_size(u16 gid, int type, void *priv)
672{
673	if (!CHECK(priv, "priv is NULL!\n"))
674		return 0;
675
676	return group_obj_stats((struct zram *)priv, gid, type);
677}
678
679struct group_swap_ops zram_group_ops = {
680	.group_read = zram_group_read,
681	.group_write = zram_group_write,
682	.group_data_size = zram_group_data_size,
683};
684
685static int register_zram_group(struct zram *zram)
686{
687	if (!CHECK(zram, "zram is NULL!\n"))
688		return -EINVAL;
689	if (!(zram->zgrp)) {
690		pr_debug("zram group is not enable!\n");
691		return -EINVAL;
692	}
693
694	zram->zgrp->gsdev = register_group_swap(&zram_group_ops, zram);
695	if (!zram->zgrp->gsdev) {
696		pr_err("register zram group failed!\n");
697		return -ENOMEM;
698	}
699
700	return 0;
701}
702
703static void unregister_zram_group(struct zram *zram)
704{
705	if (!CHECK(zram, "zram is NULL!\n"))
706		return;
707	if (!(zram->zgrp)) {
708		pr_debug("zram group is not enable!\n");
709		return;
710	}
711
712	unregister_group_swap(zram->zgrp->gsdev);
713	zram->zgrp->gsdev = NULL;
714}
715
716void zram_group_init(struct zram *zram, u32 nr_obj)
717{
718	unsigned int ctrl = zram->zgrp_ctrl;
719
720	if (ctrl == ZGRP_NONE)
721		return;
722	zram->zgrp = zram_group_meta_alloc(nr_obj, ZGRP_MAX_GRP - 1);
723#ifdef CONFIG_ZRAM_GROUP_WRITEBACK
724	if (ctrl == ZGRP_WRITE)
725		zram_group_apply_writeback(zram->zgrp, hyperhold_nr_extent());
726#endif
727	register_zram_group(zram);
728}
729
730void zram_group_deinit(struct zram *zram)
731{
732	unregister_zram_group(zram);
733	zram_group_meta_free(zram->zgrp);
734	zram->zgrp = NULL;
735}
736