1/* 2 * Compressed RAM block device 3 * 4 * Copyright (C) 2008, 2009, 2010 Nitin Gupta 5 * 2012, 2013 Minchan Kim 6 * 7 * This code is released using a dual license strategy: BSD/GPL 8 * You can choose the licence that better fits your requirements. 9 * 10 * Released under the terms of 3-clause BSD License 11 * Released under the terms of GNU General Public License Version 2.0 12 * 13 */ 14 15#ifndef _ZRAM_DRV_H_ 16#define _ZRAM_DRV_H_ 17 18#include <linux/rwsem.h> 19#include <linux/zsmalloc.h> 20#include <linux/crypto.h> 21 22#include "zcomp.h" 23 24#ifdef CONFIG_ZRAM_GROUP 25#include "zram_group.h" 26#endif 27 28#define SECTORS_PER_PAGE_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) 29#define SECTORS_PER_PAGE (1 << SECTORS_PER_PAGE_SHIFT) 30#define ZRAM_LOGICAL_BLOCK_SHIFT 12 31#define ZRAM_LOGICAL_BLOCK_SIZE (1 << ZRAM_LOGICAL_BLOCK_SHIFT) 32#define ZRAM_SECTOR_PER_LOGICAL_BLOCK \ 33 (1 << (ZRAM_LOGICAL_BLOCK_SHIFT - SECTOR_SHIFT)) 34 35 36/* 37 * The lower ZRAM_FLAG_SHIFT bits of table.flags is for 38 * object size (excluding header), the higher bits is for 39 * zram_pageflags. 40 * 41 * zram is mainly used for memory efficiency so we want to keep memory 42 * footprint small so we can squeeze size and flags into a field. 43 * The lower ZRAM_FLAG_SHIFT bits is for object size (excluding header), 44 * the higher bits is for zram_pageflags. 45 */ 46#ifdef CONFIG_ZRAM_GROUP 47/* reserve 16 bits for group id */ 48#define ZRAM_SIZE_SHIFT 24 49#define ZRAM_GRPID_SHIFT 16 50#define ZRAM_GRPID_MASK (((1UL << ZRAM_GRPID_SHIFT) - 1) << ZRAM_SIZE_SHIFT) 51#define ZRAM_FLAG_SHIFT (ZRAM_SIZE_SHIFT + ZRAM_GRPID_SHIFT) 52#else 53#define ZRAM_FLAG_SHIFT 24 54#endif 55 56/* Flags for zram pages (table[page_no].flags) */ 57enum zram_pageflags { 58 /* zram slot is locked */ 59 ZRAM_LOCK = ZRAM_FLAG_SHIFT, 60 ZRAM_SAME, /* Page consists the same element */ 61 ZRAM_WB, /* page is stored on backing_device */ 62 ZRAM_UNDER_WB, /* page is under writeback */ 63 ZRAM_HUGE, /* Incompressible page */ 64 ZRAM_IDLE, /* not accessed page since last idle marking */ 65#ifdef CONFIG_ZRAM_GROUP_WRITEBACK 66 ZRAM_GWB, /* obj is group writeback*/ 67 ZRAM_FAULT, /* obj is needed by a pagefault req */ 68#endif 69 70 __NR_ZRAM_PAGEFLAGS, 71}; 72 73/*-- Data structures */ 74 75/* Allocated for each disk page */ 76struct zram_table_entry { 77 union { 78 unsigned long handle; 79 unsigned long element; 80 }; 81 unsigned long flags; 82#ifdef CONFIG_ZRAM_MEMORY_TRACKING 83 ktime_t ac_time; 84#endif 85}; 86 87struct zram_stats { 88 atomic64_t compr_data_size; /* compressed size of pages stored */ 89 atomic64_t num_reads; /* failed + successful */ 90 atomic64_t num_writes; /* --do-- */ 91 atomic64_t failed_reads; /* can happen when memory is too low */ 92 atomic64_t failed_writes; /* can happen when memory is too low */ 93 atomic64_t invalid_io; /* non-page-aligned I/O requests */ 94 atomic64_t notify_free; /* no. of swap slot free notifications */ 95 atomic64_t same_pages; /* no. of same element filled pages */ 96 atomic64_t huge_pages; /* no. of huge pages */ 97 atomic64_t pages_stored; /* no. of pages currently stored */ 98 atomic_long_t max_used_pages; /* no. of maximum pages stored */ 99 atomic64_t writestall; /* no. of write slow paths */ 100 atomic64_t miss_free; /* no. of missed free */ 101#ifdef CONFIG_ZRAM_WRITEBACK 102 atomic64_t bd_count; /* no. of pages in backing device */ 103 atomic64_t bd_reads; /* no. of reads from backing device */ 104 atomic64_t bd_writes; /* no. of writes from backing device */ 105#endif 106}; 107 108struct zram { 109 struct zram_table_entry *table; 110#ifdef CONFIG_ZRAM_GROUP 111 struct zram_group *zgrp; 112 unsigned int zgrp_ctrl; 113#endif 114 struct zs_pool *mem_pool; 115 struct zcomp *comp; 116 struct gendisk *disk; 117 /* Prevent concurrent execution of device init */ 118 struct rw_semaphore init_lock; 119 /* 120 * the number of pages zram can consume for storing compressed data 121 */ 122 unsigned long limit_pages; 123 124 struct zram_stats stats; 125 /* 126 * This is the limit on amount of *uncompressed* worth of data 127 * we can store in a disk. 128 */ 129 u64 disksize; /* bytes */ 130 char compressor[CRYPTO_MAX_ALG_NAME]; 131 /* 132 * zram is claimed so open request will be failed 133 */ 134 bool claim; /* Protected by bdev->bd_mutex */ 135 struct file *backing_dev; 136#ifdef CONFIG_ZRAM_WRITEBACK 137 spinlock_t wb_limit_lock; 138 bool wb_limit_enable; 139 u64 bd_wb_limit; 140 struct block_device *bdev; 141 unsigned int old_block_size; 142 unsigned long *bitmap; 143 unsigned long nr_pages; 144#endif 145#ifdef CONFIG_ZRAM_MEMORY_TRACKING 146 struct dentry *debugfs_dir; 147#endif 148}; 149 150static inline int zram_slot_trylock(struct zram *zram, u32 index) 151{ 152 return bit_spin_trylock(ZRAM_LOCK, &zram->table[index].flags); 153} 154 155static inline void zram_slot_lock(struct zram *zram, u32 index) 156{ 157 bit_spin_lock(ZRAM_LOCK, &zram->table[index].flags); 158} 159 160static inline void zram_slot_unlock(struct zram *zram, u32 index) 161{ 162 bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags); 163} 164 165static inline unsigned long zram_get_handle(struct zram *zram, u32 index) 166{ 167 return zram->table[index].handle; 168} 169 170static inline void zram_set_handle(struct zram *zram, u32 index, unsigned long handle) 171{ 172 zram->table[index].handle = handle; 173} 174 175/* flag operations require table entry bit_spin_lock() being held */ 176static inline bool zram_test_flag(struct zram *zram, u32 index, 177 enum zram_pageflags flag) 178{ 179 return zram->table[index].flags & BIT(flag); 180} 181 182static inline void zram_set_flag(struct zram *zram, u32 index, 183 enum zram_pageflags flag) 184{ 185 zram->table[index].flags |= BIT(flag); 186} 187 188static inline void zram_clear_flag(struct zram *zram, u32 index, 189 enum zram_pageflags flag) 190{ 191 zram->table[index].flags &= ~BIT(flag); 192} 193#ifdef CONFIG_ZRAM_GROUP 194static inline size_t zram_get_obj_size(struct zram *zram, u32 index) 195{ 196 return zram->table[index].flags & (BIT(ZRAM_SIZE_SHIFT) - 1); 197} 198 199static inline void zram_set_obj_size(struct zram *zram, u32 index, size_t size) 200{ 201 unsigned long flags = zram->table[index].flags >> ZRAM_SIZE_SHIFT; 202 203 zram->table[index].flags = (flags << ZRAM_SIZE_SHIFT) | size; 204} 205 206void zram_group_init(struct zram *zram, u32 nr_obj); 207void zram_group_deinit(struct zram *zram); 208void zram_group_track_obj(struct zram *zram, u32 index, struct mem_cgroup *memcg); 209void zram_group_untrack_obj(struct zram *zram, u32 index); 210#ifdef CONFIG_ZRAM_GROUP_WRITEBACK 211int zram_group_fault_obj(struct zram *zram, u32 index); 212#endif 213 214#ifdef CONFIG_ZRAM_GROUP_DEBUG 215void group_debug(struct zram *zram, u32 op, u32 index, u32 gid); 216#endif 217 218#else 219static inline size_t zram_get_obj_size(struct zram *zram, u32 index) 220{ 221 return zram->table[index].flags & (BIT(ZRAM_FLAG_SHIFT) - 1); 222} 223 224static inline void zram_set_obj_size(struct zram *zram, u32 index, size_t size) 225{ 226 unsigned long flags = zram->table[index].flags >> ZRAM_FLAG_SHIFT; 227 228 zram->table[index].flags = (flags << ZRAM_FLAG_SHIFT) | size; 229} 230#endif 231#endif 232