Lines Matching refs:pool
5 * memory buffer pool support. Such pools are mostly used
25 static void poison_error(mempool_t *pool, void *element, size_t size,
28 const int nr = pool->curr_nr;
34 pr_err("Mempool %p size %zu\n", pool, size);
42 static void __check_element(mempool_t *pool, void *element, size_t size)
51 poison_error(pool, element, size, i);
58 static void check_element(mempool_t *pool, void *element)
61 if (pool->free == mempool_free_slab || pool->free == mempool_kfree) {
62 __check_element(pool, element, ksize(element));
63 } else if (pool->free == mempool_free_pages) {
65 int order = (int)(long)pool->pool_data;
68 __check_element(pool, addr, 1UL << (PAGE_SHIFT + order));
81 static void poison_element(mempool_t *pool, void *element)
84 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc) {
86 } else if (pool->alloc == mempool_alloc_pages) {
88 int order = (int)(long)pool->pool_data;
96 static inline void check_element(mempool_t *pool, void *element)
99 static inline void poison_element(mempool_t *pool, void *element)
104 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)
106 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
108 else if (pool->alloc == mempool_alloc_pages)
109 kasan_free_pages(element, (unsigned long)pool->pool_data);
112 static void kasan_unpoison_element(mempool_t *pool, void *element)
114 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
116 else if (pool->alloc == mempool_alloc_pages)
117 kasan_alloc_pages(element, (unsigned long)pool->pool_data);
120 static __always_inline void add_element(mempool_t *pool, void *element)
122 BUG_ON(pool->curr_nr >= pool->min_nr);
123 poison_element(pool, element);
124 kasan_poison_element(pool, element);
125 pool->elements[pool->curr_nr++] = element;
128 static void *remove_element(mempool_t *pool)
130 void *element = pool->elements[--pool->curr_nr];
132 BUG_ON(pool->curr_nr < 0);
133 kasan_unpoison_element(pool, element);
134 check_element(pool, element);
140 * @pool: pointer to the memory pool which was initialized with
143 * Free all reserved elements in @pool and @pool itself. This function
149 void mempool_exit(mempool_t *pool)
151 while (pool->curr_nr) {
152 void *element = remove_element(pool);
153 pool->free(element, pool->pool_data);
155 kfree(pool->elements);
156 pool->elements = NULL;
161 * mempool_destroy - deallocate a memory pool
162 * @pool: pointer to the memory pool which was allocated via
165 * Free all reserved elements in @pool and @pool itself. This function
168 void mempool_destroy(mempool_t *pool)
170 if (unlikely(!pool))
173 mempool_exit(pool);
174 kfree(pool);
178 int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
182 spin_lock_init(&pool->lock);
183 pool->min_nr = min_nr;
184 pool->pool_data = pool_data;
185 pool->alloc = alloc_fn;
186 pool->free = free_fn;
187 init_waitqueue_head(&pool->wait);
189 pool->elements = kmalloc_array_node(min_nr, sizeof(void *),
191 if (!pool->elements)
197 while (pool->curr_nr < pool->min_nr) {
200 element = pool->alloc(gfp_mask, pool->pool_data);
202 mempool_exit(pool);
205 add_element(pool, element);
213 * mempool_init - initialize a memory pool
214 * @pool: pointer to the memory pool that should be initialized
216 * allocated for this pool.
221 * Like mempool_create(), but initializes the pool in (i.e. embedded in another
226 int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
229 return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
236 * mempool_create - create a memory pool
238 * allocated for this pool.
244 * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
249 * Return: pointer to the created memory pool object or %NULL on error.
263 mempool_t *pool;
265 pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);
266 if (!pool)
269 if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,
271 kfree(pool);
275 return pool;
280 * mempool_resize - resize an existing memory pool
281 * @pool: pointer to the memory pool which was allocated via
284 * allocated for this pool.
286 * This function shrinks/grows the pool. In the case of growing,
287 * it cannot be guaranteed that the pool will be grown to the new
297 int mempool_resize(mempool_t *pool, int new_min_nr)
306 spin_lock_irqsave(&pool->lock, flags);
307 if (new_min_nr <= pool->min_nr) {
308 while (new_min_nr < pool->curr_nr) {
309 element = remove_element(pool);
310 spin_unlock_irqrestore(&pool->lock, flags);
311 pool->free(element, pool->pool_data);
312 spin_lock_irqsave(&pool->lock, flags);
314 pool->min_nr = new_min_nr;
317 spin_unlock_irqrestore(&pool->lock, flags);
319 /* Grow the pool */
325 spin_lock_irqsave(&pool->lock, flags);
326 if (unlikely(new_min_nr <= pool->min_nr)) {
328 spin_unlock_irqrestore(&pool->lock, flags);
332 memcpy(new_elements, pool->elements,
333 pool->curr_nr * sizeof(*new_elements));
334 kfree(pool->elements);
335 pool->elements = new_elements;
336 pool->min_nr = new_min_nr;
338 while (pool->curr_nr < pool->min_nr) {
339 spin_unlock_irqrestore(&pool->lock, flags);
340 element = pool->alloc(GFP_KERNEL, pool->pool_data);
343 spin_lock_irqsave(&pool->lock, flags);
344 if (pool->curr_nr < pool->min_nr) {
345 add_element(pool, element);
347 spin_unlock_irqrestore(&pool->lock, flags);
348 pool->free(element, pool->pool_data); /* Raced */
353 spin_unlock_irqrestore(&pool->lock, flags);
360 * mempool_alloc - allocate an element from a specific memory pool
361 * @pool: pointer to the memory pool which was allocated via
373 void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
391 element = pool->alloc(gfp_temp, pool->pool_data);
395 spin_lock_irqsave(&pool->lock, flags);
396 if (likely(pool->curr_nr)) {
397 element = remove_element(pool);
398 spin_unlock_irqrestore(&pool->lock, flags);
411 * alloc failed with that and @pool was empty, retry immediately.
414 spin_unlock_irqrestore(&pool->lock, flags);
421 spin_unlock_irqrestore(&pool->lock, flags);
425 /* Let's wait for someone else to return an element to @pool */
427 prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
429 spin_unlock_irqrestore(&pool->lock, flags);
437 finish_wait(&pool->wait, &wait);
443 * mempool_free - return an element to the pool.
444 * @element: pool element pointer.
445 * @pool: pointer to the memory pool which was allocated via
450 void mempool_free(void *element, mempool_t *pool)
459 * for @element and the following @pool->curr_nr. This ensures
460 * that the visible value of @pool->curr_nr is from after the
488 * pool waking up the waiters.
490 if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {
491 spin_lock_irqsave(&pool->lock, flags);
492 if (likely(pool->curr_nr < pool->min_nr)) {
493 add_element(pool, element);
494 spin_unlock_irqrestore(&pool->lock, flags);
495 wake_up(&pool->wait);
498 spin_unlock_irqrestore(&pool->lock, flags);
500 pool->free(element, pool->pool_data);