Lines Matching refs:pool
5 * memory buffer pool support. Such pools are mostly used
24 static void poison_error(mempool_t *pool, void *element, size_t size,
27 const int nr = pool->curr_nr;
33 pr_err("Mempool %p size %zu\n", pool, size);
41 static void __check_element(mempool_t *pool, void *element, size_t size)
50 poison_error(pool, element, size, i);
57 static void check_element(mempool_t *pool, void *element)
60 if (pool->free == mempool_kfree) {
61 __check_element(pool, element, (size_t)pool->pool_data);
62 } else if (pool->free == mempool_free_slab) {
63 __check_element(pool, element, kmem_cache_size(pool->pool_data));
64 } else if (pool->free == mempool_free_pages) {
66 int order = (int)(long)pool->pool_data;
69 __check_element(pool, addr, 1UL << (PAGE_SHIFT + order));
82 static void poison_element(mempool_t *pool, void *element)
85 if (pool->alloc == mempool_kmalloc) {
86 __poison_element(element, (size_t)pool->pool_data);
87 } else if (pool->alloc == mempool_alloc_slab) {
88 __poison_element(element, kmem_cache_size(pool->pool_data));
89 } else if (pool->alloc == mempool_alloc_pages) {
91 int order = (int)(long)pool->pool_data;
99 static inline void check_element(mempool_t *pool, void *element)
102 static inline void poison_element(mempool_t *pool, void *element)
107 static __always_inline void kasan_poison_element(mempool_t *pool, void *element)
109 if (pool->alloc == mempool_alloc_slab || pool->alloc == mempool_kmalloc)
111 else if (pool->alloc == mempool_alloc_pages)
112 kasan_poison_pages(element, (unsigned long)pool->pool_data,
116 static void kasan_unpoison_element(mempool_t *pool, void *element)
118 if (pool->alloc == mempool_kmalloc)
119 kasan_unpoison_range(element, (size_t)pool->pool_data);
120 else if (pool->alloc == mempool_alloc_slab)
121 kasan_unpoison_range(element, kmem_cache_size(pool->pool_data));
122 else if (pool->alloc == mempool_alloc_pages)
123 kasan_unpoison_pages(element, (unsigned long)pool->pool_data,
127 static __always_inline void add_element(mempool_t *pool, void *element)
129 BUG_ON(pool->curr_nr >= pool->min_nr);
130 poison_element(pool, element);
131 kasan_poison_element(pool, element);
132 pool->elements[pool->curr_nr++] = element;
135 static void *remove_element(mempool_t *pool)
137 void *element = pool->elements[--pool->curr_nr];
139 BUG_ON(pool->curr_nr < 0);
140 kasan_unpoison_element(pool, element);
141 check_element(pool, element);
147 * @pool: pointer to the memory pool which was initialized with
150 * Free all reserved elements in @pool and @pool itself. This function
156 void mempool_exit(mempool_t *pool)
158 while (pool->curr_nr) {
159 void *element = remove_element(pool);
160 pool->free(element, pool->pool_data);
162 kfree(pool->elements);
163 pool->elements = NULL;
168 * mempool_destroy - deallocate a memory pool
169 * @pool: pointer to the memory pool which was allocated via
172 * Free all reserved elements in @pool and @pool itself. This function
175 void mempool_destroy(mempool_t *pool)
177 if (unlikely(!pool))
180 mempool_exit(pool);
181 kfree(pool);
185 int mempool_init_node(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
189 spin_lock_init(&pool->lock);
190 pool->min_nr = min_nr;
191 pool->pool_data = pool_data;
192 pool->alloc = alloc_fn;
193 pool->free = free_fn;
194 init_waitqueue_head(&pool->wait);
196 pool->elements = kmalloc_array_node(min_nr, sizeof(void *),
198 if (!pool->elements)
204 while (pool->curr_nr < pool->min_nr) {
207 element = pool->alloc(gfp_mask, pool->pool_data);
209 mempool_exit(pool);
212 add_element(pool, element);
220 * mempool_init - initialize a memory pool
221 * @pool: pointer to the memory pool that should be initialized
223 * allocated for this pool.
228 * Like mempool_create(), but initializes the pool in (i.e. embedded in another
233 int mempool_init(mempool_t *pool, int min_nr, mempool_alloc_t *alloc_fn,
236 return mempool_init_node(pool, min_nr, alloc_fn, free_fn,
243 * mempool_create - create a memory pool
245 * allocated for this pool.
251 * memory pool. The pool can be used from the mempool_alloc() and mempool_free()
256 * Return: pointer to the created memory pool object or %NULL on error.
270 mempool_t *pool;
272 pool = kzalloc_node(sizeof(*pool), gfp_mask, node_id);
273 if (!pool)
276 if (mempool_init_node(pool, min_nr, alloc_fn, free_fn, pool_data,
278 kfree(pool);
282 return pool;
287 * mempool_resize - resize an existing memory pool
288 * @pool: pointer to the memory pool which was allocated via
291 * allocated for this pool.
293 * This function shrinks/grows the pool. In the case of growing,
294 * it cannot be guaranteed that the pool will be grown to the new
304 int mempool_resize(mempool_t *pool, int new_min_nr)
313 spin_lock_irqsave(&pool->lock, flags);
314 if (new_min_nr <= pool->min_nr) {
315 while (new_min_nr < pool->curr_nr) {
316 element = remove_element(pool);
317 spin_unlock_irqrestore(&pool->lock, flags);
318 pool->free(element, pool->pool_data);
319 spin_lock_irqsave(&pool->lock, flags);
321 pool->min_nr = new_min_nr;
324 spin_unlock_irqrestore(&pool->lock, flags);
326 /* Grow the pool */
332 spin_lock_irqsave(&pool->lock, flags);
333 if (unlikely(new_min_nr <= pool->min_nr)) {
335 spin_unlock_irqrestore(&pool->lock, flags);
339 memcpy(new_elements, pool->elements,
340 pool->curr_nr * sizeof(*new_elements));
341 kfree(pool->elements);
342 pool->elements = new_elements;
343 pool->min_nr = new_min_nr;
345 while (pool->curr_nr < pool->min_nr) {
346 spin_unlock_irqrestore(&pool->lock, flags);
347 element = pool->alloc(GFP_KERNEL, pool->pool_data);
350 spin_lock_irqsave(&pool->lock, flags);
351 if (pool->curr_nr < pool->min_nr) {
352 add_element(pool, element);
354 spin_unlock_irqrestore(&pool->lock, flags);
355 pool->free(element, pool->pool_data); /* Raced */
360 spin_unlock_irqrestore(&pool->lock, flags);
367 * mempool_alloc - allocate an element from a specific memory pool
368 * @pool: pointer to the memory pool which was allocated via
380 void *mempool_alloc(mempool_t *pool, gfp_t gfp_mask)
398 element = pool->alloc(gfp_temp, pool->pool_data);
402 spin_lock_irqsave(&pool->lock, flags);
403 if (likely(pool->curr_nr)) {
404 element = remove_element(pool);
405 spin_unlock_irqrestore(&pool->lock, flags);
418 * alloc failed with that and @pool was empty, retry immediately.
421 spin_unlock_irqrestore(&pool->lock, flags);
428 spin_unlock_irqrestore(&pool->lock, flags);
432 /* Let's wait for someone else to return an element to @pool */
434 prepare_to_wait(&pool->wait, &wait, TASK_UNINTERRUPTIBLE);
436 spin_unlock_irqrestore(&pool->lock, flags);
444 finish_wait(&pool->wait, &wait);
450 * mempool_free - return an element to the pool.
451 * @element: pool element pointer.
452 * @pool: pointer to the memory pool which was allocated via
457 void mempool_free(void *element, mempool_t *pool)
466 * for @element and the following @pool->curr_nr. This ensures
467 * that the visible value of @pool->curr_nr is from after the
495 * pool waking up the waiters.
497 if (unlikely(READ_ONCE(pool->curr_nr) < pool->min_nr)) {
498 spin_lock_irqsave(&pool->lock, flags);
499 if (likely(pool->curr_nr < pool->min_nr)) {
500 add_element(pool, element);
501 spin_unlock_irqrestore(&pool->lock, flags);
502 wake_up(&pool->wait);
505 spin_unlock_irqrestore(&pool->lock, flags);
507 pool->free(element, pool->pool_data);