Lines Matching refs:pool
85 * At the next level we can use various sub-allocators. The state pool is a
86 * pool of smaller, fixed size objects, which operates much like the block
87 * pool. It uses a free list for freeing objects, but when it runs out of
88 * space it just allocates a new block from the block pool. This allocator is
193 /* Assert that we only ever grow the pool */
206 /* Just leak the old map until we destroy the pool. We can't munmap it
237 /* The block pool is always initialized to a nonzero size and this function
298 /* We allocated the first block outside the pool so we have to grow
299 * the pool. pool_state->next acts a mutex: threads who try to
365 anv_block_pool_expand_range(struct anv_block_pool *pool,
369 anv_block_pool_init(struct anv_block_pool *pool,
378 /* Make sure VMA addresses are 2MiB aligned for the block pool */
383 pool->name = name;
384 pool->device = device;
385 pool->use_relocations = anv_use_relocations(device->physical);
386 pool->nbos = 0;
387 pool->size = 0;
388 pool->center_bo_offset = 0;
389 pool->start_address = intel_canonical_address(start_address);
390 pool->map = NULL;
392 if (!pool->use_relocations) {
393 pool->bo = NULL;
394 pool->fd = -1;
400 pool->fd = os_create_anonymous_file(BLOCK_POOL_MEMFD_SIZE, "block pool");
401 if (pool->fd == -1)
404 pool->wrapper_bo = (struct anv_bo) {
409 pool->bo = &pool->wrapper_bo;
412 if (!u_vector_init(&pool->mmap_cleanups, 8,
418 pool->state.next = 0;
419 pool->state.end = 0;
420 pool->back_state.next = 0;
421 pool->back_state.end = 0;
423 result = anv_block_pool_expand_range(pool, 0, initial_size);
427 /* Make the entire pool available in the front of the pool. If back
430 pool->state.end = pool->size;
435 u_vector_finish(&pool->mmap_cleanups);
437 if (pool->fd >= 0)
438 close(pool->fd);
444 anv_block_pool_finish(struct anv_block_pool *pool)
446 anv_block_pool_foreach_bo(bo, pool) {
448 anv_device_release_bo(pool->device, bo);
452 u_vector_foreach(cleanup, &pool->mmap_cleanups)
454 u_vector_finish(&pool->mmap_cleanups);
456 if (pool->fd >= 0)
457 close(pool->fd);
461 anv_block_pool_expand_range(struct anv_block_pool *pool,
464 /* Assert that we only ever grow the pool */
465 assert(center_bo_offset >= pool->back_state.end);
466 assert(size - center_bo_offset >= pool->state.end);
470 assert(!pool->use_relocations ||
474 /* For state pool BOs we have to be a bit careful about where we place them
485 * the pool during command buffer building so we don't actually have a
501 if (pool->use_relocations)
504 if (!pool->use_relocations) {
505 uint32_t new_bo_size = size - pool->size;
508 VkResult result = anv_device_alloc_bo(pool->device,
509 pool->name,
516 pool->start_address + pool->size,
521 pool->bos[pool->nbos++] = new_bo;
524 pool->bo = pool->bos[0];
526 /* Just leak the old map until we destroy the pool. We can't munmap it
533 MAP_SHARED | MAP_POPULATE, pool->fd,
536 return vk_errorf(pool->device, VK_ERROR_MEMORY_MAP_FAILED,
540 VkResult result = anv_device_import_bo_from_host_ptr(pool->device,
550 struct anv_mmap_cleanup *cleanup = u_vector_add(&pool->mmap_cleanups);
553 anv_device_release_bo(pool->device, new_bo);
554 return vk_error(pool->device, VK_ERROR_OUT_OF_HOST_MEMORY);
560 * center_bo_offset back into pool and update pool->map. */
561 pool->center_bo_offset = center_bo_offset;
562 pool->map = map + center_bo_offset;
564 pool->bos[pool->nbos++] = new_bo;
565 pool->wrapper_bo.map = new_bo;
568 assert(pool->nbos < ANV_MAX_BLOCK_POOL_BOS);
569 pool->size = size;
574 /** Returns current memory map of the block pool.
577 * offset. The offset parameter is relative to the "center" of the block pool
578 * rather than the start of the block pool BO map.
581 anv_block_pool_map(struct anv_block_pool *pool, int32_t offset, uint32_t size)
583 if (!pool->use_relocations) {
586 anv_block_pool_foreach_bo(iter_bo, pool) {
599 return pool->map + offset;
603 /** Grows and re-centers the block pool.
605 * We grow the block pool in one or both directions in such a way that the
608 * 1) The size of the entire pool is always a power of two.
610 * 2) The pool only grows on both ends. Neither end can get
614 * allocated for each end as we have used. This way the pool doesn't
618 * the pool retains a size of zero. (This makes it easier for users of
619 * the block pool that only want a one-sided pool.)
624 * 6) The center of the pool is always aligned to both the block_size of
625 * the pool and a 4K CPU page.
628 anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state,
633 pthread_mutex_lock(&pool->device->mutex);
635 assert(state == &pool->state || state == &pool->back_state);
637 /* Gather a little usage information on the pool. Since we may have
642 * which ever side tries to grow the pool.
647 uint32_t back_used = align_u32(pool->back_state.next, PAGE_SIZE);
648 uint32_t front_used = align_u32(pool->state.next, PAGE_SIZE);
651 assert(state == &pool->state || back_used > 0);
653 uint32_t old_size = pool->size;
655 /* The block pool is always initialized to a nonzero size and this function
660 const uint32_t old_back = pool->center_bo_offset;
661 const uint32_t old_front = old_size - pool->center_bo_offset;
670 if (!pool->use_relocations) {
671 /* With softpin, the pool is made up of a bunch of buffers with separate
691 assert(size > pool->size);
694 * of the pool, we maintain the ratio of how much is used by each side.
715 /* Make sure we don't shrink the back end of the pool */
719 /* Make sure that we don't shrink the front end of the pool */
726 result = anv_block_pool_expand_range(pool, center_bo_offset, size);
729 pthread_mutex_unlock(&pool->device->mutex);
736 if (state == &pool->state) {
737 return pool->size - pool->center_bo_offset;
739 assert(pool->center_bo_offset > 0);
740 return pool->center_bo_offset;
748 anv_block_pool_alloc_new(struct anv_block_pool *pool,
763 if (!pool->use_relocations && state.next < state.end) {
764 /* We need to grow the block pool, but still have some leftover
770 /* If there is some leftover space in the pool, the caller must
779 /* We allocated the first block outside the pool so we have to grow
780 * the pool. pool_state->next acts a mutex: threads who try to
786 new.end = anv_block_pool_grow(pool, pool_state, block_size);
801 anv_block_pool_alloc(struct anv_block_pool *pool,
806 offset = anv_block_pool_alloc_new(pool, &pool->state, block_size, padding);
811 /* Allocates a block out of the back of the block pool.
813 * This will allocated a block earlier than the "start" of the block pool.
815 * be correct relative to the block pool's map pointer.
818 * gymnastics with the block pool's BO when doing relocations.
821 anv_block_pool_alloc_back(struct anv_block_pool *pool,
824 int32_t offset = anv_block_pool_alloc_new(pool, &pool->back_state,
837 anv_state_pool_init(struct anv_state_pool *pool,
851 VkResult result = anv_block_pool_init(&pool->block_pool, device, name,
857 pool->start_offset = start_offset;
859 result = anv_state_table_init(&pool->table, device, 64);
861 anv_block_pool_finish(&pool->block_pool);
866 pool->block_size = block_size;
867 pool->back_alloc_free_list = ANV_FREE_LIST_EMPTY;
869 pool->buckets[i].free_list = ANV_FREE_LIST_EMPTY;
870 pool->buckets[i].block.next = 0;
871 pool->buckets[i].block.end = 0;
873 VG(VALGRIND_CREATE_MEMPOOL(pool, 0, false));
879 anv_state_pool_finish(struct anv_state_pool *pool)
881 VG(VALGRIND_DESTROY_MEMPOOL(pool));
882 anv_state_table_finish(&pool->table);
883 anv_block_pool_finish(&pool->block_pool);
887 anv_fixed_size_state_pool_alloc_new(struct anv_fixed_size_state_pool *pool,
910 block.u64 = __sync_fetch_and_add(&pool->block.u64, state_size);
918 old.u64 = __sync_lock_test_and_set(&pool->block.u64, new.u64);
920 futex_wake(&pool->block.end, INT_MAX);
923 futex_wait(&pool->block.end, block.end, NULL);
951 anv_state_pool_return_blocks(struct anv_state_pool *pool,
962 UNUSED VkResult result = anv_state_table_add(&pool->table, &st_idx, count);
966 struct anv_state *state_i = anv_state_table_get(&pool->table,
969 state_i->offset = pool->start_offset + chunk_offset + block_size * i;
970 state_i->map = anv_block_pool_map(&pool->block_pool,
976 anv_free_list_push(&pool->buckets[block_bucket].free_list,
977 &pool->table, st_idx, count);
980 /** Returns a chunk of memory back to the state pool.
983 * (pool->block_size), we return as many divisor sized blocks as we can, from
991 anv_state_pool_return_chunk(struct anv_state_pool *pool,
995 uint32_t divisor = pool->block_size;
1006 anv_state_pool_return_blocks(pool, offset, nblocks, divisor);
1024 anv_state_pool_return_blocks(pool, chunk_offset + rest,
1033 anv_state_pool_alloc_no_vg(struct anv_state_pool *pool,
1043 state = anv_free_list_pop(&pool->buckets[bucket].free_list,
1044 &pool->table);
1046 assert(state->offset >= pool->start_offset);
1052 state = anv_free_list_pop(&pool->buckets[b].free_list, &pool->table);
1062 /* Now return the unused part of the chunk back to the pool as free
1093 anv_state_pool_return_chunk(pool, chunk_offset + alloc_size,
1100 offset = anv_fixed_size_state_pool_alloc_new(&pool->buckets[bucket],
1101 &pool->block_pool,
1103 pool->block_size,
1105 /* Every time we allocate a new state, add it to the state pool */
1107 UNUSED VkResult result = anv_state_table_add(&pool->table, &idx, 1);
1110 state = anv_state_table_get(&pool->table, idx);
1111 state->offset = pool->start_offset + offset;
1113 state->map = anv_block_pool_map(&pool->block_pool, offset, alloc_size);
1117 anv_state_pool_return_chunk(pool, return_offset, padding, 0);
1125 anv_state_pool_alloc(struct anv_state_pool *pool, uint32_t size, uint32_t align)
1130 struct anv_state state = anv_state_pool_alloc_no_vg(pool, size, align);
1131 VG(VALGRIND_MEMPOOL_ALLOC(pool, state.map, size));
1136 anv_state_pool_alloc_back(struct anv_state_pool *pool)
1139 uint32_t alloc_size = pool->block_size;
1142 assert(pool->start_offset == 0);
1144 state = anv_free_list_pop(&pool->back_alloc_free_list, &pool->table);
1146 assert(state->offset < pool->start_offset);
1151 offset = anv_block_pool_alloc_back(&pool->block_pool,
1152 pool->block_size);
1154 UNUSED VkResult result = anv_state_table_add(&pool->table, &idx, 1);
1157 state = anv_state_table_get(&pool->table, idx);
1158 state->offset = pool->start_offset + offset;
1160 state->map = anv_block_pool_map(&pool->block_pool, offset, alloc_size);
1163 VG(VALGRIND_MEMPOOL_ALLOC(pool, state->map, state->alloc_size));
1168 anv_state_pool_free_no_vg(struct anv_state_pool *pool, struct anv_state state)
1173 if (state.offset < pool->start_offset) {
1174 assert(state.alloc_size == pool->block_size);
1175 anv_free_list_push(&pool->back_alloc_free_list,
1176 &pool->table, state.idx, 1);
1178 anv_free_list_push(&pool->buckets[bucket].free_list,
1179 &pool->table, state.idx, 1);
1184 anv_state_pool_free(struct anv_state_pool *pool, struct anv_state state)
1189 VG(VALGRIND_MEMPOOL_FREE(pool, state.map));
1190 anv_state_pool_free_no_vg(pool, state);
1293 anv_state_reserved_pool_init(struct anv_state_reserved_pool *pool,
1297 pool->pool = parent;
1298 pool->reserved_blocks = ANV_FREE_LIST_EMPTY;
1299 pool->count = count;
1302 struct anv_state state = anv_state_pool_alloc(pool->pool, size, alignment);
1303 anv_free_list_push(&pool->reserved_blocks, &pool->pool->table, state.idx, 1);
1308 anv_state_reserved_pool_finish(struct anv_state_reserved_pool *pool)
1312 while ((state = anv_free_list_pop(&pool->reserved_blocks, &pool->pool->table))) {
1313 anv_state_pool_free(pool->pool, *state);
1314 pool->count--;
1316 assert(pool->count == 0);
1320 anv_state_reserved_pool_alloc(struct anv_state_reserved_pool *pool)
1322 return *anv_free_list_pop(&pool->reserved_blocks, &pool->pool->table);
1326 anv_state_reserved_pool_free(struct anv_state_reserved_pool *pool,
1329 anv_free_list_push(&pool->reserved_blocks, &pool->pool->table, state.idx, 1);
1333 anv_bo_pool_init(struct anv_bo_pool *pool, struct anv_device *device,
1336 pool->name = name;
1337 pool->device = device;
1338 for (unsigned i = 0; i < ARRAY_SIZE(pool->free_list); i++) {
1339 util_sparse_array_free_list_init(&pool->free_list[i],
1344 VG(VALGRIND_CREATE_MEMPOOL(pool, 0, false));
1348 anv_bo_pool_finish(struct anv_bo_pool *pool)
1350 for (unsigned i = 0; i < ARRAY_SIZE(pool->free_list); i++) {
1353 util_sparse_array_free_list_pop_elem(&pool->free_list[i]);
1359 anv_device_release_bo(pool->device, bo);
1363 VG(VALGRIND_DESTROY_MEMPOOL(pool));
1367 anv_bo_pool_alloc(struct anv_bo_pool *pool, uint32_t size,
1373 assert(bucket < ARRAY_SIZE(pool->free_list));
1376 util_sparse_array_free_list_pop_elem(&pool->free_list[bucket]);
1378 VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, size));
1383 VkResult result = anv_device_alloc_bo(pool->device,
1384 pool->name,
1395 /* We want it to look like it came from this pool */
1397 VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, size));
1405 anv_bo_pool_free(struct anv_bo_pool *pool, struct anv_bo *bo)
1407 VG(VALGRIND_MEMPOOL_FREE(pool, bo->map));
1412 assert(bucket < ARRAY_SIZE(pool->free_list));
1414 assert(util_sparse_array_get(&pool->device->bo_cache.bo_map,
1416 util_sparse_array_free_list_push(&pool->free_list[bucket],
1420 // Scratch pool
1423 anv_scratch_pool_init(struct anv_device *device, struct anv_scratch_pool *pool)
1425 memset(pool, 0, sizeof(*pool));
1429 anv_scratch_pool_finish(struct anv_device *device, struct anv_scratch_pool *pool)
1431 for (unsigned s = 0; s < ARRAY_SIZE(pool->bos[0]); s++) {
1433 if (pool->bos[i][s] != NULL)
1434 anv_device_release_bo(device, pool->bos[i][s]);
1439 if (pool->surf_states[i].map != NULL) {
1441 pool->surf_states[i]);
1447 anv_scratch_pool_alloc(struct anv_device *device, struct anv_scratch_pool *pool,
1456 assert(stage < ARRAY_SIZE(pool->bos));
1468 struct anv_bo *bo = p_atomic_read(&pool->bos[scratch_size_log2][stage]);
1502 p_atomic_cmpxchg(&pool->bos[scratch_size_log2][stage], NULL, bo);
1513 struct anv_scratch_pool *pool,
1522 uint32_t surf = p_atomic_read(&pool->surfs[scratch_size_log2]);
1527 anv_scratch_pool_alloc(device, pool, MESA_SHADER_COMPUTE,
1544 uint32_t current = p_atomic_cmpxchg(&pool->surfs[scratch_size_log2],
1550 pool->surf_states[scratch_size_log2] = state;