Lines Matching refs:device
163 radv_get_adjusted_vram_size(struct radv_physical_device *device)
165 int ov = driQueryOptioni(&device->instance->dri_options, "override_vram_size");
167 return MIN2((uint64_t)device->rad_info.vram_size_kb * 1024, (uint64_t)ov << 20);
168 return (uint64_t)device->rad_info.vram_size_kb * 1024;
172 radv_get_visible_vram_size(struct radv_physical_device *device)
174 return MIN2(radv_get_adjusted_vram_size(device), (uint64_t)device->rad_info.vram_vis_size_kb * 1024);
178 radv_get_vram_size(struct radv_physical_device *device)
180 uint64_t total_size = radv_get_adjusted_vram_size(device);
181 return total_size - MIN2(total_size, (uint64_t)device->rad_info.vram_vis_size_kb * 1024);
192 radv_physical_device_init_mem_types(struct radv_physical_device *device)
194 uint64_t visible_vram_size = radv_get_visible_vram_size(device);
195 uint64_t vram_size = radv_get_vram_size(device);
196 uint64_t gtt_size = (uint64_t)device->rad_info.gart_size_kb * 1024;
199 device->memory_properties.memoryHeapCount = 0;
200 device->heaps = 0;
202 if (!device->rad_info.has_dedicated_vram) {
208 visible_vram_size = align64((total_size * 2) / 3, device->rad_info.gart_page_size);
216 vram_index = device->memory_properties.memoryHeapCount++;
217 device->heaps |= RADV_HEAP_VRAM;
218 device->memory_properties.memoryHeaps[vram_index] = (VkMemoryHeap){
225 gart_index = device->memory_properties.memoryHeapCount++;
226 device->heaps |= RADV_HEAP_GTT;
227 device->memory_properties.memoryHeaps[gart_index] = (VkMemoryHeap){
234 visible_vram_index = device->memory_properties.memoryHeapCount++;
235 device->heaps |= RADV_HEAP_VRAM_VIS;
236 device->memory_properties.memoryHeaps[visible_vram_index] = (VkMemoryHeap){
245 device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
246 device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS;
247 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType){
252 device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
253 device->memory_flags[type_count] = RADEON_FLAG_NO_CPU_ACCESS | RADEON_FLAG_32BIT;
254 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType){
261 device->memory_domains[type_count] = RADEON_DOMAIN_GTT;
262 device->memory_flags[type_count] = RADEON_FLAG_GTT_WC | RADEON_FLAG_CPU_ACCESS;
263 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType){
270 device->memory_domains[type_count] = RADEON_DOMAIN_VRAM;
271 device->memory_flags[type_count] = RADEON_FLAG_CPU_ACCESS;
272 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType){
281 device->memory_domains[type_count] = RADEON_DOMAIN_GTT;
282 device->memory_flags[type_count] = RADEON_FLAG_CPU_ACCESS;
283 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType){
289 device->memory_properties.memoryTypeCount = type_count;
291 if (device->rad_info.has_l2_uncached) {
292 for (int i = 0; i < device->memory_properties.memoryTypeCount; i++) {
293 VkMemoryType mem_type = device->memory_properties.memoryTypes[i];
298 !(device->memory_flags[i] & RADEON_FLAG_32BIT)) {
304 device->memory_domains[type_count] = device->memory_domains[i];
305 device->memory_flags[type_count] = device->memory_flags[i] | RADEON_FLAG_VA_UNCACHED;
306 device->memory_properties.memoryTypes[type_count++] = (VkMemoryType){
312 device->memory_properties.memoryTypeCount = type_count;
316 if (device->memory_flags[i] & RADEON_FLAG_32BIT)
317 device->memory_types_32bit |= BITFIELD_BIT(i);
449 radv_physical_device_get_supported_extensions(const struct radv_physical_device *device,
455 .KHR_acceleration_structure = radv_enable_rt(device, false),
475 .KHR_fragment_shading_rate = device->rad_info.gfx_level >= GFX10_3,
487 .KHR_performance_query = radv_perf_query_supported(device),
489 .KHR_pipeline_library = !device->use_llvm,
491 .KHR_ray_query = radv_enable_rt(device, false),
492 .KHR_ray_tracing_maintenance1 = radv_enable_rt(device, false),
493 .KHR_ray_tracing_pipeline = radv_enable_rt(device, true),
523 .EXT_border_color_swizzle = device->rad_info.gfx_level >= GFX10,
528 .EXT_conservative_rasterization = device->rad_info.gfx_level >= GFX9,
542 .EXT_external_memory_host = device->rad_info.has_userptr,
547 .EXT_image_drm_format_modifier = device->rad_info.gfx_level >= GFX9,
550 .EXT_index_type_uint8 = device->rad_info.gfx_level >= GFX8,
563 .EXT_post_depth_coverage = device->rad_info.gfx_level >= GFX10,
570 .EXT_sample_locations = device->rad_info.gfx_level < GFX10,
572 .EXT_scalar_block_layout = device->rad_info.gfx_level >= GFX7,
576 .EXT_shader_atomic_float2 = !device->use_llvm || LLVM_VERSION_MAJOR >= 14,
589 .EXT_transform_feedback = device->rad_info.gfx_level < GFX11,
591 .EXT_vertex_input_dynamic_state = !device->use_llvm,
597 .AMD_gpu_shader_half_float = device->rad_info.has_packed_math_16bit,
598 .AMD_gpu_shader_int16 = device->rad_info.has_packed_math_16bit,
601 .AMD_rasterization_order = device->rad_info.has_out_of_order_rast,
606 .AMD_shader_fragment_mask = device->rad_info.gfx_level < GFX11,
619 .NV_device_generated_commands = device->rad_info.gfx_level >= GFX7 &&
620 !(device->instance->debug_flags & RADV_DEBUG_NO_IBS) &&
621 driQueryOptionb(&device->instance->dri_options, "radv_dgc"),
623 radv_taskmesh_enabled(device) && device->instance->perftest_flags & RADV_PERFTEST_NV_MS,
628 device->vk.instance->app_info.engine_name &&
629 strcmp(device->vk.instance->app_info.engine_name, "vkd3d") == 0,
676 "Could not open device %s: %m", path);
684 "Could not get the kernel driver version for device %s: %m", path);
697 fprintf(stderr, "radv: info: Found compatible device '%s'.\n", path);
701 struct radv_physical_device *device = vk_zalloc2(&instance->vk.alloc, NULL, sizeof(*device), 8,
703 if (!device) {
714 result = vk_physical_device_init(&device->vk, &instance->vk, NULL, &dispatch_table);
719 device->instance = instance;
722 device->ws = radv_null_winsys_create();
727 device->ws = radv_amdgpu_winsys_create(fd, instance->debug_flags, instance->perftest_flags,
730 device->ws = radv_null_winsys_create();
734 if (!device->ws) {
739 device->vk.supported_sync_types = device->ws->get_sync_types(device->ws);
760 device->master_fd = master_fd;
761 device->local_fd = fd;
762 device->ws->query_info(device->ws, &device->rad_info);
764 if (device->rad_info.gfx_level >= GFX11) {
770 device->use_llvm = instance->debug_flags & RADV_DEBUG_LLVM;
772 if (device->use_llvm) {
780 device->emulate_etc2 = !radv_device_supports_etc(device);
782 device->emulate_etc2 = !radv_device_supports_etc(device) &&
783 driQueryOptionb(&device->instance->dri_options, "radv_require_etc2");
786 snprintf(device->name, sizeof(device->name), "AMD RADV %s%s", device->rad_info.name,
787 radv_get_compiler_string(device));
789 const char *marketing_name = device->ws->get_chip_name(device->ws);
790 snprintf(device->marketing_name, sizeof(device->name), "%s (RADV %s%s)",
791 marketing_name ? marketing_name : "AMD Unknown", device->rad_info.name,
792 radv_get_compiler_string(device));
795 if (radv_device_get_cache_uuid(device, device->cache_uuid)) {
804 disk_cache_format_hex_id(buf, device->cache_uuid, VK_UUID_SIZE * 2);
805 device->disk_cache = disk_cache_create(device->name, buf, 0);
808 if (!radv_is_conformant(device))
811 radv_get_driver_uuid(&device->driver_uuid);
812 radv_get_device_uuid(&device->rad_info, &device->device_uuid);
814 device->out_of_order_rast_allowed =
815 device->rad_info.has_out_of_order_rast &&
816 !(device->instance->debug_flags & RADV_DEBUG_NO_OUT_OF_ORDER);
818 device->dcc_msaa_allowed = (device->instance->perftest_flags & RADV_PERFTEST_DCC_MSAA);
820 device->use_ngg = (device->rad_info.gfx_level >= GFX10 &&
821 device->rad_info.family != CHIP_NAVI14 &&
822 !(device->instance->debug_flags & RADV_DEBUG_NO_NGG)) ||
823 device->rad_info.gfx_level >= GFX11;
825 device->use_ngg_culling = device->use_ngg && device->rad_info.max_render_backends > 1 &&
826 (device->rad_info.gfx_level >= GFX10_3 ||
827 (device->instance->perftest_flags & RADV_PERFTEST_NGGC)) &&
828 !(device->instance->debug_flags & RADV_DEBUG_NO_NGGC);
830 device->use_ngg_streamout = false;
833 device->cs_wave_size = 64;
834 device->ps_wave_size = 64;
835 device->ge_wave_size = 64;
836 device->rt_wave_size = 64;
838 if (device->rad_info.gfx_level >= GFX10) {
839 if (device->instance->perftest_flags & RADV_PERFTEST_CS_WAVE_32)
840 device->cs_wave_size = 32;
843 if (device->instance->perftest_flags & RADV_PERFTEST_PS_WAVE_32)
844 device->ps_wave_size = 32;
846 if (device->instance->perftest_flags & RADV_PERFTEST_GE_WAVE_32)
847 device->ge_wave_size = 32;
849 if (!(device->instance->perftest_flags & RADV_PERFTEST_RT_WAVE_64))
850 device->rt_wave_size = 32;
853 radv_physical_device_init_mem_types(device);
855 radv_physical_device_get_supported_extensions(device, &device->vk.supported_extensions);
857 radv_get_nir_options(device);
863 device->available_nodes = drm_device->available_nodes;
864 device->bus_info = *drm_device->businfo.pci;
873 device->primary_devid = primary_stat.st_rdev;
882 device->render_devid = render_stat.st_rdev;
886 if ((device->instance->debug_flags & RADV_DEBUG_INFO))
887 ac_print_gpu_info(&device->rad_info, stdout);
889 radv_physical_device_init_queue_table(device);
892 ac_init_perfcounters(&device->rad_info, false, false, &device->ac_perfcounters);
898 result = radv_init_wsi(device);
904 device->gs_table_depth =
905 ac_get_gs_table_depth(device->rad_info.gfx_level, device->rad_info.family);
907 ac_get_hs_info(&device->rad_info, &device->hs);
908 ac_get_task_info(&device->rad_info, &device->task_info);
910 *device_out = device;
915 ac_destroy_perfcounters(&device->ac_perfcounters);
916 disk_cache_destroy(device->disk_cache);
918 device->ws->destroy(device->ws);
920 vk_physical_device_finish(&device->vk);
922 vk_free(&instance->vk.alloc, device);
932 radv_physical_device_destroy(struct radv_physical_device *device)
934 radv_finish_wsi(device);
935 ac_destroy_perfcounters(&device->ac_perfcounters);
936 device->ws->destroy(device->ws);
937 disk_cache_destroy(device->disk_cache);
938 if (device->local_fd != -1)
939 close(device->local_fd);
940 if (device->master_fd != -1)
941 close(device->master_fd);
942 vk_physical_device_finish(&device->vk);
943 vk_free(&device->instance->vk.alloc, device);
1178 * device that allows to test the compiler without having an
1209 /* Incompatible DRM device, skip. */
1215 /* Error creating the physical device, report the error. */
2687 RADV_FROM_HANDLE(radv_physical_device, device, physicalDevice);
2688 VkPhysicalDeviceMemoryProperties *memory_properties = &device->memory_properties;
2694 * currently allocated device memory.
2699 if (!device->rad_info.has_dedicated_vram) {
2704 assert(device->heaps == (RADV_HEAP_GTT | RADV_HEAP_VRAM_VIS));
2705 assert(device->memory_properties.memoryHeaps[0].flags == 0); /* GTT */
2706 assert(device->memory_properties.memoryHeaps[1].flags == VK_MEMORY_HEAP_DEVICE_LOCAL_BIT);
2710 uint64_t gtt_heap_size = device->memory_properties.memoryHeaps[gtt_heap_idx].size;
2711 uint64_t vram_vis_heap_size = device->memory_properties.memoryHeaps[vram_vis_heap_idx].size;
2713 uint64_t vram_vis_internal_usage = device->ws->query_value(device->ws, RADEON_ALLOCATED_VRAM_VIS) +
2714 device->ws->query_value(device->ws, RADEON_ALLOCATED_VRAM);
2715 uint64_t gtt_internal_usage = device->ws->query_value(device->ws, RADEON_ALLOCATED_GTT);
2720 uint64_t total_system_usage = device->ws->query_value(device->ws, RADEON_VRAM_VIS_USAGE) +
2721 device->ws->query_value(device->ws, RADEON_GTT_USAGE);
2735 device->rad_info.gart_page_size);
2743 unsigned mask = device->heaps;
2751 internal_usage = device->ws->query_value(device->ws, RADEON_ALLOCATED_VRAM);
2752 system_usage = device->ws->query_value(device->ws, RADEON_VRAM_USAGE);
2755 internal_usage = device->ws->query_value(device->ws, RADEON_ALLOCATED_VRAM_VIS);
2756 if (!(device->heaps & RADV_HEAP_VRAM))
2757 internal_usage += device->ws->query_value(device->ws, RADEON_ALLOCATED_VRAM);
2758 system_usage = device->ws->query_value(device->ws, RADEON_VRAM_VIS_USAGE);
2761 internal_usage = device->ws->query_value(device->ws, RADEON_ALLOCATED_GTT);
2762 system_usage = device->ws->query_value(device->ws, RADEON_GTT_USAGE);
2768 uint64_t free_space = device->memory_properties.memoryHeaps[heap].size -
2769 MIN2(device->memory_properties.memoryHeaps[heap].size, total_usage);
2807 RADV_FROM_HANDLE(radv_device, device, _device);
2811 const struct radv_physical_device *physical_device = device->physical_device;
2851 radv_queue_init(struct radv_device *device, struct radv_queue *queue, int idx,
2855 queue->device = device;
2857 queue->hw_ctx = device->hw_ctx[queue->priority];
2858 queue->state.qf = vk_queue_to_radv(device->physical_device, create_info->queueFamilyIndex);
2860 VkResult result = vk_queue_init(&queue->vk, &device->vk, create_info, idx);
2906 radv_queue_state_finish(queue->ace_internal_state, queue->device->ws);
2910 radv_queue_state_finish(&queue->state, queue->device->ws);
2929 radv_device_init_border_color(struct radv_device *device)
2933 result = device->ws->buffer_create(
2934 device->ws, RADV_BORDER_COLOR_BUFFER_SIZE, 4096, RADEON_DOMAIN_VRAM,
2936 RADV_BO_PRIORITY_SHADER, 0, &device->border_color_data.bo);
2939 return vk_error(device, result);
2941 result = device->ws->buffer_make_resident(device->ws, device->border_color_data.bo, true);
2943 return vk_error(device, result);
2945 device->border_color_data.colors_gpu_ptr = device->ws->buffer_map(device->border_color_data.bo);
2946 if (!device->border_color_data.colors_gpu_ptr)
2947 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
2948 mtx_init(&device->border_color_data.mutex, mtx_plain);
2954 radv_device_finish_border_color(struct radv_device *device)
2956 if (device->border_color_data.bo) {
2957 device->ws->buffer_make_resident(device->ws, device->border_color_data.bo, false);
2958 device->ws->buffer_destroy(device->ws, device->border_color_data.bo);
2960 mtx_destroy(&device->border_color_data.mutex);
2965 radv_device_init_vs_prologs(struct radv_device *device)
2967 u_rwlock_init(&device->vs_prologs_lock);
2968 device->vs_prologs = _mesa_hash_table_create(NULL, &radv_hash_vs_prolog, &radv_cmp_vs_prolog);
2969 if (!device->vs_prologs)
2970 return vk_error(device->physical_device->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
2973 if (device->instance->debug_flags & RADV_DEBUG_DUMP_PROLOGS)
2987 key.is_ngg = device->physical_device->use_ngg;
2989 key.wave32 = device->physical_device->ge_wave_size == 32;
2997 device->simple_vs_prologs[i - 1] = radv_create_vs_prolog(device, &key);
2998 if (!device->simple_vs_prologs[i - 1])
2999 return vk_error(device->physical_device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
3015 struct radv_shader_part *prolog = radv_create_vs_prolog(device, &key);
3017 return vk_error(device->physical_device->instance, VK_ERROR_OUT_OF_DEVICE_MEMORY);
3021 device->instance_rate_vs_prologs[idx++] = prolog;
3025 assert(idx == ARRAY_SIZE(device->instance_rate_vs_prologs));
3031 radv_device_finish_vs_prologs(struct radv_device *device)
3033 if (device->vs_prologs) {
3034 hash_table_foreach(device->vs_prologs, entry)
3037 radv_shader_part_destroy(device, entry->data);
3039 _mesa_hash_table_destroy(device->vs_prologs, NULL);
3042 for (unsigned i = 0; i < ARRAY_SIZE(device->simple_vs_prologs); i++)
3043 radv_shader_part_destroy(device, device->simple_vs_prologs[i]);
3045 for (unsigned i = 0; i < ARRAY_SIZE(device->instance_rate_vs_prologs); i++)
3046 radv_shader_part_destroy(device, device->instance_rate_vs_prologs[i]);
3050 radv_device_init_vrs_state(struct radv_device *device)
3077 result = radv_CreateImage(radv_device_to_handle(device), &image_create_info,
3078 &device->meta_state.alloc, &image);
3089 result = radv_CreateBuffer(radv_device_to_handle(device), &buffer_create_info,
3090 &device->meta_state.alloc, &buffer);
3101 radv_GetBufferMemoryRequirements2(radv_device_to_handle(device), &info, &mem_req);
3108 result = radv_AllocateMemory(radv_device_to_handle(device), &alloc_info,
3109 &device->meta_state.alloc, &mem);
3120 result = radv_BindBufferMemory2(radv_device_to_handle(device), 1, &bind_info);
3124 device->vrs.image = radv_image_from_handle(image);
3125 device->vrs.buffer = radv_buffer_from_handle(buffer);
3126 device->vrs.mem = radv_device_memory_from_handle(mem);
3131 radv_FreeMemory(radv_device_to_handle(device), mem, &device->meta_state.alloc);
3133 radv_DestroyBuffer(radv_device_to_handle(device), buffer, &device->meta_state.alloc);
3135 radv_DestroyImage(radv_device_to_handle(device), image, &device->meta_state.alloc);
3141 radv_device_finish_vrs_image(struct radv_device *device)
3143 if (!device->vrs.image)
3146 radv_FreeMemory(radv_device_to_handle(device), radv_device_memory_to_handle(device->vrs.mem),
3147 &device->meta_state.alloc);
3148 radv_DestroyBuffer(radv_device_to_handle(device), radv_buffer_to_handle(device->vrs.buffer),
3149 &device->meta_state.alloc);
3150 radv_DestroyImage(radv_device_to_handle(device), radv_image_to_handle(device->vrs.image),
3151 &device->meta_state.alloc);
3206 struct radv_device *device = data;
3207 struct radv_notifier *notifier = &device->notifier;
3223 device->force_vrs = radv_parse_force_vrs_config_file(file);
3225 fprintf(stderr, "radv: Updated the per-vertex VRS rate to '%d'.\n", device->force_vrs);
3243 radv_device_init_notifier(struct radv_device *device)
3248 struct radv_notifier *notifier = &device->notifier;
3260 ret = thrd_create(¬ifier->thread, radv_notifier_thread_run, device);
3276 radv_device_finish_notifier(struct radv_device *device)
3279 struct radv_notifier *notifier = &device->notifier;
3292 radv_device_finish_perf_counter_lock_cs(struct radv_device *device)
3294 if (!device->perf_counter_lock_cs)
3298 if (device->perf_counter_lock_cs[i])
3299 device->ws->cs_destroy(device->perf_counter_lock_cs[i]);
3302 free(device->perf_counter_lock_cs);
3311 struct radv_device *device;
3423 device = vk_zalloc2(&physical_device->instance->vk.alloc, pAllocator, sizeof(*device), 8,
3425 if (!device)
3447 vk_device_init(&device->vk, &physical_device->vk, &dispatch_table, pCreateInfo, pAllocator);
3449 vk_free(&device->vk.alloc, device);
3453 device->instance = physical_device->instance;
3454 device->physical_device = physical_device;
3455 simple_mtx_init(&device->trace_mtx, mtx_plain);
3456 simple_mtx_init(&device->pstate_mtx, mtx_plain);
3458 device->ws = physical_device->ws;
3459 vk_device_set_drm_fd(&device->vk, device->ws->get_fd(device->ws));
3464 device->use_global_bo_list = global_bo_list ||
3465 (device->instance->perftest_flags & RADV_PERFTEST_BO_LIST) ||
3466 device->vk.enabled_extensions.EXT_descriptor_indexing ||
3467 device->vk.enabled_extensions.EXT_buffer_device_address ||
3468 device->vk.enabled_extensions.KHR_buffer_device_address ||
3469 device->vk.enabled_extensions.KHR_ray_tracing_pipeline ||
3470 device->vk.enabled_extensions.KHR_acceleration_structure ||
3471 device->vk.enabled_extensions.VALVE_descriptor_set_host_mapping;
3473 device->robust_buffer_access = robust_buffer_access || robust_buffer_access2;
3474 device->robust_buffer_access2 = robust_buffer_access2;
3476 device->attachment_vrs_enabled = attachment_vrs_enabled;
3478 device->image_float32_atomics = image_float32_atomics;
3480 device->image_2d_view_of_3d = image_2d_view_of_3d;
3482 device->primitives_generated_query = primitives_generated_query;
3483 device->uses_device_generated_commands = use_dgc;
3485 radv_init_shader_arenas(device);
3487 device->overallocation_disallowed = overallocation_disallowed;
3488 mtx_init(&device->overallocation_mutex, mtx_plain);
3497 if (device->hw_ctx[priority])
3500 result = device->ws->ctx_create(device->ws, priority, &device->hw_ctx[priority]);
3511 device->queues[qfi] =
3512 vk_alloc(&device->vk.alloc, queue_create->queueCount * sizeof(struct radv_queue), 8,
3514 if (!device->queues[qfi]) {
3519 memset(device->queues[qfi], 0, queue_create->queueCount * sizeof(struct radv_queue));
3521 device->queue_count[qfi] = queue_create->queueCount;
3524 result = radv_queue_init(device, &device->queues[qfi][q], q, queue_create, global_priority);
3529 device->private_sdma_queue = VK_NULL_HANDLE;
3531 device->pbb_allowed = device->physical_device->rad_info.gfx_level >= GFX9 &&
3532 !(device->instance->debug_flags & RADV_DEBUG_NOBINNING);
3547 device->scratch_waves =
3550 device->dispatch_initiator = S_00B800_COMPUTE_SHADER_EN(1);
3552 if (device->physical_device->rad_info.gfx_level >= GFX7) {
3556 device->dispatch_initiator |= S_00B800_ORDER_MODE(1);
3563 device->dispatch_initiator_task =
3564 device->dispatch_initiator | S_00B800_DISABLE_DISP_PREMPT_EN(1);
3566 if (device->instance->debug_flags & RADV_DEBUG_HANG) {
3572 if (!radv_init_trace(device))
3585 device->instance->debug_flags |= RADV_DEBUG_SYNC_SHADERS;
3587 radv_dump_enabled_options(device, stderr);
3591 if (device->physical_device->rad_info.gfx_level < GFX8 ||
3592 device->physical_device->rad_info.gfx_level > GFX10_3) {
3599 if (!radv_thread_trace_init(device))
3604 device->thread_trace.buffer_size / (1024 * 1024),
3609 if (device->physical_device->rad_info.gfx_level >= GFX10) {
3610 if (!radv_spm_init(device))
3614 device->physical_device->name);
3621 assert(device->physical_device->rad_info.gfx_level == GFX8);
3632 if (!radv_trap_handler_init(device))
3636 if (device->physical_device->rad_info.gfx_level >= GFX10_3) {
3640 device->force_vrs = radv_parse_force_vrs_config_file(file);
3642 if (radv_device_init_notifier(device)) {
3643 device->force_vrs_enabled = true;
3650 device->force_vrs = radv_parse_vrs_rates(vrs_rates);
3651 device->force_vrs_enabled = device->force_vrs != RADV_FORCE_VRS_1x1;
3656 device->load_grid_size_from_user_sgpr = device->physical_device->rad_info.gfx_level >= GFX10_3;
3658 device->keep_shader_info = keep_shader_info;
3659 result = radv_device_init_meta(device);
3663 radv_device_init_msaa(device);
3667 result = radv_device_init_border_color(device);
3673 result = radv_device_init_vs_prologs(device);
3678 if (device->physical_device->rad_info.gfx_level >= GFX7)
3679 cik_create_gfx_config(device);
3688 result = radv_CreatePipelineCache(radv_device_to_handle(device), &ci, NULL, &pc);
3692 device->mem_cache = radv_pipeline_cache_from_handle(pc);
3694 device->force_aniso = MIN2(16, radv_get_int_debug_option("RADV_TEX_ANISO", -1));
3695 if (device->force_aniso >= 0) {
3697 1 << util_logbase2(device->force_aniso));
3703 device->ws->buffer_create(device->ws, bo_size, 4096, RADEON_DOMAIN_GTT,
3705 RADV_BO_PRIORITY_UPLOAD_BUFFER, 0, &device->perf_counter_bo);
3709 device->perf_counter_lock_cs =
3711 if (!device->perf_counter_lock_cs) {
3716 if (!device->physical_device->ac_perfcounters.blocks) {
3722 *pDevice = radv_device_to_handle(device);
3726 radv_DestroyPipelineCache(radv_device_to_handle(device), pc, NULL);
3728 radv_device_finish_meta(device);
3730 radv_thread_trace_finish(device);
3732 radv_spm_finish(device);
3734 radv_trap_handler_finish(device);
3735 radv_finish_trace(device);
3737 radv_device_finish_perf_counter_lock_cs(device);
3738 if (device->perf_counter_bo)
3739 device->ws->buffer_destroy(device->ws, device->perf_counter_bo);
3740 if (device->gfx_init)
3741 device->ws->buffer_destroy(device->ws, device->gfx_init);
3743 radv_device_finish_notifier(device);
3744 radv_device_finish_vs_prologs(device);
3745 radv_device_finish_border_color(device);
3748 for (unsigned q = 0; q < device->queue_count[i]; q++)
3749 radv_queue_finish(&device->queues[i][q]);
3750 if (device->queue_count[i])
3751 vk_free(&device->vk.alloc, device->queues[i]);
3755 if (device->hw_ctx[i])
3756 device->ws->ctx_destroy(device->hw_ctx[i]);
3759 simple_mtx_destroy(&device->pstate_mtx);
3760 simple_mtx_destroy(&device->trace_mtx);
3761 mtx_destroy(&device->overallocation_mutex);
3763 vk_device_finish(&device->vk);
3764 vk_free(&device->vk.alloc, device);
3771 RADV_FROM_HANDLE(radv_device, device, _device);
3773 if (!device)
3776 radv_device_finish_perf_counter_lock_cs(device);
3777 if (device->perf_counter_bo)
3778 device->ws->buffer_destroy(device->ws, device->perf_counter_bo);
3780 if (device->gfx_init)
3781 device->ws->buffer_destroy(device->ws, device->gfx_init);
3783 radv_device_finish_notifier(device);
3784 radv_device_finish_vs_prologs(device);
3785 radv_device_finish_border_color(device);
3786 radv_device_finish_vrs_image(device);
3789 for (unsigned q = 0; q < device->queue_count[i]; q++)
3790 radv_queue_finish(&device->queues[i][q]);
3791 if (device->queue_count[i])
3792 vk_free(&device->vk.alloc, device->queues[i]);
3794 if (device->private_sdma_queue != VK_NULL_HANDLE) {
3795 radv_queue_finish(device->private_sdma_queue);
3796 vk_free(&device->vk.alloc, device->private_sdma_queue);
3800 if (device->hw_ctx[i])
3801 device->ws->ctx_destroy(device->hw_ctx[i]);
3804 mtx_destroy(&device->overallocation_mutex);
3805 simple_mtx_destroy(&device->pstate_mtx);
3806 simple_mtx_destroy(&device->trace_mtx);
3808 radv_device_finish_meta(device);
3810 VkPipelineCache pc = radv_pipeline_cache_to_handle(device->mem_cache);
3811 radv_DestroyPipelineCache(radv_device_to_handle(device), pc, NULL);
3813 radv_trap_handler_finish(device);
3814 radv_finish_trace(device);
3816 radv_destroy_shader_arenas(device);
3818 radv_thread_trace_finish(device);
3820 radv_spm_finish(device);
3822 vk_device_finish(&device->vk);
3823 vk_free(&device->vk.alloc, device);
3852 radv_fill_shader_rings(struct radv_device *device, uint32_t *map, bool add_sample_positions,
3873 if (device->physical_device->rad_info.gfx_level >= GFX11)
3878 if (device->physical_device->rad_info.gfx_level >= GFX11) {
3881 } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
3884 } else if (device->physical_device->rad_info.gfx_level >= GFX8) {
3902 if (device->physical_device->rad_info.gfx_level >= GFX11) {
3905 } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
3928 if (device->physical_device->rad_info.gfx_level >= GFX11) {
3931 } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
3949 if (device->physical_device->rad_info.gfx_level >= GFX11)
3954 if (device->physical_device->rad_info.gfx_level >= GFX11) {
3957 } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
3960 } else if (device->physical_device->rad_info.gfx_level >= GFX8) {
3974 uint64_t tess_offchip_va = tess_va + device->physical_device->hs.tess_offchip_ring_offset;
3978 desc[2] = device->physical_device->hs.tess_factor_ring_size;
3982 if (device->physical_device->rad_info.gfx_level >= GFX11) {
3985 } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
3995 desc[6] = device->physical_device->hs.tess_offchip_ring_size;
3999 if (device->physical_device->rad_info.gfx_level >= GFX11) {
4002 } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
4015 uint64_t task_draw_ring_va = task_va + device->physical_device->task_info.draw_ring_offset;
4016 uint64_t task_payload_ring_va = task_va + device->physical_device->task_info.payload_ring_offset;
4020 desc[2] = device->physical_device->task_info.num_entries * AC_TASK_DRAW_ENTRY_BYTES;
4024 if (device->physical_device->rad_info.gfx_level >= GFX11) {
4028 assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
4035 desc[6] = device->physical_device->task_info.num_entries * AC_TASK_PAYLOAD_ENTRY_BYTES;
4039 if (device->physical_device->rad_info.gfx_level >= GFX11) {
4043 assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
4060 if (device->physical_device->rad_info.gfx_level >= GFX11) {
4064 assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
4074 memcpy(desc, device->sample_locations_1x, 8);
4076 memcpy(desc, device->sample_locations_2x, 16);
4078 memcpy(desc, device->sample_locations_4x, 32);
4080 memcpy(desc, device->sample_locations_8x, 64);
4085 radv_emit_gs_ring_sizes(struct radv_device *device, struct radeon_cmdbuf *cs,
4093 radv_cs_add_buffer(device->ws, cs, esgs_ring_bo);
4096 radv_cs_add_buffer(device->ws, cs, gsvs_ring_bo);
4098 if (device->physical_device->rad_info.gfx_level >= GFX7) {
4110 radv_emit_tess_factor_ring(struct radv_device *device, struct radeon_cmdbuf *cs,
4118 tf_ring_size = device->physical_device->hs.tess_factor_ring_size / 4;
4121 radv_cs_add_buffer(device->ws, cs, tess_rings_bo);
4123 if (device->physical_device->rad_info.gfx_level >= GFX7) {
4124 if (device->physical_device->rad_info.gfx_level >= GFX11) {
4126 tf_ring_size /= device->physical_device->rad_info.max_se;
4132 if (device->physical_device->rad_info.gfx_level >= GFX10) {
4135 } else if (device->physical_device->rad_info.gfx_level == GFX9) {
4139 radeon_set_uconfig_reg(cs, R_03093C_VGT_HS_OFFCHIP_PARAM, device->physical_device->hs.hs_offchip_param);
4143 radeon_set_config_reg(cs, R_0089B0_VGT_HS_OFFCHIP_PARAM, device->physical_device->hs.hs_offchip_param);
4148 radv_initialise_task_control_buffer(struct radv_device *device,
4151 uint32_t *ptr = (uint32_t *)device->ws->buffer_map(task_rings_bo);
4155 const uint32_t num_entries = device->physical_device->task_info.num_entries;
4157 const uint64_t task_draw_ring_va = task_va + device->physical_device->task_info.draw_ring_offset;
4175 device->ws->buffer_unmap(task_rings_bo);
4180 radv_emit_task_rings(struct radv_device *device, struct radeon_cmdbuf *cs,
4188 radv_cs_add_buffer(device->ws, cs, task_rings_bo);
4199 radv_emit_graphics_scratch(struct radv_device *device, struct radeon_cmdbuf *cs,
4203 struct radeon_info *info = &device->physical_device->rad_info;
4208 radv_cs_add_buffer(device->ws, cs, scratch_bo);
4228 radv_emit_compute_scratch(struct radv_device *device, struct radeon_cmdbuf *cs,
4232 struct radeon_info *info = &device->physical_device->rad_info;
4242 if (device->physical_device->rad_info.gfx_level >= GFX11)
4247 radv_cs_add_buffer(device->ws, cs, compute_scratch_bo);
4267 radv_emit_compute_shader_pointers(struct radv_device *device, struct radeon_cmdbuf *cs,
4274 radv_cs_add_buffer(device->ws, cs, descriptor_bo);
4279 radv_emit_shader_pointer(device, cs, R_00B908_COMPUTE_USER_DATA_2, va, true);
4283 radv_emit_graphics_shader_pointers(struct radv_device *device, struct radeon_cmdbuf *cs,
4293 radv_cs_add_buffer(device->ws, cs, descriptor_bo);
4295 if (device->physical_device->rad_info.gfx_level >= GFX11) {
4301 radv_emit_shader_pointer(device, cs, regs[i], va, true);
4303 } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
4309 radv_emit_shader_pointer(device, cs, regs[i], va, true);
4311 } else if (device->physical_device->rad_info.gfx_level == GFX9) {
4317 radv_emit_shader_pointer(device, cs, regs[i], va, true);
4325 radv_emit_shader_pointer(device, cs, regs[i], va, true);
4331 radv_init_graphics_state(struct radeon_cmdbuf *cs, struct radv_device *device)
4333 if (device->gfx_init) {
4334 uint64_t va = radv_buffer_get_va(device->gfx_init);
4339 radeon_emit(cs, device->gfx_init_size_dw & 0xffff);
4341 radv_cs_add_buffer(device->ws, cs, device->gfx_init);
4343 si_emit_graphics(device, cs);
4348 radv_init_compute_state(struct radeon_cmdbuf *cs, struct radv_device *device)
4350 si_emit_compute(device, cs);
4354 radv_update_preamble_cs(struct radv_queue_state *queue, struct radv_device *device,
4357 struct radeon_winsys *ws = device->ws;
4411 ws, device->physical_device->hs.tess_offchip_ring_offset + device->physical_device->hs.tess_offchip_ring_size, 256,
4418 assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
4426 result = ws->buffer_create(ws, device->physical_device->task_info.bo_size_bytes, 256,
4432 result = radv_initialise_task_control_buffer(device, task_rings_bo);
4438 assert(device->physical_device->rad_info.gfx_level >= GFX10_3);
4448 assert(device->physical_device->rad_info.gfx_level >= GFX10);
4460 assert(device->physical_device->rad_info.gfx_level >= GFX10);
4505 if (device->physical_device->rad_info.gfx_level >= GFX11)
4515 radv_fill_shader_rings(device, map, add_sample_positions, needs->esgs_ring_size,
4526 if (!(device->instance->debug_flags & RADV_DEBUG_NO_IBS) &&
4527 device->physical_device->rad_info.gfx_level >= GFX7)
4538 cs = ws->cs_create(ws, radv_queue_family_to_ring(device->physical_device, queue->qf));
4552 radv_init_graphics_state(cs, device);
4562 radv_emit_gs_ring_sizes(device, cs, esgs_ring_bo, needs->esgs_ring_size, gsvs_ring_bo,
4564 radv_emit_tess_factor_ring(device, cs, tess_rings_bo);
4565 radv_emit_task_rings(device, cs, task_rings_bo, false);
4566 radv_emit_graphics_shader_pointers(device, cs, descriptor_bo);
4567 radv_emit_compute_scratch(device, cs, needs->compute_scratch_size_per_wave,
4569 radv_emit_graphics_scratch(device, cs, needs->scratch_size_per_wave, needs->scratch_waves,
4573 radv_init_compute_state(cs, device);
4580 radv_emit_task_rings(device, cs, task_rings_bo, true);
4581 radv_emit_compute_shader_pointers(device, cs, descriptor_bo);
4582 radv_emit_compute_scratch(device, cs, needs->compute_scratch_size_per_wave,
4596 const enum amd_gfx_level gfx_level = device->physical_device->rad_info.gfx_level;
4694 radv_create_perf_counter_lock_cs(struct radv_device *device, unsigned pass, bool unlock)
4696 struct radeon_cmdbuf **cs_ref = &device->perf_counter_lock_cs[pass * 2 + (unlock ? 1 : 0)];
4702 cs = device->ws->cs_create(device->ws, AMD_IP_GFX);
4706 ASSERTED unsigned cdw = radeon_check_space(device->ws, cs, 21);
4709 uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET;
4721 uint64_t va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_PASS_OFFSET;
4742 uint64_t mutex_va = radv_buffer_get_va(device->perf_counter_bo) + PERF_CTR_BO_LOCK_OFFSET;
4755 VkResult result = device->ws->cs_finalize(cs);
4757 device->ws->cs_destroy(cs);
4765 device->ws->cs_destroy(cs);
4772 radv_sparse_buffer_bind_memory(struct radv_device *device, const VkSparseBufferMemoryBindInfo *bind)
4783 result = device->ws->buffer_virtual_bind(device->ws, buffer->bo,
4794 radv_sparse_image_opaque_bind_memory(struct radv_device *device,
4806 result = device->ws->buffer_virtual_bind(device->ws, image->bindings[0].bo,
4817 radv_sparse_image_bind_memory(struct radv_device *device, const VkSparseImageMemoryBindInfo *bind)
4844 if (device->physical_device->rad_info.gfx_level >= GFX9) {
4863 result = device->ws->buffer_virtual_bind(device->ws, image->bindings[0].bo, offset, size,
4872 result = device->ws->buffer_virtual_bind(device->ws,
4885 radv_update_preambles(struct radv_queue_state *queue, struct radv_device *device,
4949 return radv_update_preamble_cs(queue, device, &needs);
4974 return radv_update_preamble_cs(queue->ace_internal_state, queue->device, &needs);
5017 radv_queue_submit_bind_sparse_memory(struct radv_device *device, struct vk_queue_submit *submission)
5020 VkResult result = radv_sparse_buffer_bind_memory(device, submission->buffer_binds + i);
5027 radv_sparse_image_opaque_bind_memory(device, submission->image_opaque_binds + i);
5033 VkResult result = radv_sparse_image_bind_memory(device, submission->image_binds + i);
5050 return queue->device->ws->cs_submit(ctx, 1, &submit, submission->wait_count, submission->waits,
5067 const uint32_t max_cs_submission = queue->device->trace_bo ? 1 : RADV_MAX_IBS_PER_SUBMIT;
5103 if (queue->device->trace_bo)
5104 *queue->device->trace_id_ptr = 0;
5119 result = queue->device->ws->cs_submit(
5126 if (queue->device->trace_bo) {
5130 if (queue->device->tma_bo) {
5149 uint32_t max_cs_submission = queue->device->trace_bo ? 1 : RADV_MAX_IBS_PER_SUBMIT;
5156 result = radv_update_preambles(&queue->state, queue->device, submission->command_buffers,
5161 if (queue->device->trace_bo)
5162 simple_mtx_lock(&queue->device->trace_mtx);
5186 radv_create_perf_counter_lock_cs(queue->device, submission->perf_pass_index, false);
5188 radv_create_perf_counter_lock_cs(queue->device, submission->perf_pass_index, true);
5220 if (queue->device->trace_bo)
5221 *queue->device->trace_id_ptr = 0;
5225 result = queue->device->ws->cs_submit(
5232 if (queue->device->trace_bo) {
5236 if (queue->device->tma_bo) {
5246 if (queue->device->trace_bo)
5247 simple_mtx_unlock(&queue->device->trace_mtx);
5258 result = radv_queue_submit_bind_sparse_memory(queue->device, submission);
5278 * to submit the same job again to this device.
5280 result = vk_device_set_lost(&queue->device->vk, "vkQueueSubmit() failed");
5296 VkResult result = queue->device->ws->cs_submit(ctx, 1, &submit, 0, NULL, 0, NULL, false);
5373 radv_get_memory_fd(struct radv_device *device, struct radv_device_memory *memory, int *pFD)
5378 radv_init_metadata(device, memory->image, &metadata);
5379 device->ws->buffer_set_metadata(device->ws, memory->bo, &metadata);
5382 return device->ws->buffer_get_fd(device->ws, memory->bo, pFD);
5386 radv_device_memory_init(struct radv_device_memory *mem, struct radv_device *device,
5390 vk_object_base_init(&device->vk, &mem->base, VK_OBJECT_TYPE_DEVICE_MEMORY);
5402 radv_free_memory(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
5414 if (device->overallocation_disallowed) {
5415 mtx_lock(&device->overallocation_mutex);
5416 device->allocated_memory_size[mem->heap_index] -= mem->alloc_size;
5417 mtx_unlock(&device->overallocation_mutex);
5420 if (device->use_global_bo_list)
5421 device->ws->buffer_make_resident(device->ws, mem->bo, false);
5422 device->ws->buffer_destroy(device->ws, mem->bo);
5427 vk_free2(&device->vk.alloc, pAllocator, mem);
5431 radv_alloc_memory(struct radv_device *device, const VkMemoryAllocateInfo *pAllocateInfo,
5464 vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*mem), 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
5466 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
5468 radv_device_memory_init(mem, device, NULL);
5522 result = radv_import_ahb_memory(device, mem, priority, ahb_import_info);
5527 result = radv_create_ahb_memory(device, mem, priority, pAllocateInfo);
5533 result = device->ws->buffer_from_fd(device->ws, import_info->fd, priority, &mem->bo, NULL);
5544 device->ws->buffer_get_metadata(device->ws, mem->bo, &metadata);
5552 result = radv_image_create_layout(device, create_info, NULL, mem->image);
5554 device->ws->buffer_destroy(device->ws, mem->bo);
5560 result = device->ws->buffer_from_ptr(device->ws, host_ptr_info->pHostPointer,
5572 device->physical_device->memory_properties.memoryTypes[pAllocateInfo->memoryTypeIndex]
5574 domain = device->physical_device->memory_domains[pAllocateInfo->memoryTypeIndex];
5575 flags |= device->physical_device->memory_flags[pAllocateInfo->memoryTypeIndex];
5579 if (device->use_global_bo_list) {
5588 if (device->instance->zero_vram)
5591 if (device->overallocation_disallowed) {
5593 device->physical_device->memory_properties.memoryHeaps[heap_index].size;
5595 mtx_lock(&device->overallocation_mutex);
5596 if (device->allocated_memory_size[heap_index] + alloc_size > total_size) {
5597 mtx_unlock(&device->overallocation_mutex);
5601 device->allocated_memory_size[heap_index] += alloc_size;
5602 mtx_unlock(&device->overallocation_mutex);
5605 result = device->ws->buffer_create(device->ws, alloc_size,
5606 device->physical_device->rad_info.max_alignment, domain,
5610 if (device->overallocation_disallowed) {
5611 mtx_lock(&device->overallocation_mutex);
5612 device->allocated_memory_size[heap_index] -= alloc_size;
5613 mtx_unlock(&device->overallocation_mutex);
5623 if (device->use_global_bo_list) {
5624 result = device->ws->buffer_make_resident(device->ws, mem->bo, true);
5635 radv_free_memory(device, pAllocator, mem);
5644 RADV_FROM_HANDLE(radv_device, device, _device);
5645 return radv_alloc_memory(device, pAllocateInfo, pAllocator, pMem);
5651 RADV_FROM_HANDLE(radv_device, device, _device);
5654 radv_free_memory(device, pAllocator, mem);
5661 RADV_FROM_HANDLE(radv_device, device, _device);
5667 *ppData = device->ws->buffer_map(mem->bo);
5674 return vk_error(device, VK_ERROR_MEMORY_MAP_FAILED);
5680 RADV_FROM_HANDLE(radv_device, device, _device);
5684 device->ws->buffer_unmap(mem->bo);
5702 radv_get_buffer_memory_requirements(struct radv_device *device, VkDeviceSize size,
5707 ((1u << device->physical_device->memory_properties.memoryTypeCount) - 1u) &
5708 ~device->physical_device->memory_types_32bit;
5719 if ((usage & VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT) && device->uses_device_generated_commands)
5721 device->physical_device->memory_types_32bit;
5758 RADV_FROM_HANDLE(radv_device, device, _device);
5761 radv_get_buffer_memory_requirements(device, buffer->vk.size, buffer->vk.create_flags,
5770 RADV_FROM_HANDLE(radv_device, device, _device);
5772 radv_get_buffer_memory_requirements(device, pInfo->pCreateInfo->size, pInfo->pCreateInfo->flags,
5780 RADV_FROM_HANDLE(radv_device, device, _device);
5784 ((1u << device->physical_device->memory_properties.memoryTypeCount) - 1u) &
5785 ~device->physical_device->memory_types_32bit;
5807 radv_GetDeviceImageMemoryRequirements(VkDevice device,
5818 result = radv_CreateImage(device, pInfo->pCreateInfo, NULL, &image);
5826 radv_GetImageMemoryRequirements2(device, &info2, pMemoryRequirements);
5828 radv_DestroyImage(device, image, NULL);
5832 radv_GetDeviceMemoryCommitment(VkDevice device, VkDeviceMemory memory,
5842 RADV_FROM_HANDLE(radv_device, device, _device);
5860 return vk_errorf(device, VK_ERROR_UNKNOWN,
5875 RADV_FROM_HANDLE(radv_device, device, _device);
5893 return vk_errorf(device, VK_ERROR_UNKNOWN,
5927 radv_destroy_event(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
5931 device->ws->buffer_destroy(device->ws, event->bo);
5934 vk_free2(&device->vk.alloc, pAllocator, event);
5941 RADV_FROM_HANDLE(radv_device, device, _device);
5947 event = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*event), 8,
5950 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
5952 vk_object_base_init(&device->vk, &event->base, VK_OBJECT_TYPE_EVENT);
5962 result = device->ws->buffer_create(
5963 device->ws, 8, 8, bo_domain,
5967 radv_destroy_event(device, pAllocator, event);
5968 return vk_error(device, result);
5972 event->map = (uint64_t *)device->ws->buffer_map(event->bo);
5974 radv_destroy_event(device, pAllocator, event);
5975 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
5987 RADV_FROM_HANDLE(radv_device, device, _device);
5993 radv_destroy_event(device, pAllocator, event);
5999 RADV_FROM_HANDLE(radv_device, device, _device);
6002 if (vk_device_is_lost(&device->vk))
6029 radv_buffer_init(struct radv_buffer *buffer, struct radv_device *device,
6038 vk_buffer_init(&device->vk, &buffer->vk, &createInfo);
6051 radv_destroy_buffer(struct radv_device *device, const VkAllocationCallbacks *pAllocator,
6055 device->ws->buffer_destroy(device->ws, buffer->bo);
6058 vk_free2(&device->vk.alloc, pAllocator, buffer);
6065 RADV_FROM_HANDLE(radv_device, device, _device);
6070 buffer = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*buffer), 8,
6073 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
6075 vk_buffer_init(&device->vk, &buffer->vk, pCreateInfo);
6091 device->ws->buffer_create(device->ws, align64(buffer->vk.size, 4096), 4096, 0, flags,
6094 radv_destroy_buffer(device, pAllocator, buffer);
6095 return vk_error(device, result);
6107 RADV_FROM_HANDLE(radv_device, device, _device);
6113 radv_destroy_buffer(device, pAllocator, buffer);
6117 radv_GetBufferDeviceAddress(VkDevice device, const VkBufferDeviceAddressInfo *pInfo)
6124 radv_GetBufferOpaqueCaptureAddress(VkDevice device, const VkBufferDeviceAddressInfo *pInfo)
6131 radv_GetDeviceMemoryOpaqueCaptureAddress(VkDevice device,
6155 get_dcc_max_uncompressed_block_size(const struct radv_device *device,
6158 if (device->physical_device->rad_info.gfx_level < GFX10 && iview->image->info.samples > 1) {
6169 get_dcc_min_compressed_block_size(const struct radv_device *device)
6171 if (!device->physical_device->rad_info.has_dedicated_vram) {
6184 radv_init_dcc_control_reg(struct radv_device *device, struct radv_image_view *iview)
6186 unsigned max_uncompressed_block_size = get_dcc_max_uncompressed_block_size(device, iview);
6187 unsigned min_compressed_block_size = get_dcc_min_compressed_block_size(device);
6197 if (device->physical_device->rad_info.gfx_level >= GFX9) {
6227 if (device->physical_device->rad_info.gfx_level >= GFX11) {
6239 radv_initialise_color_surface(struct radv_device *device, struct radv_color_buffer_info *cb,
6254 if (device->physical_device->rad_info.gfx_level >= GFX11)
6265 if (device->physical_device->rad_info.gfx_level >= GFX9) {
6266 if (device->physical_device->rad_info.gfx_level >= GFX11) {
6269 } else if (device->physical_device->rad_info.gfx_level >= GFX10) {
6311 if (device->physical_device->rad_info.gfx_level >= GFX7)
6318 if (device->physical_device->rad_info.gfx_level >= GFX7)
6334 device->physical_device->rad_info.gfx_level <= GFX8)
6351 if (device->physical_device->rad_info.gfx_level >= GFX11)
6403 if (device->physical_device->rad_info.gfx_level >= GFX11)
6410 if (device->physical_device->rad_info.gfx_level == GFX6) {
6423 if (device->physical_device->rad_info.gfx_level == GFX8) {
6433 !(device->instance->debug_flags & RADV_DEBUG_NO_FAST_CLEARS))
6437 device->physical_device->rad_info.gfx_level < GFX11)
6440 cb->cb_dcc_control = radv_init_dcc_control_reg(device, iview);
6443 if (!radv_image_has_fmask(iview->image) && device->physical_device->rad_info.gfx_level == GFX6) {
6448 if (device->physical_device->rad_info.gfx_level >= GFX9) {
6457 if (device->physical_device->rad_info.gfx_level >= GFX10) {
6462 S_028EE0_RESOURCE_LEVEL(device->physical_device->rad_info.gfx_level >= GFX11 ? 0 : 1);
6475 radv_calc_decompress_on_z_planes(struct radv_device *device, struct radv_image_view *iview)
6481 if (device->physical_device->rad_info.gfx_level >= GFX9) {
6489 if (device->physical_device->rad_info.has_two_planes_iterate256_bug &&
6490 radv_image_get_iterate256(device, iview->image) &&
6491 !radv_image_tile_stencil_disabled(device, iview->image) &&
6545 radv_initialise_ds_surface(struct radv_device *device, struct radv_ds_buffer_info *ds,
6558 if (!device->instance->absolute_depth_bias) {
6584 if (device->physical_device->rad_info.gfx_level >= GFX10) {
6595 if (device->physical_device->rad_info.gfx_level >= GFX9) {
6604 S_028040_ITERATE_256(device->physical_device->rad_info.gfx_level >= GFX11);
6607 S_028044_ITERATE_256(device->physical_device->rad_info.gfx_level >= GFX11);
6609 if (device->physical_device->rad_info.gfx_level == GFX9) {
6622 unsigned max_zplanes = radv_calc_decompress_on_z_planes(device, iview);
6626 if (device->physical_device->rad_info.gfx_level >= GFX10) {
6627 bool iterate256 = radv_image_get_iterate256(device, iview->image);
6639 if (radv_image_tile_stencil_disabled(device, iview->image)) {
6648 if (device->physical_device->rad_info.gfx_level == GFX9) {
6652 if (radv_image_has_vrs_htile(device, iview->image)) {
6672 if (device->physical_device->rad_info.gfx_level >= GFX7) {
6673 struct radeon_info *info = &device->physical_device->rad_info;
6709 if (radv_image_tile_stencil_disabled(device, iview->image)) {
6719 unsigned max_zplanes = radv_calc_decompress_on_z_planes(device, iview);
6852 radv_get_max_anisotropy(struct radv_device *device, const VkSamplerCreateInfo *pCreateInfo)
6854 if (device->force_aniso >= 0)
6855 return device->force_aniso;
6864 radv_register_border_color(struct radv_device *device, VkClearColorValue value)
6868 mtx_lock(&device->border_color_data.mutex);
6871 if (!device->border_color_data.used[slot]) {
6873 util_memcpy_cpu_to_le32(&device->border_color_data.colors_gpu_ptr[slot], &value,
6876 device->border_color_data.used[slot] = true;
6881 mtx_unlock(&device->border_color_data.mutex);
6887 radv_unregister_border_color(struct radv_device *device, uint32_t slot)
6889 mtx_lock(&device->border_color_data.mutex);
6891 device->border_color_data.used[slot] = false;
6893 mtx_unlock(&device->border_color_data.mutex);
6897 radv_init_sampler(struct radv_device *device, struct radv_sampler *sampler,
6900 uint32_t max_aniso = radv_get_max_anisotropy(device, pCreateInfo);
6902 bool compat_mode = device->physical_device->rad_info.gfx_level == GFX8 ||
6903 device->physical_device->rad_info.gfx_level == GFX9;
6934 radv_register_border_color(device, custom_border_color->customBorderColor);
6965 if (device->physical_device->rad_info.gfx_level >= GFX10) {
6967 S_008F38_ANISO_OVERRIDE_GFX10(device->instance->disable_aniso_single_level);
6970 S_008F38_DISABLE_LSB_CEIL(device->physical_device->rad_info.gfx_level <= GFX8) |
6972 S_008F38_ANISO_OVERRIDE_GFX8(device->instance->disable_aniso_single_level &&
6973 device->physical_device->rad_info.gfx_level >= GFX8);
6976 if (device->physical_device->rad_info.gfx_level >= GFX11) {
6987 RADV_FROM_HANDLE(radv_device, device, _device);
6995 sampler = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*sampler), 8,
6998 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
7000 vk_object_base_init(&device->vk, &sampler->base, VK_OBJECT_TYPE_SAMPLER);
7002 radv_init_sampler(device, sampler, pCreateInfo);
7015 RADV_FROM_HANDLE(radv_device, device, _device);
7022 radv_unregister_border_color(device, sampler->border_color_slot);
7025 vk_free2(&device->vk.alloc, pAllocator, sampler);
7078 RADV_FROM_HANDLE(radv_device, device, _device);
7087 bool ret = radv_get_memory_fd(device, memory, pFD);
7089 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
7145 RADV_FROM_HANDLE(radv_device, device, _device);
7151 if (!device->ws->buffer_get_flags_from_fd(device->ws, fd, &domains, &flags))
7152 return vk_error(device, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7155 radv_compute_valid_memory_types(device->physical_device, domains, flags);
7166 return vk_error(device, VK_ERROR_INVALID_EXTERNAL_HANDLE);
7171 radv_GetDeviceGroupPeerMemoryFeatures(VkDevice device, uint32_t heapIndex,
7231 RADV_FROM_HANDLE(radv_device, device, _device);
7232 uint32_t clock_crystal_freq = device->physical_device->rad_info.clock_crystal_freq;
7246 pTimestamps[d] = device->ws->query_value(device->ws, RADEON_TIMESTAMP);
7368 radv_thread_trace_set_pstate(struct radv_device *device, bool enable)
7370 struct radeon_winsys *ws = device->ws;
7373 if (device->physical_device->rad_info.has_stable_pstate) {
7374 /* pstate is per-device; setting it for one ctx is sufficient.
7377 if (device->hw_ctx[i])
7378 return ws->ctx_set_pstate(device->hw_ctx[i], pstate) >= 0;
7385 radv_device_acquire_performance_counters(struct radv_device *device)
7388 simple_mtx_lock(&device->pstate_mtx);
7390 if (device->pstate_cnt == 0) {
7391 result = radv_thread_trace_set_pstate(device, true);
7393 ++device->pstate_cnt;
7396 simple_mtx_unlock(&device->pstate_mtx);
7401 radv_device_release_performance_counters(struct radv_device *device)
7403 simple_mtx_lock(&device->pstate_mtx);
7405 if (--device->pstate_cnt == 0)
7406 radv_thread_trace_set_pstate(device, false);
7408 simple_mtx_unlock(&device->pstate_mtx);
7414 RADV_FROM_HANDLE(radv_device, device, _device);
7415 bool result = radv_device_acquire_performance_counters(device);
7422 RADV_FROM_HANDLE(radv_device, device, _device);
7423 radv_device_release_performance_counters(device);