Lines Matching defs:device
58 get_nir_options_for_stage(struct radv_physical_device *device, gl_shader_stage stage)
61 device->instance->debug_flags & RADV_DEBUG_SPLIT_FMA;
62 device->nir_options[stage] = (nir_shader_compiler_options){
85 .lower_ffma16 = split_fma || device->rad_info.gfx_level < GFX9,
86 .lower_ffma32 = split_fma || device->rad_info.gfx_level < GFX10_3,
91 .lower_iadd_sat = device->rad_info.gfx_level <= GFX8,
96 .has_sdot_4x8 = device->rad_info.has_accelerated_dot_product,
97 .has_udot_4x8 = device->rad_info.has_accelerated_dot_product,
98 .has_dot_2x16 = device->rad_info.has_accelerated_dot_product,
101 .has_fmulz = !device->use_llvm || LLVM_VERSION_MAJOR >= 12,
120 radv_get_nir_options(struct radv_physical_device *device)
123 get_nir_options_for_stage(device, stage);
147 radv_can_dump_shader(struct radv_device *device, nir_shader *nir, bool meta_shader)
149 if (!(device->instance->debug_flags & RADV_DEBUG_DUMP_SHADERS))
153 !(device->instance->debug_flags & RADV_DEBUG_DUMP_META_SHADERS))
160 radv_can_dump_shader_stats(struct radv_device *device, nir_shader *nir)
163 return device->instance->debug_flags & RADV_DEBUG_DUMP_SHADER_STATS && !is_meta_shader(nir);
280 struct radv_device *device;
289 struct radv_instance *instance = debug_data->device->instance;
307 struct radv_instance *instance = debug_data->device->instance;
457 radv_force_primitive_shading_rate(nir_shader *nir, struct radv_device *device)
671 radv_shader_spirv_to_nir(struct radv_device *device, const struct radv_pipeline_stage *stage,
699 bool dump_meta = device->instance->debug_flags & RADV_DEBUG_DUMP_META_SHADERS;
700 if ((device->instance->debug_flags & RADV_DEBUG_DUMP_SPIRV) &&
701 (!device->app_shaders_internal || dump_meta))
708 .device = device,
729 .float16 = device->physical_device->rad_info.has_packed_math_16bit,
774 .fragment_shading_rate = device->physical_device->rad_info.gfx_level >= GFX10_3,
792 &device->physical_device->nir_options[stage->stage]);
793 nir->info.internal |= device->app_shaders_internal;
879 if (device->physical_device->rad_info.gfx_level == GFX6) {
925 NIR_PASS(_, nir, radv_nir_lower_ray_queries, device);
960 bool gfx7minus = device->physical_device->rad_info.gfx_level <= GFX7;
977 NIR_PASS(_, nir, nir_opt_shrink_stores, !device->instance->disable_shrink_image_store);
1048 device->physical_device->rad_info.gfx_level);
1056 if (ac_nir_lower_indirect_derefs(nir, device->physical_device->rad_info.gfx_level) &&
1139 radv_lower_io(struct radv_device *device, nir_shader *nir, bool is_mesh_shading)
1159 radv_lower_io_to_mem(struct radv_device *device, struct radv_pipeline_stage *stage,
1172 device->physical_device->rad_info.gfx_level,
1179 device->physical_device->rad_info.gfx_level,
1191 device->physical_device->rad_info.gfx_level,
1198 device->physical_device->rad_info.gfx_level, false);
1203 device->physical_device->task_info.num_entries);
1207 device->physical_device->task_info.num_entries);
1271 void radv_lower_ngg(struct radv_device *device, struct radv_pipeline_stage *ngg_stage,
1336 device->physical_device->rad_info.family,
1369 remove_hole(struct radv_device *device, union radv_shader_arena_block *hole)
1373 if (list_is_empty(&device->shader_free_lists[size_class]))
1374 device->shader_free_list_mask &= ~(1u << size_class);
1378 add_hole(struct radv_device *device, union radv_shader_arena_block *hole)
1381 list_addtail(&hole->freelist, &device->shader_free_lists[size_class]);
1382 device->shader_free_list_mask |= 1u << size_class;
1386 alloc_block_obj(struct radv_device *device)
1388 if (!list_is_empty(&device->shader_block_obj_pool)) {
1390 list_first_entry(&device->shader_block_obj_pool, union radv_shader_arena_block, pool);
1399 free_block_obj(struct radv_device *device, union radv_shader_arena_block *block)
1401 list_add(&block->pool, &device->shader_block_obj_pool);
1412 * Typically, shaders are allocated and only free'd when the device is destroyed. For this pattern,
1417 radv_alloc_shader_memory(struct radv_device *device, uint32_t size, void *ptr)
1421 mtx_lock(&device->shader_arena_mutex);
1428 ffs(device->shader_free_list_mask & (free_list_mask << get_size_class(size, true)));
1433 &device->shader_free_lists[size_class], freelist)
1441 remove_hole(device, hole);
1443 mtx_unlock(&device->shader_arena_mutex);
1446 union radv_shader_arena_block *alloc = alloc_block_obj(device);
1448 mtx_unlock(&device->shader_arena_mutex);
1458 remove_hole(device, hole);
1461 add_hole(device, hole);
1463 mtx_unlock(&device->shader_arena_mutex);
1477 << MIN2(RADV_SHADER_ALLOC_MAX_ARENA_SIZE_SHIFT, device->shader_arena_shift),
1479 VkResult result = device->ws->buffer_create(
1480 device->ws, arena_size, RADV_SHADER_ALLOC_ALIGNMENT, RADEON_DOMAIN_VRAM,
1482 (device->physical_device->rad_info.cpdma_prefetch_writes_memory ? 0
1490 arena->ptr = (char *)device->ws->buffer_map(arena->bo);
1494 alloc = alloc_block_obj(device);
1495 hole = arena_size - size > 0 ? alloc_block_obj(device) : alloc;
1511 add_hole(device, hole);
1514 ++device->shader_arena_shift;
1515 list_addtail(&arena->list, &device->shader_arenas);
1517 mtx_unlock(&device->shader_arena_mutex);
1521 mtx_unlock(&device->shader_arena_mutex);
1525 device->ws->buffer_destroy(device->ws, arena->bo);
1541 radv_free_shader_memory(struct radv_device *device, union radv_shader_arena_block *alloc)
1543 mtx_lock(&device->shader_arena_mutex);
1552 remove_hole(device, hole_prev);
1556 free_block_obj(device, hole);
1563 remove_hole(device, hole_next);
1568 free_block_obj(device, hole);
1575 free_block_obj(device, hole);
1577 device->ws->buffer_destroy(device->ws, arena->bo);
1581 add_hole(device, hole);
1584 mtx_unlock(&device->shader_arena_mutex);
1588 radv_init_shader_arenas(struct radv_device *device)
1590 mtx_init(&device->shader_arena_mutex, mtx_plain);
1592 device->shader_free_list_mask = 0;
1594 list_inithead(&device->shader_arenas);
1595 list_inithead(&device->shader_block_obj_pool);
1597 list_inithead(&device->shader_free_lists[i]);
1601 radv_destroy_shader_arenas(struct radv_device *device)
1603 list_for_each_entry_safe(union radv_shader_arena_block, block, &device->shader_block_obj_pool,
1606 list_for_each_entry_safe(struct radv_shader_arena, arena, &device->shader_arenas, list)
1608 device->ws->buffer_destroy(device->ws, arena->bo);
1611 mtx_destroy(&device->shader_arena_mutex);
1625 radv_should_use_wgp_mode(const struct radv_device *device, gl_shader_stage stage,
1628 enum amd_gfx_level chip = device->physical_device->rad_info.gfx_level;
1644 radv_postprocess_config(const struct radv_device *device, const struct ac_shader_config *config_in,
1649 const struct radv_physical_device *pdevice = device->physical_device;
1651 bool trap_enabled = !!device->trap_handler_shader;
1701 bool wgp_mode = radv_should_use_wgp_mode(device, stage, info);
1914 radv_open_rtld_binary(struct radv_device *device, const struct radv_shader *shader,
1922 if (device->physical_device->rad_info.gfx_level >= GFX9 &&
1944 .info = &device->physical_device->rad_info,
1959 radv_shader_binary_upload(struct radv_device *device, const struct radv_shader_binary *binary,
1968 if (!radv_open_rtld_binary(device, shader, binary, &rtld_binary)) {
1980 radv_shader_destroy(device, shader);
2004 radv_shader_create(struct radv_device *device, const struct radv_shader_binary *binary,
2021 if (!radv_open_rtld_binary(device, shader, binary, &rtld_binary)) {
2026 if (!ac_rtld_read_config(&device->physical_device->rad_info, &rtld_binary, &config)) {
2033 unsigned encode_granularity = device->physical_device->rad_info.lds_encode_granularity;
2062 radv_postprocess_config(device, &config, &binary->info, binary->stage, args, &shader->config);
2073 if (!radv_open_rtld_binary(device, shader, binary, &rtld_binary)) {
2078 if (keep_shader_info || (device->instance->debug_flags & RADV_DEBUG_DUMP_SHADERS)) {
2083 radv_shader_destroy(device, shader);
2196 shader_compile(struct radv_device *device, struct nir_shader *const *shaders, int shader_count, gl_shader_stage stage,
2202 enum radeon_family chip_family = device->physical_device->rad_info.family;
2206 .device = device,
2211 options->gfx_level = device->physical_device->rad_info.gfx_level;
2212 options->has_3d_cube_border_color_mipmap = device->physical_device->rad_info.has_3d_cube_border_color_mipmap;
2213 options->dump_shader = radv_can_dump_shader(device, shaders[0], gs_copy_shader || trap_handler_shader);
2215 options->dump_shader && device->instance->debug_flags & RADV_DEBUG_PREOPTIR;
2218 options->check_ir = device->instance->debug_flags & RADV_DEBUG_CHECKIR;
2219 options->address32_hi = device->physical_device->rad_info.address32_hi;
2220 options->has_ls_vgpr_init_bug = device->physical_device->rad_info.has_ls_vgpr_init_bug;
2227 if (radv_use_llvm_for_stage(device, stage) || options->dump_shader || options->record_ir)
2230 if (radv_use_llvm_for_stage(device, stage)) {
2245 struct radv_shader *shader = radv_shader_create(device, binary, keep_shader_info, false, args);
2271 radv_shader_nir_to_asm(struct radv_device *device, struct radv_pipeline_stage *pl_stage,
2282 options.robust_buffer_access = device->robust_buffer_access;
2283 options.wgp_mode = radv_should_use_wgp_mode(device, stage, &pl_stage->info);
2285 return shader_compile(device, shaders, shader_count, stage, &pl_stage->info,
2291 radv_create_gs_copy_shader(struct radv_device *device, struct nir_shader *shader,
2301 return shader_compile(device, &shader, 1, stage, info, args, &options, true, false,
2306 radv_create_trap_handler_shader(struct radv_device *device)
2319 nir_builder b = radv_meta_init_shader(device, MESA_SHADER_COMPUTE, "meta_trap_handler");
2321 options.wgp_mode = radv_should_use_wgp_mode(device, MESA_SHADER_COMPUTE, &info);
2327 radv_declare_shader_args(device->physical_device->rad_info.gfx_level, &key, &info,
2330 shader = shader_compile(device, &b.shader, 1, MESA_SHADER_COMPUTE, &info, &args, &options,
2333 trap->alloc = radv_alloc_shader_memory(device, shader->code_size, NULL);
2354 radv_trap_handler_shader_destroy(struct radv_device *device, struct radv_trap_handler_shader *trap)
2359 radv_free_shader_memory(device, trap->alloc);
2364 upload_shader_part(struct radv_device *device, struct radv_shader_part_binary *bin, unsigned wave_size)
2371 shader_part->alloc = radv_alloc_shader_memory(device, code_size, NULL);
2425 radv_create_vs_prolog(struct radv_device *device, const struct radv_vs_prolog_key *key)
2429 options.family = device->physical_device->rad_info.family;
2430 options.gfx_level = device->physical_device->rad_info.gfx_level;
2431 options.has_3d_cube_border_color_mipmap = device->physical_device->rad_info.has_3d_cube_border_color_mipmap;
2432 options.address32_hi = device->physical_device->rad_info.address32_hi;
2433 options.dump_shader = device->instance->debug_flags & RADV_DEBUG_DUMP_PROLOGS;
2434 options.record_ir = device->instance->debug_flags & RADV_DEBUG_HANG;
2470 struct radv_shader_part *prolog = upload_shader_part(device, binary, info.wave_size);
2488 radv_create_ps_epilog(struct radv_device *device, const struct radv_ps_epilog_key *key)
2492 options.family = device->physical_device->rad_info.family;
2493 options.gfx_level = device->physical_device->rad_info.gfx_level;
2494 options.address32_hi = device->physical_device->rad_info.address32_hi;
2495 options.dump_shader = device->instance->debug_flags & RADV_DEBUG_DUMP_EPILOGS;
2496 options.record_ir = device->instance->debug_flags & RADV_DEBUG_HANG;
2497 options.dump_preoptir = device->instance->debug_flags & RADV_DEBUG_DUMP_EPILOGS;
2498 options.dump_shader = device->instance->debug_flags & RADV_DEBUG_DUMP_EPILOGS;
2504 radv_declare_ps_epilog_args(device->physical_device->rad_info.gfx_level, key, &args);
2520 struct radv_shader_part *epilog = upload_shader_part(device, binary, info.wave_size);
2537 radv_shader_destroy(struct radv_device *device, struct radv_shader *shader)
2551 radv_shader_part_destroy(struct radv_device *device, struct radv_shader_part *shader_part)
2556 radv_free_shader_memory(device, shader_part->alloc);
2568 radv_find_shader(struct radv_device *device, uint64_t pc)
2570 mtx_lock(&device->shader_arena_mutex);
2571 list_for_each_entry(struct radv_shader_arena, arena, &device->shader_arenas, list)
2584 mtx_unlock(&device->shader_arena_mutex);
2600 mtx_unlock(&device->shader_arena_mutex);
2642 radv_get_max_waves(const struct radv_device *device, struct radv_shader *shader,
2645 struct radeon_info *info = &device->physical_device->rad_info;
2740 radv_dump_shader_stats(struct radv_device *device, struct radv_pipeline *pipeline,
2752 result = radv_GetPipelineExecutablePropertiesKHR(radv_device_to_handle(device), &pipeline_info,
2761 result = radv_GetPipelineExecutablePropertiesKHR(radv_device_to_handle(device), &pipeline_info,
2777 result = radv_GetPipelineExecutableStatisticsKHR(radv_device_to_handle(device), &exec_info,
2788 result = radv_GetPipelineExecutableStatisticsKHR(radv_device_to_handle(device), &exec_info,