1/* 2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org> 3 * Copyright 2018 Advanced Micro Devices, Inc. 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * on the rights to use, copy, modify, merge, publish, distribute, sub 10 * license, and/or sell copies of the Software, and to permit persons to whom 11 * the Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, 21 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 22 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 23 * USE OR OTHER DEALINGS IN THE SOFTWARE. 24 */ 25 26#include "si_pipe.h" 27 28#include "driver_ddebug/dd_util.h" 29#include "radeon_uvd.h" 30#include "si_compute.h" 31#include "si_public.h" 32#include "si_shader_internal.h" 33#include "sid.h" 34#include "ac_shadowed_regs.h" 35#include "compiler/nir/nir.h" 36#include "util/disk_cache.h" 37#include "util/u_cpu_detect.h" 38#include "util/u_log.h" 39#include "util/u_memory.h" 40#include "util/u_suballoc.h" 41#include "util/u_tests.h" 42#include "util/u_upload_mgr.h" 43#include "util/xmlconfig.h" 44#include "vl/vl_decoder.h" 45 46#include <xf86drm.h> 47 48static struct pipe_context *si_create_context(struct pipe_screen *screen, unsigned flags); 49 50static const struct debug_named_value radeonsi_debug_options[] = { 51 /* Shader logging options: */ 52 {"vs", DBG(VS), "Print vertex shaders"}, 53 {"ps", DBG(PS), "Print pixel shaders"}, 54 {"gs", DBG(GS), "Print geometry shaders"}, 55 {"tcs", DBG(TCS), "Print tessellation control shaders"}, 56 {"tes", DBG(TES), "Print tessellation evaluation shaders"}, 57 {"cs", DBG(CS), "Print compute shaders"}, 58 {"noir", DBG(NO_IR), "Don't print the LLVM IR"}, 59 {"nonir", DBG(NO_NIR), "Don't print NIR when printing shaders"}, 60 {"noasm", DBG(NO_ASM), "Don't print disassembled shaders"}, 61 {"preoptir", DBG(PREOPT_IR), "Print the LLVM IR before initial optimizations"}, 62 63 /* Shader compiler options the shader cache should be aware of: */ 64 {"w32ge", DBG(W32_GE), "Use Wave32 for vertex, tessellation, and geometry shaders."}, 65 {"w32ps", DBG(W32_PS), "Use Wave32 for pixel shaders."}, 66 {"w32psdiscard", DBG(W32_PS_DISCARD), "Use Wave32 for pixel shaders even if they contain discard and LLVM is buggy."}, 67 {"w32cs", DBG(W32_CS), "Use Wave32 for computes shaders."}, 68 {"w64ge", DBG(W64_GE), "Use Wave64 for vertex, tessellation, and geometry shaders."}, 69 {"w64ps", DBG(W64_PS), "Use Wave64 for pixel shaders."}, 70 {"w64cs", DBG(W64_CS), "Use Wave64 for computes shaders."}, 71 72 /* Shader compiler options (with no effect on the shader cache): */ 73 {"checkir", DBG(CHECK_IR), "Enable additional sanity checks on shader IR"}, 74 {"mono", DBG(MONOLITHIC_SHADERS), "Use old-style monolithic shaders compiled on demand"}, 75 {"nooptvariant", DBG(NO_OPT_VARIANT), "Disable compiling optimized shader variants."}, 76 77 /* Information logging options: */ 78 {"info", DBG(INFO), "Print driver information"}, 79 {"tex", DBG(TEX), "Print texture info"}, 80 {"compute", DBG(COMPUTE), "Print compute info"}, 81 {"vm", DBG(VM), "Print virtual addresses when creating resources"}, 82 {"cache_stats", DBG(CACHE_STATS), "Print shader cache statistics."}, 83 {"ib", DBG(IB), "Print command buffers."}, 84 85 /* Driver options: */ 86 {"nowc", DBG(NO_WC), "Disable GTT write combining"}, 87 {"check_vm", DBG(CHECK_VM), "Check VM faults and dump debug info."}, 88 {"reserve_vmid", DBG(RESERVE_VMID), "Force VMID reservation per context."}, 89 {"shadowregs", DBG(SHADOW_REGS), "Enable CP register shadowing."}, 90 {"nofastdlist", DBG(NO_FAST_DISPLAY_LIST), "Disable fast display lists"}, 91 92 /* Multimedia options: */ 93 { "noefc", DBG(NO_EFC), "Disable hardware based encoder colour format conversion."}, 94 95 /* 3D engine options: */ 96 {"nogfx", DBG(NO_GFX), "Disable graphics. Only multimedia compute paths can be used."}, 97 {"nongg", DBG(NO_NGG), "Disable NGG and use the legacy pipeline."}, 98 {"nggc", DBG(ALWAYS_NGG_CULLING_ALL), "Always use NGG culling even when it can hurt."}, 99 {"nonggc", DBG(NO_NGG_CULLING), "Disable NGG culling."}, 100 {"switch_on_eop", DBG(SWITCH_ON_EOP), "Program WD/IA to switch on end-of-packet."}, 101 {"nooutoforder", DBG(NO_OUT_OF_ORDER), "Disable out-of-order rasterization"}, 102 {"nodpbb", DBG(NO_DPBB), "Disable DPBB."}, 103 {"dpbb", DBG(DPBB), "Enable DPBB."}, 104 {"nohyperz", DBG(NO_HYPERZ), "Disable Hyper-Z"}, 105 {"no2d", DBG(NO_2D_TILING), "Disable 2D tiling"}, 106 {"notiling", DBG(NO_TILING), "Disable tiling"}, 107 {"nodisplaytiling", DBG(NO_DISPLAY_TILING), "Disable display tiling"}, 108 {"nodisplaydcc", DBG(NO_DISPLAY_DCC), "Disable display DCC"}, 109 {"noexporteddcc", DBG(NO_EXPORTED_DCC), "Disable DCC for all exported buffers (via DMABUF, etc.)"}, 110 {"nodcc", DBG(NO_DCC), "Disable DCC."}, 111 {"nodccclear", DBG(NO_DCC_CLEAR), "Disable DCC fast clear."}, 112 {"nodccstore", DBG(NO_DCC_STORE), "Disable DCC stores"}, 113 {"dccstore", DBG(DCC_STORE), "Enable DCC stores"}, 114 {"nodccmsaa", DBG(NO_DCC_MSAA), "Disable DCC for MSAA"}, 115 {"nofmask", DBG(NO_FMASK), "Disable MSAA compression"}, 116 {"nodma", DBG(NO_DMA), "Disable SDMA-copy for DRI_PRIME"}, 117 118 {"tmz", DBG(TMZ), "Force allocation of scanout/depth/stencil buffer as encrypted"}, 119 {"sqtt", DBG(SQTT), "Enable SQTT"}, 120 121 DEBUG_NAMED_VALUE_END /* must be last */ 122}; 123 124static const struct debug_named_value test_options[] = { 125 /* Tests: */ 126 {"imagecopy", DBG(TEST_IMAGE_COPY), "Invoke resource_copy_region tests with images and exit."}, 127 {"cbresolve", DBG(TEST_CB_RESOLVE), "Invoke MSAA resolve tests and exit."}, 128 {"computeblit", DBG(TEST_COMPUTE_BLIT), "Invoke blits tests and exit."}, 129 {"testvmfaultcp", DBG(TEST_VMFAULT_CP), "Invoke a CP VM fault test and exit."}, 130 {"testvmfaultshader", DBG(TEST_VMFAULT_SHADER), "Invoke a shader VM fault test and exit."}, 131 {"testdmaperf", DBG(TEST_DMA_PERF), "Test DMA performance"}, 132 {"testgds", DBG(TEST_GDS), "Test GDS."}, 133 {"testgdsmm", DBG(TEST_GDS_MM), "Test GDS memory management."}, 134 {"testgdsoamm", DBG(TEST_GDS_OA_MM), "Test GDS OA memory management."}, 135 136 DEBUG_NAMED_VALUE_END /* must be last */ 137}; 138 139bool si_init_compiler(struct si_screen *sscreen, struct ac_llvm_compiler *compiler) 140{ 141 /* Only create the less-optimizing version of the compiler on APUs 142 * predating Ryzen (Raven). */ 143 bool create_low_opt_compiler = 144 !sscreen->info.has_dedicated_vram && sscreen->info.gfx_level <= GFX8; 145 146 enum ac_target_machine_options tm_options = 147 (sscreen->debug_flags & DBG(CHECK_IR) ? AC_TM_CHECK_IR : 0) | 148 (create_low_opt_compiler ? AC_TM_CREATE_LOW_OPT : 0); 149 150 ac_init_llvm_once(); 151 152 if (!ac_init_llvm_compiler(compiler, sscreen->info.family, tm_options)) 153 return false; 154 155 compiler->passes = ac_create_llvm_passes(compiler->tm); 156 if (compiler->low_opt_tm) 157 compiler->low_opt_passes = ac_create_llvm_passes(compiler->low_opt_tm); 158 159 return true; 160} 161 162void si_init_aux_async_compute_ctx(struct si_screen *sscreen) 163{ 164 assert(!sscreen->async_compute_context); 165 sscreen->async_compute_context = si_create_context( 166 &sscreen->b, 167 SI_CONTEXT_FLAG_AUX | 168 (sscreen->options.aux_debug ? PIPE_CONTEXT_DEBUG : 0) | 169 PIPE_CONTEXT_COMPUTE_ONLY); 170 171 /* Limit the numbers of waves allocated for this context. */ 172 if (sscreen->async_compute_context) 173 ((struct si_context*)sscreen->async_compute_context)->cs_max_waves_per_sh = 2; 174} 175 176static void si_destroy_compiler(struct ac_llvm_compiler *compiler) 177{ 178 ac_destroy_llvm_compiler(compiler); 179} 180 181 182static void decref_implicit_resource(struct hash_entry *entry) 183{ 184 pipe_resource_reference((struct pipe_resource**)&entry->data, NULL); 185} 186 187/* 188 * pipe_context 189 */ 190static void si_destroy_context(struct pipe_context *context) 191{ 192 struct si_context *sctx = (struct si_context *)context; 193 int i; 194 195 /* Unreference the framebuffer normally to disable related logic 196 * properly. 197 */ 198 struct pipe_framebuffer_state fb = {}; 199 if (context->set_framebuffer_state) 200 context->set_framebuffer_state(context, &fb); 201 202 si_release_all_descriptors(sctx); 203 204 if (sctx->gfx_level >= GFX10 && sctx->has_graphics) 205 gfx10_destroy_query(sctx); 206 207 if (sctx->thread_trace) 208 si_destroy_thread_trace(sctx); 209 210 pipe_resource_reference(&sctx->esgs_ring, NULL); 211 pipe_resource_reference(&sctx->gsvs_ring, NULL); 212 pipe_resource_reference(&sctx->tess_rings, NULL); 213 pipe_resource_reference(&sctx->tess_rings_tmz, NULL); 214 pipe_resource_reference(&sctx->null_const_buf.buffer, NULL); 215 pipe_resource_reference(&sctx->sample_pos_buffer, NULL); 216 si_resource_reference(&sctx->border_color_buffer, NULL); 217 free(sctx->border_color_table); 218 si_resource_reference(&sctx->scratch_buffer, NULL); 219 si_resource_reference(&sctx->compute_scratch_buffer, NULL); 220 si_resource_reference(&sctx->wait_mem_scratch, NULL); 221 si_resource_reference(&sctx->wait_mem_scratch_tmz, NULL); 222 si_resource_reference(&sctx->small_prim_cull_info_buf, NULL); 223 si_resource_reference(&sctx->pipeline_stats_query_buf, NULL); 224 225 if (sctx->cs_preamble_state) 226 si_pm4_free_state(sctx, sctx->cs_preamble_state, ~0); 227 if (sctx->cs_preamble_state_tmz) 228 si_pm4_free_state(sctx, sctx->cs_preamble_state_tmz, ~0); 229 230 for (i = 0; i < ARRAY_SIZE(sctx->vgt_shader_config); i++) 231 si_pm4_free_state(sctx, sctx->vgt_shader_config[i], SI_STATE_IDX(vgt_shader_config)); 232 233 if (sctx->fixed_func_tcs_shader_cache) { 234 hash_table_foreach(sctx->fixed_func_tcs_shader_cache, entry) { 235 sctx->b.delete_tcs_state(&sctx->b, entry->data); 236 } 237 _mesa_hash_table_destroy(sctx->fixed_func_tcs_shader_cache, NULL); 238 } 239 240 if (sctx->custom_dsa_flush) 241 sctx->b.delete_depth_stencil_alpha_state(&sctx->b, sctx->custom_dsa_flush); 242 if (sctx->custom_blend_resolve) 243 sctx->b.delete_blend_state(&sctx->b, sctx->custom_blend_resolve); 244 if (sctx->custom_blend_fmask_decompress) 245 sctx->b.delete_blend_state(&sctx->b, sctx->custom_blend_fmask_decompress); 246 if (sctx->custom_blend_eliminate_fastclear) 247 sctx->b.delete_blend_state(&sctx->b, sctx->custom_blend_eliminate_fastclear); 248 if (sctx->custom_blend_dcc_decompress) 249 sctx->b.delete_blend_state(&sctx->b, sctx->custom_blend_dcc_decompress); 250 if (sctx->vs_blit_pos) 251 sctx->b.delete_vs_state(&sctx->b, sctx->vs_blit_pos); 252 if (sctx->vs_blit_pos_layered) 253 sctx->b.delete_vs_state(&sctx->b, sctx->vs_blit_pos_layered); 254 if (sctx->vs_blit_color) 255 sctx->b.delete_vs_state(&sctx->b, sctx->vs_blit_color); 256 if (sctx->vs_blit_color_layered) 257 sctx->b.delete_vs_state(&sctx->b, sctx->vs_blit_color_layered); 258 if (sctx->vs_blit_texcoord) 259 sctx->b.delete_vs_state(&sctx->b, sctx->vs_blit_texcoord); 260 if (sctx->cs_clear_buffer) 261 sctx->b.delete_compute_state(&sctx->b, sctx->cs_clear_buffer); 262 if (sctx->cs_clear_buffer_rmw) 263 sctx->b.delete_compute_state(&sctx->b, sctx->cs_clear_buffer_rmw); 264 if (sctx->cs_copy_buffer) 265 sctx->b.delete_compute_state(&sctx->b, sctx->cs_copy_buffer); 266 for (unsigned i = 0; i < ARRAY_SIZE(sctx->cs_copy_image); i++) { 267 for (unsigned j = 0; j < ARRAY_SIZE(sctx->cs_copy_image[i]); j++) { 268 if (sctx->cs_copy_image[i][j]) 269 sctx->b.delete_compute_state(&sctx->b, sctx->cs_copy_image[i][j]); 270 } 271 } 272 if (sctx->cs_clear_render_target) 273 sctx->b.delete_compute_state(&sctx->b, sctx->cs_clear_render_target); 274 if (sctx->cs_clear_render_target_1d_array) 275 sctx->b.delete_compute_state(&sctx->b, sctx->cs_clear_render_target_1d_array); 276 if (sctx->cs_clear_12bytes_buffer) 277 sctx->b.delete_compute_state(&sctx->b, sctx->cs_clear_12bytes_buffer); 278 for (unsigned i = 0; i < ARRAY_SIZE(sctx->cs_dcc_retile); i++) { 279 if (sctx->cs_dcc_retile[i]) 280 sctx->b.delete_compute_state(&sctx->b, sctx->cs_dcc_retile[i]); 281 } 282 if (sctx->no_velems_state) 283 sctx->b.delete_vertex_elements_state(&sctx->b, sctx->no_velems_state); 284 285 for (unsigned i = 0; i < ARRAY_SIZE(sctx->cs_fmask_expand); i++) { 286 for (unsigned j = 0; j < ARRAY_SIZE(sctx->cs_fmask_expand[i]); j++) { 287 if (sctx->cs_fmask_expand[i][j]) { 288 sctx->b.delete_compute_state(&sctx->b, sctx->cs_fmask_expand[i][j]); 289 } 290 } 291 } 292 293 for (unsigned i = 0; i < ARRAY_SIZE(sctx->cs_clear_dcc_msaa); i++) { 294 for (unsigned j = 0; j < ARRAY_SIZE(sctx->cs_clear_dcc_msaa[i]); j++) { 295 for (unsigned k = 0; k < ARRAY_SIZE(sctx->cs_clear_dcc_msaa[i][j]); k++) { 296 for (unsigned l = 0; l < ARRAY_SIZE(sctx->cs_clear_dcc_msaa[i][j][k]); l++) { 297 for (unsigned m = 0; m < ARRAY_SIZE(sctx->cs_clear_dcc_msaa[i][j][k][l]); m++) { 298 if (sctx->cs_clear_dcc_msaa[i][j][k][l][m]) 299 sctx->b.delete_compute_state(&sctx->b, sctx->cs_clear_dcc_msaa[i][j][k][l][m]); 300 } 301 } 302 } 303 } 304 } 305 306 if (sctx->blitter) 307 util_blitter_destroy(sctx->blitter); 308 309 if (sctx->query_result_shader) 310 sctx->b.delete_compute_state(&sctx->b, sctx->query_result_shader); 311 if (sctx->sh_query_result_shader) 312 sctx->b.delete_compute_state(&sctx->b, sctx->sh_query_result_shader); 313 314 sctx->ws->cs_destroy(&sctx->gfx_cs); 315 if (sctx->ctx) 316 sctx->ws->ctx_destroy(sctx->ctx); 317 if (sctx->sdma_cs) { 318 sctx->ws->cs_destroy(sctx->sdma_cs); 319 free(sctx->sdma_cs); 320 } 321 322 if (sctx->dirty_implicit_resources) 323 _mesa_hash_table_destroy(sctx->dirty_implicit_resources, 324 decref_implicit_resource); 325 326 if (sctx->b.stream_uploader) 327 u_upload_destroy(sctx->b.stream_uploader); 328 if (sctx->b.const_uploader && sctx->b.const_uploader != sctx->b.stream_uploader) 329 u_upload_destroy(sctx->b.const_uploader); 330 if (sctx->cached_gtt_allocator) 331 u_upload_destroy(sctx->cached_gtt_allocator); 332 333 slab_destroy_child(&sctx->pool_transfers); 334 slab_destroy_child(&sctx->pool_transfers_unsync); 335 336 u_suballocator_destroy(&sctx->allocator_zeroed_memory); 337 338 sctx->ws->fence_reference(&sctx->last_gfx_fence, NULL); 339 si_resource_reference(&sctx->eop_bug_scratch, NULL); 340 si_resource_reference(&sctx->eop_bug_scratch_tmz, NULL); 341 si_resource_reference(&sctx->shadowed_regs, NULL); 342 343 si_destroy_compiler(&sctx->compiler); 344 345 si_saved_cs_reference(&sctx->current_saved_cs, NULL); 346 347 _mesa_hash_table_destroy(sctx->tex_handles, NULL); 348 _mesa_hash_table_destroy(sctx->img_handles, NULL); 349 350 util_dynarray_fini(&sctx->resident_tex_handles); 351 util_dynarray_fini(&sctx->resident_img_handles); 352 util_dynarray_fini(&sctx->resident_tex_needs_color_decompress); 353 util_dynarray_fini(&sctx->resident_img_needs_color_decompress); 354 util_dynarray_fini(&sctx->resident_tex_needs_depth_decompress); 355 356 if (!(sctx->context_flags & SI_CONTEXT_FLAG_AUX)) 357 p_atomic_dec(&context->screen->num_contexts); 358 359 FREE(sctx); 360} 361 362static enum pipe_reset_status si_get_reset_status(struct pipe_context *ctx) 363{ 364 struct si_context *sctx = (struct si_context *)ctx; 365 if (sctx->context_flags & SI_CONTEXT_FLAG_AUX) 366 return PIPE_NO_RESET; 367 368 bool needs_reset; 369 enum pipe_reset_status status = sctx->ws->ctx_query_reset_status(sctx->ctx, false, &needs_reset); 370 371 if (status != PIPE_NO_RESET && needs_reset && !(sctx->context_flags & SI_CONTEXT_FLAG_AUX)) { 372 /* Call the gallium frontend to set a no-op API dispatch. */ 373 if (sctx->device_reset_callback.reset) { 374 sctx->device_reset_callback.reset(sctx->device_reset_callback.data, status); 375 } 376 } 377 return status; 378} 379 380static void si_set_device_reset_callback(struct pipe_context *ctx, 381 const struct pipe_device_reset_callback *cb) 382{ 383 struct si_context *sctx = (struct si_context *)ctx; 384 385 if (cb) 386 sctx->device_reset_callback = *cb; 387 else 388 memset(&sctx->device_reset_callback, 0, sizeof(sctx->device_reset_callback)); 389} 390 391/* Apitrace profiling: 392 * 1) qapitrace : Tools -> Profile: Measure CPU & GPU times 393 * 2) In the middle panel, zoom in (mouse wheel) on some bad draw call 394 * and remember its number. 395 * 3) In Mesa, enable queries and performance counters around that draw 396 * call and print the results. 397 * 4) glretrace --benchmark --markers .. 398 */ 399static void si_emit_string_marker(struct pipe_context *ctx, const char *string, int len) 400{ 401 struct si_context *sctx = (struct si_context *)ctx; 402 403 dd_parse_apitrace_marker(string, len, &sctx->apitrace_call_number); 404 405 if (sctx->thread_trace_enabled) 406 si_write_user_event(sctx, &sctx->gfx_cs, UserEventTrigger, string, len); 407 408 if (sctx->log) 409 u_log_printf(sctx->log, "\nString marker: %*s\n", len, string); 410} 411 412static void si_set_debug_callback(struct pipe_context *ctx, const struct util_debug_callback *cb) 413{ 414 struct si_context *sctx = (struct si_context *)ctx; 415 struct si_screen *screen = sctx->screen; 416 417 util_queue_finish(&screen->shader_compiler_queue); 418 util_queue_finish(&screen->shader_compiler_queue_low_priority); 419 420 if (cb) 421 sctx->debug = *cb; 422 else 423 memset(&sctx->debug, 0, sizeof(sctx->debug)); 424} 425 426static void si_set_log_context(struct pipe_context *ctx, struct u_log_context *log) 427{ 428 struct si_context *sctx = (struct si_context *)ctx; 429 sctx->log = log; 430 431 if (log) 432 u_log_add_auto_logger(log, si_auto_log_cs, sctx); 433} 434 435static void si_set_context_param(struct pipe_context *ctx, enum pipe_context_param param, 436 unsigned value) 437{ 438 struct radeon_winsys *ws = ((struct si_context *)ctx)->ws; 439 440 switch (param) { 441 case PIPE_CONTEXT_PARAM_PIN_THREADS_TO_L3_CACHE: 442 ws->pin_threads_to_L3_cache(ws, value); 443 break; 444 default:; 445 } 446} 447 448static void si_set_frontend_noop(struct pipe_context *ctx, bool enable) 449{ 450 struct si_context *sctx = (struct si_context *)ctx; 451 452 ctx->flush(ctx, NULL, PIPE_FLUSH_ASYNC); 453 sctx->is_noop = enable; 454} 455 456static struct pipe_context *si_create_context(struct pipe_screen *screen, unsigned flags) 457{ 458 struct si_screen *sscreen = (struct si_screen *)screen; 459 STATIC_ASSERT(DBG_COUNT <= 64); 460 461 /* Don't create a context if it's not compute-only and hw is compute-only. */ 462 if (!sscreen->info.has_graphics && !(flags & PIPE_CONTEXT_COMPUTE_ONLY)) { 463 fprintf(stderr, "radeonsi: can't create a graphics context on a compute chip\n"); 464 return NULL; 465 } 466 467 struct si_context *sctx = CALLOC_STRUCT(si_context); 468 struct radeon_winsys *ws = sscreen->ws; 469 int shader, i; 470 bool stop_exec_on_failure = (flags & PIPE_CONTEXT_LOSE_CONTEXT_ON_RESET) != 0; 471 enum radeon_ctx_priority priority; 472 473 if (!sctx) { 474 fprintf(stderr, "radeonsi: can't allocate a context\n"); 475 return NULL; 476 } 477 478 sctx->has_graphics = sscreen->info.gfx_level == GFX6 || !(flags & PIPE_CONTEXT_COMPUTE_ONLY); 479 480 if (flags & PIPE_CONTEXT_DEBUG) 481 sscreen->record_llvm_ir = true; /* racy but not critical */ 482 483 sctx->b.screen = screen; /* this must be set first */ 484 sctx->b.priv = NULL; 485 sctx->b.destroy = si_destroy_context; 486 sctx->screen = sscreen; /* Easy accessing of screen/winsys. */ 487 sctx->is_debug = (flags & PIPE_CONTEXT_DEBUG) != 0; 488 sctx->context_flags = flags; 489 490 slab_create_child(&sctx->pool_transfers, &sscreen->pool_transfers); 491 slab_create_child(&sctx->pool_transfers_unsync, &sscreen->pool_transfers); 492 493 sctx->ws = sscreen->ws; 494 sctx->family = sscreen->info.family; 495 sctx->gfx_level = sscreen->info.gfx_level; 496 497 if (sctx->gfx_level == GFX7 || sctx->gfx_level == GFX8 || sctx->gfx_level == GFX9) { 498 sctx->eop_bug_scratch = si_aligned_buffer_create( 499 &sscreen->b, PIPE_RESOURCE_FLAG_UNMAPPABLE | SI_RESOURCE_FLAG_DRIVER_INTERNAL, 500 PIPE_USAGE_DEFAULT, 16 * sscreen->info.max_render_backends, 256); 501 if (!sctx->eop_bug_scratch) { 502 fprintf(stderr, "radeonsi: can't create eop_bug_scratch\n"); 503 goto fail; 504 } 505 } 506 507 if (flags & PIPE_CONTEXT_HIGH_PRIORITY) { 508 priority = RADEON_CTX_PRIORITY_HIGH; 509 } else if (flags & PIPE_CONTEXT_LOW_PRIORITY) { 510 priority = RADEON_CTX_PRIORITY_LOW; 511 } else { 512 priority = RADEON_CTX_PRIORITY_MEDIUM; 513 } 514 515 /* Initialize the context handle and the command stream. */ 516 sctx->ctx = sctx->ws->ctx_create(sctx->ws, priority); 517 if (!sctx->ctx && priority != RADEON_CTX_PRIORITY_MEDIUM) { 518 /* Context priority should be treated as a hint. If context creation 519 * fails with the requested priority, for example because the caller 520 * lacks CAP_SYS_NICE capability or other system resource constraints, 521 * fallback to normal priority. 522 */ 523 priority = RADEON_CTX_PRIORITY_MEDIUM; 524 sctx->ctx = sctx->ws->ctx_create(sctx->ws, priority); 525 } 526 if (!sctx->ctx) { 527 fprintf(stderr, "radeonsi: can't create radeon_winsys_ctx\n"); 528 goto fail; 529 } 530 531 ws->cs_create(&sctx->gfx_cs, sctx->ctx, sctx->has_graphics ? AMD_IP_GFX : AMD_IP_COMPUTE, 532 (void *)si_flush_gfx_cs, sctx, stop_exec_on_failure); 533 534 /* Initialize private allocators. */ 535 u_suballocator_init(&sctx->allocator_zeroed_memory, &sctx->b, 128 * 1024, 0, 536 PIPE_USAGE_DEFAULT, 537 SI_RESOURCE_FLAG_CLEAR | SI_RESOURCE_FLAG_32BIT, false); 538 539 sctx->cached_gtt_allocator = u_upload_create(&sctx->b, 16 * 1024, 0, PIPE_USAGE_STAGING, 0); 540 if (!sctx->cached_gtt_allocator) { 541 fprintf(stderr, "radeonsi: can't create cached_gtt_allocator\n"); 542 goto fail; 543 } 544 545 /* Initialize public allocators. */ 546 /* Unify uploaders as follows: 547 * - dGPUs with Smart Access Memory: there is only one uploader instance writing to VRAM. 548 * - APUs: There is only one uploader instance writing to RAM. VRAM has the same perf on APUs. 549 * - Other chips: The const uploader writes to VRAM and the stream uploader writes to RAM. 550 */ 551 bool smart_access_memory = sscreen->info.smart_access_memory; 552 bool is_apu = !sscreen->info.has_dedicated_vram; 553 sctx->b.stream_uploader = 554 u_upload_create(&sctx->b, 1024 * 1024, 0, 555 smart_access_memory && !is_apu ? PIPE_USAGE_DEFAULT : PIPE_USAGE_STREAM, 556 SI_RESOURCE_FLAG_32BIT); /* same flags as const_uploader */ 557 if (!sctx->b.stream_uploader) { 558 fprintf(stderr, "radeonsi: can't create stream_uploader\n"); 559 goto fail; 560 } 561 562 if (smart_access_memory || is_apu) { 563 sctx->b.const_uploader = sctx->b.stream_uploader; 564 } else { 565 sctx->b.const_uploader = 566 u_upload_create(&sctx->b, 256 * 1024, 0, PIPE_USAGE_DEFAULT, 567 SI_RESOURCE_FLAG_32BIT); 568 if (!sctx->b.const_uploader) { 569 fprintf(stderr, "radeonsi: can't create const_uploader\n"); 570 goto fail; 571 } 572 } 573 574 /* Border colors. */ 575 if (sscreen->info.has_3d_cube_border_color_mipmap) { 576 sctx->border_color_table = malloc(SI_MAX_BORDER_COLORS * sizeof(*sctx->border_color_table)); 577 if (!sctx->border_color_table) { 578 fprintf(stderr, "radeonsi: can't create border_color_table\n"); 579 goto fail; 580 } 581 582 sctx->border_color_buffer = si_resource(pipe_buffer_create( 583 screen, 0, PIPE_USAGE_DEFAULT, SI_MAX_BORDER_COLORS * sizeof(*sctx->border_color_table))); 584 if (!sctx->border_color_buffer) { 585 fprintf(stderr, "radeonsi: can't create border_color_buffer\n"); 586 goto fail; 587 } 588 589 sctx->border_color_map = 590 ws->buffer_map(ws, sctx->border_color_buffer->buf, NULL, PIPE_MAP_WRITE); 591 if (!sctx->border_color_map) { 592 fprintf(stderr, "radeonsi: can't map border_color_buffer\n"); 593 goto fail; 594 } 595 } 596 597 sctx->ngg = sscreen->use_ngg; 598 si_shader_change_notify(sctx); 599 600 /* Initialize context functions used by graphics and compute. */ 601 if (sctx->gfx_level >= GFX10) 602 sctx->emit_cache_flush = gfx10_emit_cache_flush; 603 else 604 sctx->emit_cache_flush = si_emit_cache_flush; 605 606 sctx->b.emit_string_marker = si_emit_string_marker; 607 sctx->b.set_debug_callback = si_set_debug_callback; 608 sctx->b.set_log_context = si_set_log_context; 609 sctx->b.set_context_param = si_set_context_param; 610 sctx->b.get_device_reset_status = si_get_reset_status; 611 sctx->b.set_device_reset_callback = si_set_device_reset_callback; 612 sctx->b.set_frontend_noop = si_set_frontend_noop; 613 614 si_init_all_descriptors(sctx); 615 si_init_buffer_functions(sctx); 616 si_init_clear_functions(sctx); 617 si_init_blit_functions(sctx); 618 si_init_compute_functions(sctx); 619 si_init_compute_blit_functions(sctx); 620 si_init_debug_functions(sctx); 621 si_init_fence_functions(sctx); 622 si_init_query_functions(sctx); 623 si_init_state_compute_functions(sctx); 624 si_init_context_texture_functions(sctx); 625 626 /* Initialize graphics-only context functions. */ 627 if (sctx->has_graphics) { 628 if (sctx->gfx_level >= GFX10) 629 gfx10_init_query(sctx); 630 si_init_msaa_functions(sctx); 631 si_init_shader_functions(sctx); 632 si_init_state_functions(sctx); 633 si_init_streamout_functions(sctx); 634 si_init_viewport_functions(sctx); 635 si_init_spi_map_functions(sctx); 636 637 sctx->blitter = util_blitter_create(&sctx->b); 638 if (sctx->blitter == NULL) { 639 fprintf(stderr, "radeonsi: can't create blitter\n"); 640 goto fail; 641 } 642 sctx->blitter->skip_viewport_restore = true; 643 644 /* Some states are expected to be always non-NULL. */ 645 sctx->noop_blend = util_blitter_get_noop_blend_state(sctx->blitter); 646 sctx->queued.named.blend = sctx->noop_blend; 647 648 sctx->noop_dsa = util_blitter_get_noop_dsa_state(sctx->blitter); 649 sctx->queued.named.dsa = sctx->noop_dsa; 650 651 sctx->no_velems_state = sctx->b.create_vertex_elements_state(&sctx->b, 0, NULL); 652 sctx->vertex_elements = sctx->no_velems_state; 653 654 sctx->discard_rasterizer_state = util_blitter_get_discard_rasterizer_state(sctx->blitter); 655 sctx->queued.named.rasterizer = sctx->discard_rasterizer_state; 656 657 switch (sctx->gfx_level) { 658 case GFX6: 659 si_init_draw_functions_GFX6(sctx); 660 break; 661 case GFX7: 662 si_init_draw_functions_GFX7(sctx); 663 break; 664 case GFX8: 665 si_init_draw_functions_GFX8(sctx); 666 break; 667 case GFX9: 668 si_init_draw_functions_GFX9(sctx); 669 break; 670 case GFX10: 671 si_init_draw_functions_GFX10(sctx); 672 break; 673 case GFX10_3: 674 si_init_draw_functions_GFX10_3(sctx); 675 break; 676 case GFX11: 677 si_init_draw_functions_GFX11(sctx); 678 break; 679 default: 680 unreachable("unhandled gfx level"); 681 } 682 } 683 684 sctx->sample_mask = 0xffff; 685 686 /* Initialize multimedia functions. */ 687 if (sscreen->info.ip[AMD_IP_UVD].num_queues || sscreen->info.has_video_hw.vcn_decode || 688 sscreen->info.ip[AMD_IP_VCN_JPEG].num_queues || sscreen->info.ip[AMD_IP_VCE].num_queues || 689 sscreen->info.ip[AMD_IP_UVD_ENC].num_queues || sscreen->info.ip[AMD_IP_VCN_ENC].num_queues) { 690 sctx->b.create_video_codec = si_uvd_create_decoder; 691 sctx->b.create_video_buffer = si_video_buffer_create; 692 if (screen->resource_create_with_modifiers) 693 sctx->b.create_video_buffer_with_modifiers = si_video_buffer_create_with_modifiers; 694 } else { 695 sctx->b.create_video_codec = vl_create_decoder; 696 sctx->b.create_video_buffer = vl_video_buffer_create; 697 } 698 699 /* GFX7 cannot unbind a constant buffer (S_BUFFER_LOAD doesn't skip loads 700 * if NUM_RECORDS == 0). We need to use a dummy buffer instead. */ 701 if (sctx->gfx_level == GFX7) { 702 sctx->null_const_buf.buffer = 703 pipe_aligned_buffer_create(screen, 704 PIPE_RESOURCE_FLAG_UNMAPPABLE | SI_RESOURCE_FLAG_32BIT | 705 SI_RESOURCE_FLAG_DRIVER_INTERNAL, 706 PIPE_USAGE_DEFAULT, 16, 707 sctx->screen->info.tcc_cache_line_size); 708 if (!sctx->null_const_buf.buffer) { 709 fprintf(stderr, "radeonsi: can't create null_const_buf\n"); 710 goto fail; 711 } 712 sctx->null_const_buf.buffer_size = sctx->null_const_buf.buffer->width0; 713 714 unsigned start_shader = sctx->has_graphics ? 0 : PIPE_SHADER_COMPUTE; 715 for (shader = start_shader; shader < SI_NUM_SHADERS; shader++) { 716 for (i = 0; i < SI_NUM_CONST_BUFFERS; i++) { 717 sctx->b.set_constant_buffer(&sctx->b, shader, i, false, &sctx->null_const_buf); 718 } 719 } 720 721 si_set_internal_const_buffer(sctx, SI_HS_CONST_DEFAULT_TESS_LEVELS, &sctx->null_const_buf); 722 si_set_internal_const_buffer(sctx, SI_VS_CONST_INSTANCE_DIVISORS, &sctx->null_const_buf); 723 si_set_internal_const_buffer(sctx, SI_VS_CONST_CLIP_PLANES, &sctx->null_const_buf); 724 si_set_internal_const_buffer(sctx, SI_PS_CONST_POLY_STIPPLE, &sctx->null_const_buf); 725 si_set_internal_const_buffer(sctx, SI_PS_CONST_SAMPLE_POSITIONS, &sctx->null_const_buf); 726 } 727 728 /* Bindless handles. */ 729 sctx->tex_handles = _mesa_hash_table_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); 730 sctx->img_handles = _mesa_hash_table_create(NULL, _mesa_hash_pointer, _mesa_key_pointer_equal); 731 732 util_dynarray_init(&sctx->resident_tex_handles, NULL); 733 util_dynarray_init(&sctx->resident_img_handles, NULL); 734 util_dynarray_init(&sctx->resident_tex_needs_color_decompress, NULL); 735 util_dynarray_init(&sctx->resident_img_needs_color_decompress, NULL); 736 util_dynarray_init(&sctx->resident_tex_needs_depth_decompress, NULL); 737 738 sctx->dirty_implicit_resources = _mesa_pointer_hash_table_create(NULL); 739 if (!sctx->dirty_implicit_resources) { 740 fprintf(stderr, "radeonsi: can't create dirty_implicit_resources\n"); 741 goto fail; 742 } 743 744 /* The remainder of this function initializes the gfx CS and must be last. */ 745 assert(sctx->gfx_cs.current.cdw == 0); 746 747 if (sctx->has_graphics) { 748 si_init_cp_reg_shadowing(sctx); 749 } 750 751 /* Set immutable fields of shader keys. */ 752 if (sctx->gfx_level >= GFX9) { 753 /* The LS output / HS input layout can be communicated 754 * directly instead of via user SGPRs for merged LS-HS. 755 * This also enables jumping over the VS prolog for HS-only waves. 756 * 757 * When the LS VGPR fix is needed, monolithic shaders can: 758 * - avoid initializing EXEC in both the LS prolog 759 * and the LS main part when !vs_needs_prolog 760 * - remove the fixup for unused input VGPRs 761 */ 762 sctx->shader.tcs.key.ge.opt.prefer_mono = 1; 763 764 /* This enables jumping over the VS prolog for GS-only waves. */ 765 sctx->shader.gs.key.ge.opt.prefer_mono = 1; 766 } 767 768 si_begin_new_gfx_cs(sctx, true); 769 assert(sctx->gfx_cs.current.cdw == sctx->initial_gfx_cs_size); 770 771 if (sctx->gfx_level >= GFX9 && sctx->gfx_level < GFX11) { 772 sctx->wait_mem_scratch = 773 si_aligned_buffer_create(screen, 774 PIPE_RESOURCE_FLAG_UNMAPPABLE | 775 SI_RESOURCE_FLAG_DRIVER_INTERNAL, 776 PIPE_USAGE_DEFAULT, 4, 777 sscreen->info.tcc_cache_line_size); 778 if (!sctx->wait_mem_scratch) { 779 fprintf(stderr, "radeonsi: can't create wait_mem_scratch\n"); 780 goto fail; 781 } 782 783 si_cp_write_data(sctx, sctx->wait_mem_scratch, 0, 4, V_370_MEM, V_370_ME, 784 &sctx->wait_mem_number); 785 } 786 787 if (sctx->gfx_level == GFX7) { 788 /* Clear the NULL constant buffer, because loads should return zeros. 789 * Note that this forces CP DMA to be used, because clover deadlocks 790 * for some reason when the compute codepath is used. 791 */ 792 uint32_t clear_value = 0; 793 si_clear_buffer(sctx, sctx->null_const_buf.buffer, 0, sctx->null_const_buf.buffer->width0, 794 &clear_value, 4, SI_OP_SYNC_AFTER, SI_COHERENCY_SHADER, 795 SI_CP_DMA_CLEAR_METHOD); 796 } 797 798 if (!(flags & SI_CONTEXT_FLAG_AUX)) { 799 p_atomic_inc(&screen->num_contexts); 800 801 /* Check if the aux_context needs to be recreated */ 802 struct si_context *saux = si_get_aux_context(sscreen); 803 804 enum pipe_reset_status status = sctx->ws->ctx_query_reset_status( 805 saux->ctx, true, NULL); 806 if (status != PIPE_NO_RESET) { 807 /* We lost the aux_context, create a new one */ 808 struct u_log_context *aux_log = (saux)->log; 809 saux->b.set_log_context(&saux->b, NULL); 810 saux->b.destroy(&saux->b); 811 812 saux = (struct si_context *)si_create_context( 813 &sscreen->b, SI_CONTEXT_FLAG_AUX | 814 (sscreen->options.aux_debug ? PIPE_CONTEXT_DEBUG : 0) | 815 (sscreen->info.has_graphics ? 0 : PIPE_CONTEXT_COMPUTE_ONLY)); 816 saux->b.set_log_context(&saux->b, aux_log); 817 sscreen->aux_context = saux; 818 } 819 si_put_aux_context_flush(sscreen); 820 821 simple_mtx_lock(&sscreen->async_compute_context_lock); 822 if (status != PIPE_NO_RESET && sscreen->async_compute_context) { 823 sscreen->async_compute_context->destroy(sscreen->async_compute_context); 824 sscreen->async_compute_context = NULL; 825 } 826 simple_mtx_unlock(&sscreen->async_compute_context_lock); 827 } 828 829 sctx->initial_gfx_cs_size = sctx->gfx_cs.current.cdw; 830 return &sctx->b; 831fail: 832 fprintf(stderr, "radeonsi: Failed to create a context.\n"); 833 si_destroy_context(&sctx->b); 834 return NULL; 835} 836 837static bool si_is_resource_busy(struct pipe_screen *screen, struct pipe_resource *resource, 838 unsigned usage) 839{ 840 struct radeon_winsys *ws = ((struct si_screen *)screen)->ws; 841 842 return !ws->buffer_wait(ws, si_resource(resource)->buf, 0, 843 /* If mapping for write, we need to wait for all reads and writes. 844 * If mapping for read, we only need to wait for writes. 845 */ 846 usage & PIPE_MAP_WRITE ? RADEON_USAGE_READWRITE : RADEON_USAGE_WRITE); 847} 848 849static struct pipe_context *si_pipe_create_context(struct pipe_screen *screen, void *priv, 850 unsigned flags) 851{ 852 struct si_screen *sscreen = (struct si_screen *)screen; 853 struct pipe_context *ctx; 854 855 if (sscreen->debug_flags & DBG(CHECK_VM)) 856 flags |= PIPE_CONTEXT_DEBUG; 857 858 ctx = si_create_context(screen, flags); 859 860 if (ctx && sscreen->info.gfx_level >= GFX9 && sscreen->debug_flags & DBG(SQTT)) { 861 if (ac_check_profile_state(&sscreen->info)) { 862 fprintf(stderr, "radeonsi: Canceling RGP trace request as a hang condition has been " 863 "detected. Force the GPU into a profiling mode with e.g. " 864 "\"echo profile_peak > " 865 "/sys/class/drm/card0/device/power_dpm_force_performance_level\"\n"); 866 } else if (!si_init_thread_trace((struct si_context *)ctx)) { 867 FREE(ctx); 868 return NULL; 869 } 870 } 871 872 if (!(flags & PIPE_CONTEXT_PREFER_THREADED)) 873 return ctx; 874 875 /* Clover (compute-only) is unsupported. */ 876 if (flags & PIPE_CONTEXT_COMPUTE_ONLY) 877 return ctx; 878 879 /* When shaders are logged to stderr, asynchronous compilation is 880 * disabled too. */ 881 if (sscreen->debug_flags & DBG_ALL_SHADERS) 882 return ctx; 883 884 /* Use asynchronous flushes only on amdgpu, since the radeon 885 * implementation for fence_server_sync is incomplete. */ 886 struct pipe_context *tc = 887 threaded_context_create(ctx, &sscreen->pool_transfers, 888 si_replace_buffer_storage, 889 &(struct threaded_context_options){ 890 .create_fence = sscreen->info.is_amdgpu ? 891 si_create_fence : NULL, 892 .is_resource_busy = si_is_resource_busy, 893 .driver_calls_flush_notify = true, 894 }, 895 &((struct si_context *)ctx)->tc); 896 897 if (tc && tc != ctx) 898 threaded_context_init_bytes_mapped_limit((struct threaded_context *)tc, 4); 899 900 return tc; 901} 902 903/* 904 * pipe_screen 905 */ 906static void si_destroy_screen(struct pipe_screen *pscreen) 907{ 908 struct si_screen *sscreen = (struct si_screen *)pscreen; 909 struct si_shader_part *parts[] = {sscreen->vs_prologs, sscreen->tcs_epilogs, 910 sscreen->ps_prologs, sscreen->ps_epilogs}; 911 unsigned i; 912 913 if (!sscreen->ws->unref(sscreen->ws)) 914 return; 915 916 if (sscreen->debug_flags & DBG(CACHE_STATS)) { 917 printf("live shader cache: hits = %u, misses = %u\n", sscreen->live_shader_cache.hits, 918 sscreen->live_shader_cache.misses); 919 printf("memory shader cache: hits = %u, misses = %u\n", sscreen->num_memory_shader_cache_hits, 920 sscreen->num_memory_shader_cache_misses); 921 printf("disk shader cache: hits = %u, misses = %u\n", sscreen->num_disk_shader_cache_hits, 922 sscreen->num_disk_shader_cache_misses); 923 } 924 925 si_resource_reference(&sscreen->attribute_ring, NULL); 926 927 if (sscreen->aux_context) { 928 struct si_context *saux = si_get_aux_context(sscreen); 929 struct u_log_context *aux_log = saux->log; 930 if (aux_log) { 931 saux->b.set_log_context(&saux->b, NULL); 932 u_log_context_destroy(aux_log); 933 FREE(aux_log); 934 } 935 936 saux->b.destroy(&saux->b); 937 mtx_unlock(&sscreen->aux_context_lock); 938 } 939 mtx_destroy(&sscreen->aux_context_lock); 940 941 simple_mtx_destroy(&sscreen->async_compute_context_lock); 942 if (sscreen->async_compute_context) { 943 sscreen->async_compute_context->destroy(sscreen->async_compute_context); 944 } 945 946 util_queue_destroy(&sscreen->shader_compiler_queue); 947 util_queue_destroy(&sscreen->shader_compiler_queue_low_priority); 948 949 /* Release the reference on glsl types of the compiler threads. */ 950 glsl_type_singleton_decref(); 951 952 for (i = 0; i < ARRAY_SIZE(sscreen->compiler); i++) 953 si_destroy_compiler(&sscreen->compiler[i]); 954 955 for (i = 0; i < ARRAY_SIZE(sscreen->compiler_lowp); i++) 956 si_destroy_compiler(&sscreen->compiler_lowp[i]); 957 958 /* Free shader parts. */ 959 for (i = 0; i < ARRAY_SIZE(parts); i++) { 960 while (parts[i]) { 961 struct si_shader_part *part = parts[i]; 962 963 parts[i] = part->next; 964 si_shader_binary_clean(&part->binary); 965 FREE(part); 966 } 967 } 968 simple_mtx_destroy(&sscreen->shader_parts_mutex); 969 si_destroy_shader_cache(sscreen); 970 971 si_destroy_perfcounters(sscreen); 972 si_gpu_load_kill_thread(sscreen); 973 974 simple_mtx_destroy(&sscreen->gpu_load_mutex); 975 simple_mtx_destroy(&sscreen->gds_mutex); 976 977 radeon_bo_reference(sscreen->ws, &sscreen->gds, NULL); 978 radeon_bo_reference(sscreen->ws, &sscreen->gds_oa, NULL); 979 980 slab_destroy_parent(&sscreen->pool_transfers); 981 982 disk_cache_destroy(sscreen->disk_shader_cache); 983 util_live_shader_cache_deinit(&sscreen->live_shader_cache); 984 util_idalloc_mt_fini(&sscreen->buffer_ids); 985 util_vertex_state_cache_deinit(&sscreen->vertex_state_cache); 986 987 sscreen->ws->destroy(sscreen->ws); 988 FREE(sscreen); 989} 990 991static void si_init_gs_info(struct si_screen *sscreen) 992{ 993 sscreen->gs_table_depth = ac_get_gs_table_depth(sscreen->info.gfx_level, sscreen->info.family); 994} 995 996static void si_test_vmfault(struct si_screen *sscreen, uint64_t test_flags) 997{ 998 struct pipe_context *ctx = sscreen->aux_context; 999 struct si_context *sctx = (struct si_context *)ctx; 1000 struct pipe_resource *buf = pipe_buffer_create_const0(&sscreen->b, 0, PIPE_USAGE_DEFAULT, 64); 1001 1002 if (!buf) { 1003 puts("Buffer allocation failed."); 1004 exit(1); 1005 } 1006 1007 si_resource(buf)->gpu_address = 0; /* cause a VM fault */ 1008 1009 if (test_flags & DBG(TEST_VMFAULT_CP)) { 1010 si_cp_dma_copy_buffer(sctx, buf, buf, 0, 4, 4, SI_OP_SYNC_BEFORE_AFTER, 1011 SI_COHERENCY_NONE, L2_BYPASS); 1012 ctx->flush(ctx, NULL, 0); 1013 puts("VM fault test: CP - done."); 1014 } 1015 if (test_flags & DBG(TEST_VMFAULT_SHADER)) { 1016 util_test_constant_buffer(ctx, buf); 1017 puts("VM fault test: Shader - done."); 1018 } 1019 exit(0); 1020} 1021 1022static void si_test_gds_memory_management(struct si_context *sctx, unsigned alloc_size, 1023 unsigned alignment, enum radeon_bo_domain domain) 1024{ 1025 struct radeon_winsys *ws = sctx->ws; 1026 struct radeon_cmdbuf cs[8]; 1027 struct pb_buffer *gds_bo[ARRAY_SIZE(cs)]; 1028 1029 for (unsigned i = 0; i < ARRAY_SIZE(cs); i++) { 1030 ws->cs_create(&cs[i], sctx->ctx, AMD_IP_COMPUTE, NULL, NULL, false); 1031 gds_bo[i] = ws->buffer_create(ws, alloc_size, alignment, domain, 0); 1032 assert(gds_bo[i]); 1033 } 1034 1035 for (unsigned iterations = 0; iterations < 20000; iterations++) { 1036 for (unsigned i = 0; i < ARRAY_SIZE(cs); i++) { 1037 /* This clears GDS with CP DMA. 1038 * 1039 * We don't care if GDS is present. Just add some packet 1040 * to make the GPU busy for a moment. 1041 */ 1042 si_cp_dma_clear_buffer( 1043 sctx, &cs[i], NULL, 0, alloc_size, 0, 1044 SI_OP_CPDMA_SKIP_CHECK_CS_SPACE, 0, 1045 0); 1046 1047 ws->cs_add_buffer(&cs[i], gds_bo[i], RADEON_USAGE_READWRITE, domain); 1048 ws->cs_flush(&cs[i], PIPE_FLUSH_ASYNC, NULL); 1049 } 1050 } 1051 exit(0); 1052} 1053 1054static void si_disk_cache_create(struct si_screen *sscreen) 1055{ 1056 /* Don't use the cache if shader dumping is enabled. */ 1057 if (sscreen->debug_flags & DBG_ALL_SHADERS) 1058 return; 1059 1060 struct mesa_sha1 ctx; 1061 unsigned char sha1[20]; 1062 char cache_id[20 * 2 + 1]; 1063 1064 _mesa_sha1_init(&ctx); 1065 1066 if (!disk_cache_get_function_identifier(si_disk_cache_create, &ctx) || 1067 !disk_cache_get_function_identifier(LLVMInitializeAMDGPUTargetInfo, &ctx)) 1068 return; 1069 1070 _mesa_sha1_final(&ctx, sha1); 1071 disk_cache_format_hex_id(cache_id, sha1, 20 * 2); 1072 1073 sscreen->disk_shader_cache = disk_cache_create(sscreen->info.name, cache_id, 1074 sscreen->info.address32_hi); 1075} 1076 1077static void si_set_max_shader_compiler_threads(struct pipe_screen *screen, unsigned max_threads) 1078{ 1079 struct si_screen *sscreen = (struct si_screen *)screen; 1080 1081 /* This function doesn't allow a greater number of threads than 1082 * the queue had at its creation. */ 1083 util_queue_adjust_num_threads(&sscreen->shader_compiler_queue, max_threads); 1084 /* Don't change the number of threads on the low priority queue. */ 1085} 1086 1087static bool si_is_parallel_shader_compilation_finished(struct pipe_screen *screen, void *shader, 1088 enum pipe_shader_type shader_type) 1089{ 1090 struct si_shader_selector *sel = (struct si_shader_selector *)shader; 1091 1092 return util_queue_fence_is_signalled(&sel->ready); 1093} 1094 1095static struct pipe_screen *radeonsi_screen_create_impl(struct radeon_winsys *ws, 1096 const struct pipe_screen_config *config) 1097{ 1098 struct si_screen *sscreen = CALLOC_STRUCT(si_screen); 1099 unsigned hw_threads, num_comp_hi_threads, num_comp_lo_threads; 1100 uint64_t test_flags; 1101 1102 if (!sscreen) { 1103 return NULL; 1104 } 1105 1106 { 1107#define OPT_BOOL(name, dflt, description) \ 1108 sscreen->options.name = driQueryOptionb(config->options, "radeonsi_" #name); 1109#define OPT_INT(name, dflt, description) \ 1110 sscreen->options.name = driQueryOptioni(config->options, "radeonsi_" #name); 1111#include "si_debug_options.h" 1112 } 1113 1114 sscreen->ws = ws; 1115 ws->query_info(ws, &sscreen->info, 1116 sscreen->options.enable_sam, 1117 sscreen->options.disable_sam); 1118 1119 if (sscreen->info.gfx_level >= GFX9) { 1120 sscreen->se_tile_repeat = 32 * sscreen->info.max_se; 1121 } else { 1122 ac_get_raster_config(&sscreen->info, &sscreen->pa_sc_raster_config, 1123 &sscreen->pa_sc_raster_config_1, &sscreen->se_tile_repeat); 1124 } 1125 1126 sscreen->debug_flags = debug_get_flags_option("R600_DEBUG", radeonsi_debug_options, 0); 1127 sscreen->debug_flags |= debug_get_flags_option("AMD_DEBUG", radeonsi_debug_options, 0); 1128 test_flags = debug_get_flags_option("AMD_TEST", test_options, 0); 1129 1130 if (sscreen->debug_flags & DBG(NO_GFX)) 1131 sscreen->info.has_graphics = false; 1132 1133 if ((sscreen->debug_flags & DBG(TMZ)) && 1134 !sscreen->info.has_tmz_support) { 1135 fprintf(stderr, "radeonsi: requesting TMZ features but TMZ is not supported\n"); 1136 FREE(sscreen); 1137 return NULL; 1138 } 1139 1140 /* Initialize just one compiler instance to check for errors. The other compiler instances are 1141 * initialized on demand. 1142 */ 1143 if (!si_init_compiler(sscreen, &sscreen->compiler[0])) { 1144 /* The callee prints the error message. */ 1145 FREE(sscreen); 1146 return NULL; 1147 } 1148 1149 util_idalloc_mt_init_tc(&sscreen->buffer_ids); 1150 1151 /* Set functions first. */ 1152 sscreen->b.context_create = si_pipe_create_context; 1153 sscreen->b.destroy = si_destroy_screen; 1154 sscreen->b.set_max_shader_compiler_threads = si_set_max_shader_compiler_threads; 1155 sscreen->b.is_parallel_shader_compilation_finished = si_is_parallel_shader_compilation_finished; 1156 sscreen->b.finalize_nir = si_finalize_nir; 1157 1158 si_init_screen_get_functions(sscreen); 1159 si_init_screen_buffer_functions(sscreen); 1160 si_init_screen_fence_functions(sscreen); 1161 si_init_screen_state_functions(sscreen); 1162 si_init_screen_texture_functions(sscreen); 1163 si_init_screen_query_functions(sscreen); 1164 si_init_screen_live_shader_cache(sscreen); 1165 1166 sscreen->max_texel_buffer_elements = sscreen->b.get_param( 1167 &sscreen->b, PIPE_CAP_MAX_TEXEL_BUFFER_ELEMENTS_UINT); 1168 1169 /* Set these flags in debug_flags early, so that the shader cache takes 1170 * them into account. 1171 * 1172 * Enable FS_CORRECT_DERIVS_AFTER_KILL by default if LLVM is >= 13. This makes 1173 * nir_opt_move_discards_to_top more effective. 1174 */ 1175 if (driQueryOptionb(config->options, "glsl_correct_derivatives_after_discard") || 1176 LLVM_VERSION_MAJOR >= 13) 1177 sscreen->debug_flags |= DBG(FS_CORRECT_DERIVS_AFTER_KILL); 1178 1179 if (sscreen->debug_flags & DBG(INFO)) 1180 ac_print_gpu_info(&sscreen->info, stdout); 1181 1182 slab_create_parent(&sscreen->pool_transfers, sizeof(struct si_transfer), 64); 1183 1184 sscreen->force_aniso = MIN2(16, debug_get_num_option("R600_TEX_ANISO", -1)); 1185 if (sscreen->force_aniso == -1) { 1186 sscreen->force_aniso = MIN2(16, debug_get_num_option("AMD_TEX_ANISO", -1)); 1187 } 1188 1189 if (sscreen->force_aniso >= 0) { 1190 printf("radeonsi: Forcing anisotropy filter to %ix\n", 1191 /* round down to a power of two */ 1192 1 << util_logbase2(sscreen->force_aniso)); 1193 } 1194 1195 (void)mtx_init(&sscreen->aux_context_lock, mtx_recursive); 1196 (void)simple_mtx_init(&sscreen->async_compute_context_lock, mtx_plain); 1197 (void)simple_mtx_init(&sscreen->gpu_load_mutex, mtx_plain); 1198 (void)simple_mtx_init(&sscreen->gds_mutex, mtx_plain); 1199 1200 si_init_gs_info(sscreen); 1201 if (!si_init_shader_cache(sscreen)) { 1202 FREE(sscreen); 1203 return NULL; 1204 } 1205 1206 if (sscreen->info.gfx_level < GFX10_3) 1207 sscreen->options.vrs2x2 = false; 1208 1209 si_disk_cache_create(sscreen); 1210 1211 /* Determine the number of shader compiler threads. */ 1212 const struct util_cpu_caps_t *caps = util_get_cpu_caps(); 1213 hw_threads = caps->nr_cpus; 1214 1215 if (hw_threads >= 12) { 1216 num_comp_hi_threads = hw_threads * 3 / 4; 1217 num_comp_lo_threads = hw_threads / 3; 1218 } else if (hw_threads >= 6) { 1219 num_comp_hi_threads = hw_threads - 2; 1220 num_comp_lo_threads = hw_threads / 2; 1221 } else if (hw_threads >= 2) { 1222 num_comp_hi_threads = hw_threads - 1; 1223 num_comp_lo_threads = hw_threads / 2; 1224 } else { 1225 num_comp_hi_threads = 1; 1226 num_comp_lo_threads = 1; 1227 } 1228 1229#ifndef NDEBUG 1230 nir_process_debug_variable(); 1231 1232 /* Use a single compilation thread if NIR printing is enabled to avoid 1233 * multiple shaders being printed at the same time. 1234 */ 1235 if (NIR_DEBUG(PRINT)) { 1236 num_comp_hi_threads = 1; 1237 num_comp_lo_threads = 1; 1238 } 1239#endif 1240 1241 num_comp_hi_threads = MIN2(num_comp_hi_threads, ARRAY_SIZE(sscreen->compiler)); 1242 num_comp_lo_threads = MIN2(num_comp_lo_threads, ARRAY_SIZE(sscreen->compiler_lowp)); 1243 1244 /* Take a reference on the glsl types for the compiler threads. */ 1245 glsl_type_singleton_init_or_ref(); 1246 1247 /* Start with a single thread and a single slot. 1248 * Each time we'll hit the "all slots are in use" case, the number of threads and 1249 * slots will be increased. 1250 */ 1251 int num_slots = num_comp_hi_threads == 1 ? 64 : 1; 1252 if (!util_queue_init(&sscreen->shader_compiler_queue, "sh", num_slots, 1253 num_comp_hi_threads, 1254 UTIL_QUEUE_INIT_RESIZE_IF_FULL | 1255 UTIL_QUEUE_INIT_SCALE_THREADS | 1256 UTIL_QUEUE_INIT_SET_FULL_THREAD_AFFINITY, NULL)) { 1257 si_destroy_shader_cache(sscreen); 1258 FREE(sscreen); 1259 glsl_type_singleton_decref(); 1260 return NULL; 1261 } 1262 1263 if (!util_queue_init(&sscreen->shader_compiler_queue_low_priority, "shlo", num_slots, 1264 num_comp_lo_threads, 1265 UTIL_QUEUE_INIT_RESIZE_IF_FULL | 1266 UTIL_QUEUE_INIT_SCALE_THREADS | 1267 UTIL_QUEUE_INIT_SET_FULL_THREAD_AFFINITY | 1268 UTIL_QUEUE_INIT_USE_MINIMUM_PRIORITY, NULL)) { 1269 si_destroy_shader_cache(sscreen); 1270 FREE(sscreen); 1271 glsl_type_singleton_decref(); 1272 return NULL; 1273 } 1274 1275 if (!debug_get_bool_option("RADEON_DISABLE_PERFCOUNTERS", false)) 1276 si_init_perfcounters(sscreen); 1277 1278 sscreen->max_memory_usage_kb = sscreen->info.vram_size_kb + sscreen->info.gart_size_kb / 4 * 3; 1279 1280 ac_get_hs_info(&sscreen->info, &sscreen->hs); 1281 1282 sscreen->has_draw_indirect_multi = 1283 (sscreen->info.family >= CHIP_POLARIS10) || 1284 (sscreen->info.gfx_level == GFX8 && sscreen->info.pfp_fw_version >= 121 && 1285 sscreen->info.me_fw_version >= 87) || 1286 (sscreen->info.gfx_level == GFX7 && sscreen->info.pfp_fw_version >= 211 && 1287 sscreen->info.me_fw_version >= 173) || 1288 (sscreen->info.gfx_level == GFX6 && sscreen->info.pfp_fw_version >= 79 && 1289 sscreen->info.me_fw_version >= 142); 1290 1291 sscreen->has_out_of_order_rast = 1292 sscreen->info.has_out_of_order_rast && !(sscreen->debug_flags & DBG(NO_OUT_OF_ORDER)); 1293 1294 if (sscreen->info.gfx_level >= GFX11) { 1295 sscreen->use_ngg = true; 1296 sscreen->use_ngg_streamout = true; 1297 /* TODO: Disable for now. Investigate if it helps. */ 1298 sscreen->use_ngg_culling = (sscreen->debug_flags & DBG(ALWAYS_NGG_CULLING_ALL)) && 1299 !(sscreen->debug_flags & DBG(NO_NGG_CULLING)); 1300 } else { 1301 sscreen->use_ngg = !(sscreen->debug_flags & DBG(NO_NGG)) && 1302 sscreen->info.gfx_level >= GFX10 && 1303 (sscreen->info.family != CHIP_NAVI14 || 1304 sscreen->info.is_pro_graphics); 1305 sscreen->use_ngg_streamout = false; 1306 sscreen->use_ngg_culling = sscreen->use_ngg && 1307 sscreen->info.max_render_backends >= 2 && 1308 !(sscreen->debug_flags & DBG(NO_NGG_CULLING)) && 1309 LLVM_VERSION_MAJOR >= 12; /* hangs on 11, see #4874 */ 1310 } 1311 1312 /* Only set this for the cases that are known to work, which are: 1313 * - GFX9 if bpp >= 4 (in bytes) 1314 */ 1315 if (sscreen->info.gfx_level >= GFX10) { 1316 memset(sscreen->allow_dcc_msaa_clear_to_reg_for_bpp, true, 1317 sizeof(sscreen->allow_dcc_msaa_clear_to_reg_for_bpp)); 1318 } else if (sscreen->info.gfx_level == GFX9) { 1319 for (unsigned bpp_log2 = util_logbase2(1); bpp_log2 <= util_logbase2(16); bpp_log2++) 1320 sscreen->allow_dcc_msaa_clear_to_reg_for_bpp[bpp_log2] = true; 1321 } 1322 1323 /* DCC stores have 50% performance of uncompressed stores and sometimes 1324 * even less than that. It's risky to enable on dGPUs. 1325 */ 1326 sscreen->always_allow_dcc_stores = !(sscreen->debug_flags & DBG(NO_DCC_STORE)) && 1327 (sscreen->debug_flags & DBG(DCC_STORE) || 1328 sscreen->info.gfx_level >= GFX11 || /* always enabled on gfx11 */ 1329 (sscreen->info.gfx_level >= GFX10_3 && 1330 !sscreen->info.has_dedicated_vram)); 1331 1332 sscreen->dpbb_allowed = !(sscreen->debug_flags & DBG(NO_DPBB)) && 1333 (sscreen->info.gfx_level >= GFX10 || 1334 /* Only enable primitive binning on gfx9 APUs by default. */ 1335 (sscreen->info.gfx_level == GFX9 && !sscreen->info.has_dedicated_vram) || 1336 sscreen->debug_flags & DBG(DPBB)); 1337 1338 if (sscreen->dpbb_allowed) { 1339 if (sscreen->info.has_dedicated_vram) { 1340 if (sscreen->info.max_render_backends > 4) { 1341 sscreen->pbb_context_states_per_bin = 1; 1342 sscreen->pbb_persistent_states_per_bin = 1; 1343 } else { 1344 sscreen->pbb_context_states_per_bin = 3; 1345 sscreen->pbb_persistent_states_per_bin = 8; 1346 } 1347 } else { 1348 /* This is a workaround for: 1349 * https://bugs.freedesktop.org/show_bug.cgi?id=110214 1350 * (an alternative is to insert manual BATCH_BREAK event when 1351 * a context_roll is detected). */ 1352 sscreen->pbb_context_states_per_bin = sscreen->info.has_gfx9_scissor_bug ? 1 : 6; 1353 /* Using 32 here can cause GPU hangs on RAVEN1 */ 1354 sscreen->pbb_persistent_states_per_bin = 16; 1355 } 1356 1357 assert(sscreen->pbb_context_states_per_bin >= 1 && 1358 sscreen->pbb_context_states_per_bin <= 6); 1359 assert(sscreen->pbb_persistent_states_per_bin >= 1 && 1360 sscreen->pbb_persistent_states_per_bin <= 32); 1361 } 1362 1363 (void)simple_mtx_init(&sscreen->shader_parts_mutex, mtx_plain); 1364 sscreen->use_monolithic_shaders = (sscreen->debug_flags & DBG(MONOLITHIC_SHADERS)) != 0; 1365 1366 sscreen->barrier_flags.cp_to_L2 = SI_CONTEXT_INV_SCACHE | SI_CONTEXT_INV_VCACHE; 1367 if (sscreen->info.gfx_level <= GFX8) { 1368 sscreen->barrier_flags.cp_to_L2 |= SI_CONTEXT_INV_L2; 1369 sscreen->barrier_flags.L2_to_cp |= SI_CONTEXT_WB_L2; 1370 } 1371 1372 if (debug_get_bool_option("RADEON_DUMP_SHADERS", false)) 1373 sscreen->debug_flags |= DBG_ALL_SHADERS; 1374 1375 /* Syntax: 1376 * EQAA=s,z,c 1377 * Example: 1378 * EQAA=8,4,2 1379 1380 * That means 8 coverage samples, 4 Z/S samples, and 2 color samples. 1381 * Constraints: 1382 * s >= z >= c (ignoring this only wastes memory) 1383 * s = [2..16] 1384 * z = [2..8] 1385 * c = [2..8] 1386 * 1387 * Only MSAA color and depth buffers are overriden. 1388 */ 1389 if (sscreen->info.has_eqaa_surface_allocator) { 1390 const char *eqaa = debug_get_option("EQAA", NULL); 1391 unsigned s, z, f; 1392 1393 if (eqaa && sscanf(eqaa, "%u,%u,%u", &s, &z, &f) == 3 && s && z && f) { 1394 sscreen->eqaa_force_coverage_samples = s; 1395 sscreen->eqaa_force_z_samples = z; 1396 sscreen->eqaa_force_color_samples = f; 1397 } 1398 } 1399 1400 sscreen->ngg_subgroup_size = 128; 1401 1402 if (sscreen->info.gfx_level >= GFX11) { 1403 /* TODO: tweak this */ 1404 unsigned attr_ring_size_per_se = align(1400000, 64 * 1024); 1405 unsigned attr_ring_size = attr_ring_size_per_se * sscreen->info.max_se; 1406 assert(attr_ring_size <= 16 * 1024 * 1024); /* maximum size */ 1407 sscreen->attribute_ring = si_aligned_buffer_create(&sscreen->b, 1408 PIPE_RESOURCE_FLAG_UNMAPPABLE | 1409 SI_RESOURCE_FLAG_32BIT | 1410 SI_RESOURCE_FLAG_DRIVER_INTERNAL | 1411 SI_RESOURCE_FLAG_DISCARDABLE, 1412 PIPE_USAGE_DEFAULT, 1413 attr_ring_size, 2 * 1024 * 1024); 1414 } 1415 1416 /* Create the auxiliary context. This must be done last. */ 1417 sscreen->aux_context = si_create_context( 1418 &sscreen->b, 1419 SI_CONTEXT_FLAG_AUX | 1420 (sscreen->options.aux_debug ? PIPE_CONTEXT_DEBUG : 0) | 1421 (sscreen->info.has_graphics ? 0 : PIPE_CONTEXT_COMPUTE_ONLY)); 1422 1423 if (sscreen->options.aux_debug) { 1424 struct u_log_context *log = CALLOC_STRUCT(u_log_context); 1425 u_log_context_init(log); 1426 si_get_aux_context(sscreen)->b.set_log_context(sscreen->aux_context, log); 1427 si_put_aux_context_flush(sscreen); 1428 } 1429 1430 if (test_flags & DBG(TEST_IMAGE_COPY)) 1431 si_test_image_copy_region(sscreen); 1432 1433 if (test_flags & (DBG(TEST_CB_RESOLVE) | DBG(TEST_COMPUTE_BLIT))) 1434 si_test_blit(sscreen, test_flags); 1435 1436 if (test_flags & DBG(TEST_DMA_PERF)) { 1437 si_test_dma_perf(sscreen); 1438 } 1439 1440 if (test_flags & (DBG(TEST_VMFAULT_CP) | DBG(TEST_VMFAULT_SHADER))) 1441 si_test_vmfault(sscreen, test_flags); 1442 1443 if (test_flags & DBG(TEST_GDS)) 1444 si_test_gds((struct si_context *)sscreen->aux_context); 1445 1446 if (test_flags & DBG(TEST_GDS_MM)) { 1447 si_test_gds_memory_management((struct si_context *)sscreen->aux_context, 32 * 1024, 4, 1448 RADEON_DOMAIN_GDS); 1449 } 1450 if (test_flags & DBG(TEST_GDS_OA_MM)) { 1451 si_test_gds_memory_management((struct si_context *)sscreen->aux_context, 4, 1, 1452 RADEON_DOMAIN_OA); 1453 } 1454 1455 ac_print_shadowed_regs(&sscreen->info); 1456 1457 STATIC_ASSERT(sizeof(union si_vgt_stages_key) == 1); 1458 return &sscreen->b; 1459} 1460 1461struct pipe_screen *radeonsi_screen_create(int fd, const struct pipe_screen_config *config) 1462{ 1463 drmVersionPtr version = drmGetVersion(fd); 1464 struct radeon_winsys *rw = NULL; 1465 1466 driParseConfigFiles(config->options, config->options_info, 0, "radeonsi", 1467 NULL, NULL, NULL, 0, NULL, 0); 1468 1469 switch (version->version_major) { 1470 case 2: 1471 rw = radeon_drm_winsys_create(fd, config, radeonsi_screen_create_impl); 1472 break; 1473 case 3: 1474 rw = amdgpu_winsys_create(fd, config, radeonsi_screen_create_impl); 1475 break; 1476 } 1477 1478 drmFreeVersion(version); 1479 return rw ? rw->screen : NULL; 1480} 1481 1482struct si_context* si_get_aux_context(struct si_screen *sscreen) 1483{ 1484 mtx_lock(&sscreen->aux_context_lock); 1485 return (struct si_context*)sscreen->aux_context; 1486} 1487 1488void si_put_aux_context_flush(struct si_screen *sscreen) 1489{ 1490 struct pipe_context *c = &((struct si_context*)sscreen->aux_context)->b; 1491 c->flush(c, NULL, 0); 1492 mtx_unlock(&sscreen->aux_context_lock); 1493} 1494