1/* 2 * Copyright 2018 Collabora Ltd. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * on the rights to use, copy, modify, merge, publish, distribute, sub 8 * license, and/or sell copies of the Software, and to permit persons to whom 9 * the Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM, 19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 21 * USE OR OTHER DEALINGS IN THE SOFTWARE. 22 */ 23 24#include "zink_context.h" 25 26#include "zink_batch.h" 27#include "zink_compiler.h" 28#include "zink_kopper.h" 29#include "zink_fence.h" 30#include "zink_format.h" 31#include "zink_framebuffer.h" 32#include "zink_helpers.h" 33#include "zink_program.h" 34#include "zink_pipeline.h" 35#include "zink_query.h" 36#include "zink_render_pass.h" 37#include "zink_resource.h" 38#include "zink_screen.h" 39#include "zink_state.h" 40#include "zink_surface.h" 41#include "zink_inlines.h" 42 43#include "util/u_blitter.h" 44#include "util/u_debug.h" 45#include "util/format_srgb.h" 46#include "util/format/u_format.h" 47#include "util/u_helpers.h" 48#include "util/u_inlines.h" 49#include "util/u_thread.h" 50#include "util/u_cpu_detect.h" 51#include "util/strndup.h" 52#include "nir.h" 53 54#include "driver_trace/tr_context.h" 55 56#include "util/u_memory.h" 57#include "util/u_upload_mgr.h" 58 59#define XXH_INLINE_ALL 60#include "util/xxhash.h" 61 62static void 63calc_descriptor_hash_sampler_state(struct zink_sampler_state *sampler_state) 64{ 65 void *hash_data = &sampler_state->sampler; 66 size_t data_size = sizeof(VkSampler); 67 sampler_state->hash = XXH32(hash_data, data_size, 0); 68} 69 70void 71debug_describe_zink_buffer_view(char *buf, const struct zink_buffer_view *ptr) 72{ 73 sprintf(buf, "zink_buffer_view"); 74} 75 76ALWAYS_INLINE static void 77check_resource_for_batch_ref(struct zink_context *ctx, struct zink_resource *res) 78{ 79 if (!zink_resource_has_binds(res)) { 80 /* avoid desync between usage and tracking: 81 * - if usage exists, it must be removed before the context is destroyed 82 * - having usage does not imply having tracking 83 * - if tracking will be added here, also reapply usage to avoid dangling usage once tracking is removed 84 * TODO: somehow fix this for perf because it's an extra hash lookup 85 */ 86 if (!res->obj->dt && (res->obj->bo->reads || res->obj->bo->writes)) 87 zink_batch_reference_resource_rw(&ctx->batch, res, !!res->obj->bo->writes); 88 else 89 zink_batch_reference_resource(&ctx->batch, res); 90 } 91} 92 93static void 94zink_context_destroy(struct pipe_context *pctx) 95{ 96 struct zink_context *ctx = zink_context(pctx); 97 struct zink_screen *screen = zink_screen(pctx->screen); 98 99 if (util_queue_is_initialized(&screen->flush_queue)) 100 util_queue_finish(&screen->flush_queue); 101 if (ctx->batch.state && !screen->device_lost) { 102 VkResult result = VKSCR(QueueWaitIdle)(screen->queue); 103 104 if (result != VK_SUCCESS) 105 mesa_loge("ZINK: vkQueueWaitIdle failed (%s)", vk_Result_to_str(result)); 106 } 107 108 for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++) { 109 hash_table_foreach(&ctx->program_cache[i], entry) { 110 struct zink_program *pg = entry->data; 111 pg->removed = true; 112 screen->descriptor_program_deinit(ctx, pg); 113 } 114 } 115 hash_table_foreach(&ctx->compute_program_cache, entry) { 116 struct zink_program *pg = entry->data; 117 pg->removed = true; 118 screen->descriptor_program_deinit(ctx, pg); 119 } 120 121 if (ctx->blitter) 122 util_blitter_destroy(ctx->blitter); 123 for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++) 124 pipe_surface_release(&ctx->base, &ctx->fb_state.cbufs[i]); 125 pipe_surface_release(&ctx->base, &ctx->fb_state.zsbuf); 126 127 pipe_resource_reference(&ctx->dummy_vertex_buffer, NULL); 128 pipe_resource_reference(&ctx->dummy_xfb_buffer, NULL); 129 130 for (unsigned i = 0; i < ARRAY_SIZE(ctx->dummy_surface); i++) 131 pipe_surface_release(&ctx->base, &ctx->dummy_surface[i]); 132 zink_buffer_view_reference(screen, &ctx->dummy_bufferview, NULL); 133 134 if (ctx->dd) 135 zink_descriptors_deinit_bindless(ctx); 136 137 if (ctx->batch.state) { 138 zink_clear_batch_state(ctx, ctx->batch.state); 139 zink_batch_state_destroy(screen, ctx->batch.state); 140 } 141 struct zink_batch_state *bs = ctx->batch_states; 142 while (bs) { 143 struct zink_batch_state *bs_next = bs->next; 144 zink_clear_batch_state(ctx, bs); 145 zink_batch_state_destroy(screen, bs); 146 bs = bs_next; 147 } 148 util_dynarray_foreach(&ctx->free_batch_states, struct zink_batch_state*, bs) { 149 zink_clear_batch_state(ctx, *bs); 150 zink_batch_state_destroy(screen, *bs); 151 } 152 153 for (unsigned i = 0; i < 2; i++) { 154 util_idalloc_fini(&ctx->di.bindless[i].tex_slots); 155 util_idalloc_fini(&ctx->di.bindless[i].img_slots); 156 free(ctx->di.bindless[i].buffer_infos); 157 free(ctx->di.bindless[i].img_infos); 158 util_dynarray_fini(&ctx->di.bindless[i].updates); 159 util_dynarray_fini(&ctx->di.bindless[i].resident); 160 } 161 162 hash_table_foreach(&ctx->framebuffer_cache, he) 163 zink_destroy_framebuffer(screen, he->data); 164 165 hash_table_foreach(ctx->render_pass_cache, he) 166 zink_destroy_render_pass(screen, he->data); 167 168 zink_context_destroy_query_pools(ctx); 169 u_upload_destroy(pctx->stream_uploader); 170 u_upload_destroy(pctx->const_uploader); 171 slab_destroy_child(&ctx->transfer_pool); 172 for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++) 173 _mesa_hash_table_clear(&ctx->program_cache[i], NULL); 174 _mesa_hash_table_clear(&ctx->compute_program_cache, NULL); 175 _mesa_hash_table_destroy(ctx->render_pass_cache, NULL); 176 slab_destroy_child(&ctx->transfer_pool_unsync); 177 178 if (ctx->dd) 179 screen->descriptors_deinit(ctx); 180 181 zink_descriptor_layouts_deinit(ctx); 182 183 if (!(ctx->flags & ZINK_CONTEXT_COPY_ONLY)) 184 p_atomic_dec(&screen->base.num_contexts); 185 186 ralloc_free(ctx); 187} 188 189static void 190check_device_lost(struct zink_context *ctx) 191{ 192 if (!zink_screen(ctx->base.screen)->device_lost || ctx->is_device_lost) 193 return; 194 debug_printf("ZINK: device lost detected!\n"); 195 if (ctx->reset.reset) 196 ctx->reset.reset(ctx->reset.data, PIPE_GUILTY_CONTEXT_RESET); 197 ctx->is_device_lost = true; 198} 199 200static enum pipe_reset_status 201zink_get_device_reset_status(struct pipe_context *pctx) 202{ 203 struct zink_context *ctx = zink_context(pctx); 204 205 enum pipe_reset_status status = PIPE_NO_RESET; 206 207 if (ctx->is_device_lost) { 208 // Since we don't know what really happened to the hardware, just 209 // assume that we are in the wrong 210 status = PIPE_GUILTY_CONTEXT_RESET; 211 212 debug_printf("ZINK: device lost detected!\n"); 213 214 if (ctx->reset.reset) 215 ctx->reset.reset(ctx->reset.data, status); 216 } 217 218 return status; 219} 220 221static void 222zink_set_device_reset_callback(struct pipe_context *pctx, 223 const struct pipe_device_reset_callback *cb) 224{ 225 struct zink_context *ctx = zink_context(pctx); 226 bool had_reset = !!ctx->reset.reset; 227 228 if (cb) 229 ctx->reset = *cb; 230 else 231 memset(&ctx->reset, 0, sizeof(ctx->reset)); 232 233 bool have_reset = !!ctx->reset.reset; 234 if (had_reset != have_reset) { 235 if (have_reset) 236 p_atomic_inc(&zink_screen(pctx->screen)->robust_ctx_count); 237 else 238 p_atomic_dec(&zink_screen(pctx->screen)->robust_ctx_count); 239 } 240} 241 242static void 243zink_set_context_param(struct pipe_context *pctx, enum pipe_context_param param, 244 unsigned value) 245{ 246 struct zink_context *ctx = zink_context(pctx); 247 248 switch (param) { 249 case PIPE_CONTEXT_PARAM_PIN_THREADS_TO_L3_CACHE: 250 util_set_thread_affinity(zink_screen(ctx->base.screen)->flush_queue.threads[0], 251 util_get_cpu_caps()->L3_affinity_mask[value], 252 NULL, util_get_cpu_caps()->num_cpu_mask_bits); 253 break; 254 default: 255 break; 256 } 257} 258 259static VkSamplerMipmapMode 260sampler_mipmap_mode(enum pipe_tex_mipfilter filter) 261{ 262 switch (filter) { 263 case PIPE_TEX_MIPFILTER_NEAREST: return VK_SAMPLER_MIPMAP_MODE_NEAREST; 264 case PIPE_TEX_MIPFILTER_LINEAR: return VK_SAMPLER_MIPMAP_MODE_LINEAR; 265 case PIPE_TEX_MIPFILTER_NONE: 266 unreachable("PIPE_TEX_MIPFILTER_NONE should be dealt with earlier"); 267 } 268 unreachable("unexpected filter"); 269} 270 271static VkSamplerAddressMode 272sampler_address_mode(enum pipe_tex_wrap filter) 273{ 274 switch (filter) { 275 case PIPE_TEX_WRAP_REPEAT: return VK_SAMPLER_ADDRESS_MODE_REPEAT; 276 case PIPE_TEX_WRAP_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; 277 case PIPE_TEX_WRAP_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; 278 case PIPE_TEX_WRAP_MIRROR_REPEAT: return VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT; 279 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; 280 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; /* not technically correct, but kinda works */ 281 default: break; 282 } 283 unreachable("unexpected wrap"); 284} 285 286static VkCompareOp 287compare_op(enum pipe_compare_func op) 288{ 289 switch (op) { 290 case PIPE_FUNC_NEVER: return VK_COMPARE_OP_NEVER; 291 case PIPE_FUNC_LESS: return VK_COMPARE_OP_LESS; 292 case PIPE_FUNC_EQUAL: return VK_COMPARE_OP_EQUAL; 293 case PIPE_FUNC_LEQUAL: return VK_COMPARE_OP_LESS_OR_EQUAL; 294 case PIPE_FUNC_GREATER: return VK_COMPARE_OP_GREATER; 295 case PIPE_FUNC_NOTEQUAL: return VK_COMPARE_OP_NOT_EQUAL; 296 case PIPE_FUNC_GEQUAL: return VK_COMPARE_OP_GREATER_OR_EQUAL; 297 case PIPE_FUNC_ALWAYS: return VK_COMPARE_OP_ALWAYS; 298 } 299 unreachable("unexpected compare"); 300} 301 302static inline bool 303wrap_needs_border_color(unsigned wrap) 304{ 305 return wrap == PIPE_TEX_WRAP_CLAMP || wrap == PIPE_TEX_WRAP_CLAMP_TO_BORDER || 306 wrap == PIPE_TEX_WRAP_MIRROR_CLAMP || wrap == PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER; 307} 308 309static VkBorderColor 310get_border_color(const union pipe_color_union *color, bool is_integer, bool need_custom) 311{ 312 if (is_integer) { 313 if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 0) 314 return VK_BORDER_COLOR_INT_TRANSPARENT_BLACK; 315 if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 1) 316 return VK_BORDER_COLOR_INT_OPAQUE_BLACK; 317 if (color->ui[0] == 1 && color->ui[1] == 1 && color->ui[2] == 1 && color->ui[3] == 1) 318 return VK_BORDER_COLOR_INT_OPAQUE_WHITE; 319 return need_custom ? VK_BORDER_COLOR_INT_CUSTOM_EXT : VK_BORDER_COLOR_INT_TRANSPARENT_BLACK; 320 } 321 322 if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 0) 323 return VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; 324 if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 1) 325 return VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK; 326 if (color->f[0] == 1 && color->f[1] == 1 && color->f[2] == 1 && color->f[3] == 1) 327 return VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE; 328 return need_custom ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; 329} 330 331static void * 332zink_create_sampler_state(struct pipe_context *pctx, 333 const struct pipe_sampler_state *state) 334{ 335 struct zink_screen *screen = zink_screen(pctx->screen); 336 bool need_custom = false; 337 bool need_clamped_border_color = false; 338 VkSamplerCreateInfo sci = {0}; 339 VkSamplerCustomBorderColorCreateInfoEXT cbci = {0}; 340 VkSamplerCustomBorderColorCreateInfoEXT cbci_clamped = {0}; 341 sci.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO; 342 if (screen->info.have_EXT_non_seamless_cube_map && !state->seamless_cube_map) 343 sci.flags |= VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT; 344 sci.unnormalizedCoordinates = !state->normalized_coords; 345 sci.magFilter = zink_filter(state->mag_img_filter); 346 if (sci.unnormalizedCoordinates) 347 sci.minFilter = sci.magFilter; 348 else 349 sci.minFilter = zink_filter(state->min_img_filter); 350 351 VkSamplerReductionModeCreateInfo rci; 352 rci.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO; 353 rci.pNext = NULL; 354 switch (state->reduction_mode) { 355 case PIPE_TEX_REDUCTION_MIN: 356 rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MIN; 357 break; 358 case PIPE_TEX_REDUCTION_MAX: 359 rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MAX; 360 break; 361 default: 362 rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE; 363 break; 364 } 365 if (state->reduction_mode) 366 sci.pNext = &rci; 367 368 if (sci.unnormalizedCoordinates) { 369 sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; 370 } else if (state->min_mip_filter != PIPE_TEX_MIPFILTER_NONE) { 371 sci.mipmapMode = sampler_mipmap_mode(state->min_mip_filter); 372 sci.minLod = state->min_lod; 373 sci.maxLod = state->max_lod; 374 } else { 375 sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; 376 sci.minLod = 0; 377 sci.maxLod = 0.25f; 378 } 379 380 if (!sci.unnormalizedCoordinates) { 381 sci.addressModeU = sampler_address_mode(state->wrap_s); 382 sci.addressModeV = sampler_address_mode(state->wrap_t); 383 sci.addressModeW = sampler_address_mode(state->wrap_r); 384 } else { 385 sci.addressModeU = sci.addressModeV = sci.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE; 386 } 387 388 sci.mipLodBias = CLAMP(state->lod_bias, 389 -screen->info.props.limits.maxSamplerLodBias, 390 screen->info.props.limits.maxSamplerLodBias); 391 392 need_custom |= wrap_needs_border_color(state->wrap_s); 393 need_custom |= wrap_needs_border_color(state->wrap_t); 394 need_custom |= wrap_needs_border_color(state->wrap_r); 395 396 if (state->compare_mode == PIPE_TEX_COMPARE_NONE) 397 sci.compareOp = VK_COMPARE_OP_NEVER; 398 else { 399 sci.compareOp = compare_op(state->compare_func); 400 sci.compareEnable = VK_TRUE; 401 } 402 403 bool is_integer = state->border_color_is_integer; 404 405 sci.borderColor = get_border_color(&state->border_color, is_integer, need_custom); 406 if (sci.borderColor > VK_BORDER_COLOR_INT_OPAQUE_WHITE && need_custom) { 407 if (!screen->info.border_color_feats.customBorderColorWithoutFormat && 408 screen->info.driver_props.driverID != VK_DRIVER_ID_MESA_TURNIP) { 409 static bool warned = false; 410 warn_missing_feature(warned, "customBorderColorWithoutFormat"); 411 } 412 if (screen->info.have_EXT_custom_border_color && 413 (screen->info.border_color_feats.customBorderColorWithoutFormat || state->border_color_format)) { 414 if (!screen->info.have_EXT_border_color_swizzle) { 415 static bool warned = false; 416 warn_missing_feature(warned, "VK_EXT_border_color_swizzle"); 417 } 418 419 if (!is_integer && !screen->have_D24_UNORM_S8_UINT) { 420 union pipe_color_union clamped_border_color; 421 for (unsigned i = 0; i < 4; ++i) { 422 /* Use channel 0 on purpose, so that we can use OPAQUE_WHITE 423 * when the border color is 1.0. */ 424 clamped_border_color.f[i] = CLAMP(state->border_color.f[0], 0, 1); 425 } 426 if (memcmp(&state->border_color, &clamped_border_color, sizeof(clamped_border_color)) != 0) { 427 need_clamped_border_color = true; 428 cbci_clamped.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT; 429 cbci_clamped.format = VK_FORMAT_UNDEFINED; 430 /* these are identical unions */ 431 memcpy(&cbci_clamped.customBorderColor, &clamped_border_color, sizeof(union pipe_color_union)); 432 } 433 } 434 cbci.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT; 435 cbci.format = screen->info.border_color_feats.customBorderColorWithoutFormat ? VK_FORMAT_UNDEFINED : zink_get_format(screen, state->border_color_format); 436 /* these are identical unions */ 437 memcpy(&cbci.customBorderColor, &state->border_color, sizeof(union pipe_color_union)); 438 cbci.pNext = sci.pNext; 439 sci.pNext = &cbci; 440 UNUSED uint32_t check = p_atomic_inc_return(&screen->cur_custom_border_color_samplers); 441 assert(check <= screen->info.border_color_props.maxCustomBorderColorSamplers); 442 } else 443 sci.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; // TODO with custom shader if we're super interested? 444 if (sci.unnormalizedCoordinates) 445 sci.addressModeU = sci.addressModeV = sci.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; 446 } 447 448 if (state->max_anisotropy > 1) { 449 sci.maxAnisotropy = state->max_anisotropy; 450 sci.anisotropyEnable = VK_TRUE; 451 } 452 453 struct zink_sampler_state *sampler = CALLOC_STRUCT(zink_sampler_state); 454 if (!sampler) 455 return NULL; 456 457 VkResult result = VKSCR(CreateSampler)(screen->dev, &sci, NULL, &sampler->sampler); 458 if (result != VK_SUCCESS) { 459 mesa_loge("ZINK: vkCreateSampler failed (%s)", vk_Result_to_str(result)); 460 FREE(sampler); 461 return NULL; 462 } 463 if (need_clamped_border_color) { 464 sci.pNext = &cbci_clamped; 465 result = VKSCR(CreateSampler)(screen->dev, &sci, NULL, &sampler->sampler_clamped); 466 if (result != VK_SUCCESS) { 467 mesa_loge("ZINK: vkCreateSampler failed (%s)", vk_Result_to_str(result)); 468 VKSCR(DestroySampler)(screen->dev, sampler->sampler, NULL); 469 FREE(sampler); 470 return NULL; 471 } 472 } 473 util_dynarray_init(&sampler->desc_set_refs.refs, NULL); 474 calc_descriptor_hash_sampler_state(sampler); 475 sampler->custom_border_color = need_custom; 476 if (!screen->info.have_EXT_non_seamless_cube_map) 477 sampler->emulate_nonseamless = !state->seamless_cube_map; 478 479 return sampler; 480} 481 482ALWAYS_INLINE static VkImageLayout 483get_layout_for_binding(const struct zink_context *ctx, struct zink_resource *res, enum zink_descriptor_type type, bool is_compute) 484{ 485 if (res->obj->is_buffer) 486 return 0; 487 switch (type) { 488 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: 489 return zink_descriptor_util_image_layout_eval(ctx, res, is_compute); 490 case ZINK_DESCRIPTOR_TYPE_IMAGE: 491 return VK_IMAGE_LAYOUT_GENERAL; 492 default: 493 break; 494 } 495 return 0; 496} 497 498ALWAYS_INLINE static struct zink_surface * 499get_imageview_for_binding(struct zink_context *ctx, enum pipe_shader_type stage, enum zink_descriptor_type type, unsigned idx) 500{ 501 switch (type) { 502 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: { 503 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]); 504 if (!sampler_view || !sampler_view->base.texture) 505 return NULL; 506 /* if this is a non-seamless cube sampler, return the cube array view */ 507 return (ctx->di.emulate_nonseamless[stage] & ctx->di.cubes[stage] & BITFIELD_BIT(idx)) ? 508 sampler_view->cube_array : 509 sampler_view->image_view; 510 } 511 case ZINK_DESCRIPTOR_TYPE_IMAGE: { 512 struct zink_image_view *image_view = &ctx->image_views[stage][idx]; 513 return image_view->base.resource ? image_view->surface : NULL; 514 } 515 default: 516 break; 517 } 518 unreachable("ACK"); 519 return VK_NULL_HANDLE; 520} 521 522ALWAYS_INLINE static struct zink_buffer_view * 523get_bufferview_for_binding(struct zink_context *ctx, enum pipe_shader_type stage, enum zink_descriptor_type type, unsigned idx) 524{ 525 switch (type) { 526 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: { 527 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]); 528 return sampler_view->base.texture ? sampler_view->buffer_view : NULL; 529 } 530 case ZINK_DESCRIPTOR_TYPE_IMAGE: { 531 struct zink_image_view *image_view = &ctx->image_views[stage][idx]; 532 return image_view->base.resource ? image_view->buffer_view : NULL; 533 } 534 default: 535 break; 536 } 537 unreachable("ACK"); 538 return VK_NULL_HANDLE; 539} 540 541ALWAYS_INLINE static struct zink_resource * 542update_descriptor_state_ubo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot, struct zink_resource *res) 543{ 544 struct zink_screen *screen = zink_screen(ctx->base.screen); 545 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor; 546 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_UBO; 547 ctx->di.descriptor_res[type][shader][slot] = res; 548 ctx->di.ubos[shader][slot].offset = ctx->ubos[shader][slot].buffer_offset; 549 if (res) { 550 ctx->di.ubos[shader][slot].buffer = res->obj->buffer; 551 ctx->di.ubos[shader][slot].range = ctx->ubos[shader][slot].buffer_size; 552 assert(ctx->di.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange); 553 } else { 554 VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer; 555 ctx->di.ubos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer; 556 ctx->di.ubos[shader][slot].range = VK_WHOLE_SIZE; 557 } 558 if (!slot) { 559 if (res) 560 ctx->di.push_valid |= BITFIELD64_BIT(shader); 561 else 562 ctx->di.push_valid &= ~BITFIELD64_BIT(shader); 563 } 564 return res; 565} 566 567ALWAYS_INLINE static struct zink_resource * 568update_descriptor_state_ssbo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot, struct zink_resource *res) 569{ 570 struct zink_screen *screen = zink_screen(ctx->base.screen); 571 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor; 572 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SSBO; 573 ctx->di.descriptor_res[type][shader][slot] = res; 574 ctx->di.ssbos[shader][slot].offset = ctx->ssbos[shader][slot].buffer_offset; 575 if (res) { 576 ctx->di.ssbos[shader][slot].buffer = res->obj->buffer; 577 ctx->di.ssbos[shader][slot].range = ctx->ssbos[shader][slot].buffer_size; 578 } else { 579 VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer; 580 ctx->di.ssbos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer; 581 ctx->di.ssbos[shader][slot].range = VK_WHOLE_SIZE; 582 } 583 return res; 584} 585 586ALWAYS_INLINE static struct zink_resource * 587update_descriptor_state_sampler(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot, struct zink_resource *res) 588{ 589 struct zink_screen *screen = zink_screen(ctx->base.screen); 590 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor; 591 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW; 592 ctx->di.descriptor_res[type][shader][slot] = res; 593 if (res) { 594 if (res->obj->is_buffer) { 595 struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot); 596 ctx->di.tbos[shader][slot] = bv->buffer_view; 597 ctx->di.sampler_surfaces[shader][slot].bufferview = bv; 598 ctx->di.sampler_surfaces[shader][slot].is_buffer = true; 599 } else { 600 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot); 601 ctx->di.textures[shader][slot].imageLayout = get_layout_for_binding(ctx, res, type, shader == PIPE_SHADER_COMPUTE); 602 ctx->di.textures[shader][slot].imageView = surface->image_view; 603 if (!screen->have_D24_UNORM_S8_UINT && 604 ctx->sampler_states[shader][slot] && ctx->sampler_states[shader][slot]->sampler_clamped) { 605 struct zink_sampler_state *state = ctx->sampler_states[shader][slot]; 606 VkSampler sampler = (surface->base.format == PIPE_FORMAT_Z24X8_UNORM && surface->ivci.format == VK_FORMAT_D32_SFLOAT) || 607 (surface->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT && surface->ivci.format == VK_FORMAT_D32_SFLOAT_S8_UINT) ? 608 state->sampler_clamped : 609 state->sampler; 610 if (ctx->di.textures[shader][slot].sampler != sampler) { 611 screen->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1); 612 ctx->di.textures[shader][slot].sampler = sampler; 613 } 614 } 615 ctx->di.sampler_surfaces[shader][slot].surface = surface; 616 ctx->di.sampler_surfaces[shader][slot].is_buffer = false; 617 } 618 } else { 619 if (likely(have_null_descriptors)) { 620 ctx->di.textures[shader][slot].imageView = VK_NULL_HANDLE; 621 ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_UNDEFINED; 622 ctx->di.tbos[shader][slot] = VK_NULL_HANDLE; 623 } else { 624 struct zink_surface *null_surface = zink_csurface(ctx->dummy_surface[0]); 625 struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview; 626 ctx->di.textures[shader][slot].imageView = null_surface->image_view; 627 ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL; 628 ctx->di.tbos[shader][slot] = null_bufferview->buffer_view; 629 } 630 memset(&ctx->di.sampler_surfaces[shader][slot], 0, sizeof(ctx->di.sampler_surfaces[shader][slot])); 631 } 632 return res; 633} 634 635ALWAYS_INLINE static struct zink_resource * 636update_descriptor_state_image(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot, struct zink_resource *res) 637{ 638 struct zink_screen *screen = zink_screen(ctx->base.screen); 639 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor; 640 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_IMAGE; 641 ctx->di.descriptor_res[type][shader][slot] = res; 642 if (res) { 643 if (res->obj->is_buffer) { 644 struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot); 645 ctx->di.texel_images[shader][slot] = bv->buffer_view; 646 ctx->di.image_surfaces[shader][slot].bufferview = bv; 647 ctx->di.image_surfaces[shader][slot].is_buffer = true; 648 } else { 649 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot); 650 ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL; 651 ctx->di.images[shader][slot].imageView = surface->image_view; 652 ctx->di.image_surfaces[shader][slot].surface = surface; 653 ctx->di.image_surfaces[shader][slot].is_buffer = false; 654 } 655 } else { 656 if (likely(have_null_descriptors)) { 657 memset(&ctx->di.images[shader][slot], 0, sizeof(ctx->di.images[shader][slot])); 658 ctx->di.texel_images[shader][slot] = VK_NULL_HANDLE; 659 } else { 660 struct zink_surface *null_surface = zink_csurface(ctx->dummy_surface[0]); 661 struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview; 662 ctx->di.images[shader][slot].imageView = null_surface->image_view; 663 ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL; 664 ctx->di.texel_images[shader][slot] = null_bufferview->buffer_view; 665 } 666 memset(&ctx->di.image_surfaces[shader][slot], 0, sizeof(ctx->di.image_surfaces[shader][slot])); 667 } 668 return res; 669} 670 671static void 672update_nonseamless_shader_key(struct zink_context *ctx, enum pipe_shader_type pstage) 673{ 674 uint32_t *mask; 675 if (pstage == PIPE_SHADER_COMPUTE) 676 mask = &ctx->compute_pipeline_state.key.base.nonseamless_cube_mask; 677 else 678 mask = &ctx->gfx_pipeline_state.shader_keys.key[pstage].base.nonseamless_cube_mask; 679 680 const uint32_t new_mask = ctx->di.emulate_nonseamless[pstage] & ctx->di.cubes[pstage]; 681 if (new_mask != *mask) 682 ctx->dirty_shader_stages |= BITFIELD_BIT(pstage); 683 *mask = new_mask; 684} 685 686static void 687zink_bind_sampler_states(struct pipe_context *pctx, 688 enum pipe_shader_type shader, 689 unsigned start_slot, 690 unsigned num_samplers, 691 void **samplers) 692{ 693 struct zink_context *ctx = zink_context(pctx); 694 struct zink_screen *screen = zink_screen(pctx->screen); 695 uint32_t mask = BITFIELD_RANGE(start_slot, num_samplers); 696 ctx->di.emulate_nonseamless[shader] &= ~mask; 697 for (unsigned i = 0; i < num_samplers; ++i) { 698 struct zink_sampler_state *state = samplers[i]; 699 if (ctx->sampler_states[shader][start_slot + i] != state) 700 zink_screen(pctx->screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, 1); 701 bool was_nonseamless = false; 702 if (ctx->sampler_states[shader][start_slot + i]) 703 was_nonseamless = ctx->sampler_states[shader][start_slot + i]->emulate_nonseamless; 704 ctx->sampler_states[shader][start_slot + i] = state; 705 if (state) { 706 ctx->di.textures[shader][start_slot + i].sampler = state->sampler; 707 if (state->sampler_clamped && !screen->have_D24_UNORM_S8_UINT) { 708 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i); 709 if (surface && 710 ((surface->base.format == PIPE_FORMAT_Z24X8_UNORM && surface->ivci.format == VK_FORMAT_D32_SFLOAT) || 711 (surface->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT && surface->ivci.format == VK_FORMAT_D32_SFLOAT_S8_UINT))) 712 ctx->di.textures[shader][start_slot + i].sampler = state->sampler_clamped; 713 } 714 zink_batch_usage_set(&state->batch_uses, ctx->batch.state); 715 if (screen->info.have_EXT_non_seamless_cube_map) 716 continue; 717 const uint32_t bit = BITFIELD_BIT(start_slot + i); 718 if (state->emulate_nonseamless) 719 ctx->di.emulate_nonseamless[shader] |= bit; 720 if (state->emulate_nonseamless != was_nonseamless && (ctx->di.cubes[shader] & bit)) { 721 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i); 722 if (surface && ctx->di.image_surfaces[shader][start_slot + i].surface != surface) { 723 ctx->di.images[shader][start_slot + i].imageView = surface->image_view; 724 ctx->di.image_surfaces[shader][start_slot + i].surface = surface; 725 update_descriptor_state_sampler(ctx, shader, start_slot + i, zink_resource(surface->base.texture)); 726 screen->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot + i, 1); 727 } 728 } 729 } else { 730 ctx->di.textures[shader][start_slot + i].sampler = VK_NULL_HANDLE; 731 } 732 } 733 ctx->di.num_samplers[shader] = start_slot + num_samplers; 734 if (!screen->info.have_EXT_non_seamless_cube_map) 735 update_nonseamless_shader_key(ctx, shader); 736} 737 738static void 739zink_delete_sampler_state(struct pipe_context *pctx, 740 void *sampler_state) 741{ 742 struct zink_sampler_state *sampler = sampler_state; 743 struct zink_batch *batch = &zink_context(pctx)->batch; 744 zink_descriptor_set_refs_clear(&sampler->desc_set_refs, sampler_state); 745 /* may be called if context_create fails */ 746 if (batch->state) { 747 util_dynarray_append(&batch->state->zombie_samplers, VkSampler, 748 sampler->sampler); 749 if (sampler->sampler_clamped) 750 util_dynarray_append(&batch->state->zombie_samplers, VkSampler, 751 sampler->sampler_clamped); 752 } 753 if (sampler->custom_border_color) 754 p_atomic_dec(&zink_screen(pctx->screen)->cur_custom_border_color_samplers); 755 FREE(sampler); 756} 757 758static VkImageAspectFlags 759sampler_aspect_from_format(enum pipe_format fmt) 760{ 761 if (util_format_is_depth_or_stencil(fmt)) { 762 const struct util_format_description *desc = util_format_description(fmt); 763 if (util_format_has_depth(desc)) 764 return VK_IMAGE_ASPECT_DEPTH_BIT; 765 assert(util_format_has_stencil(desc)); 766 return VK_IMAGE_ASPECT_STENCIL_BIT; 767 } else 768 return VK_IMAGE_ASPECT_COLOR_BIT; 769} 770 771static uint32_t 772hash_bufferview(void *bvci) 773{ 774 size_t offset = offsetof(VkBufferViewCreateInfo, flags); 775 return _mesa_hash_data((char*)bvci + offset, sizeof(VkBufferViewCreateInfo) - offset); 776} 777 778static VkBufferViewCreateInfo 779create_bvci(struct zink_context *ctx, struct zink_resource *res, enum pipe_format format, uint32_t offset, uint32_t range) 780{ 781 struct zink_screen *screen = zink_screen(ctx->base.screen); 782 VkBufferViewCreateInfo bvci; 783 // Zero whole struct (including alignment holes), so hash_bufferview 784 // does not access potentially uninitialized data. 785 memset(&bvci, 0, sizeof(bvci)); 786 bvci.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO; 787 bvci.pNext = NULL; 788 if (screen->format_props[format].bufferFeatures & VK_FORMAT_FEATURE_STORAGE_TEXEL_BUFFER_BIT) 789 bvci.buffer = res->obj->storage_buffer ? res->obj->storage_buffer : res->obj->buffer; 790 else 791 bvci.buffer = res->obj->buffer; 792 bvci.format = zink_get_format(screen, format); 793 assert(bvci.format); 794 bvci.offset = offset; 795 bvci.range = !offset && range == res->base.b.width0 ? VK_WHOLE_SIZE : range; 796 unsigned blocksize = util_format_get_blocksize(format); 797 if (bvci.range != VK_WHOLE_SIZE) { 798 /* clamp out partial texels */ 799 bvci.range -= bvci.range % blocksize; 800 if (bvci.offset + bvci.range >= res->base.b.width0) 801 bvci.range = VK_WHOLE_SIZE; 802 } 803 uint64_t clamp = blocksize * screen->info.props.limits.maxTexelBufferElements; 804 if (bvci.range == VK_WHOLE_SIZE && res->base.b.width0 > clamp) 805 bvci.range = clamp; 806 bvci.flags = 0; 807 return bvci; 808} 809 810static struct zink_buffer_view * 811get_buffer_view(struct zink_context *ctx, struct zink_resource *res, VkBufferViewCreateInfo *bvci) 812{ 813 struct zink_screen *screen = zink_screen(ctx->base.screen); 814 struct zink_buffer_view *buffer_view = NULL; 815 816 uint32_t hash = hash_bufferview(bvci); 817 simple_mtx_lock(&res->bufferview_mtx); 818 struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, hash, bvci); 819 if (he) { 820 buffer_view = he->data; 821 p_atomic_inc(&buffer_view->reference.count); 822 } else { 823 VkBufferView view; 824 VkResult result = VKSCR(CreateBufferView)(screen->dev, bvci, NULL, &view); 825 if (result != VK_SUCCESS) { 826 mesa_loge("ZINK: vkCreateBufferView failed (%s)", vk_Result_to_str(result)); 827 goto out; 828 } 829 buffer_view = CALLOC_STRUCT(zink_buffer_view); 830 if (!buffer_view) { 831 VKSCR(DestroyBufferView)(screen->dev, view, NULL); 832 goto out; 833 } 834 pipe_reference_init(&buffer_view->reference, 1); 835 pipe_resource_reference(&buffer_view->pres, &res->base.b); 836 util_dynarray_init(&buffer_view->desc_set_refs.refs, NULL); 837 buffer_view->bvci = *bvci; 838 buffer_view->buffer_view = view; 839 buffer_view->hash = hash; 840 _mesa_hash_table_insert_pre_hashed(&res->bufferview_cache, hash, &buffer_view->bvci, buffer_view); 841 } 842out: 843 simple_mtx_unlock(&res->bufferview_mtx); 844 return buffer_view; 845} 846 847enum pipe_swizzle 848zink_clamp_void_swizzle(const struct util_format_description *desc, enum pipe_swizzle swizzle) 849{ 850 switch (swizzle) { 851 case PIPE_SWIZZLE_X: 852 case PIPE_SWIZZLE_Y: 853 case PIPE_SWIZZLE_Z: 854 case PIPE_SWIZZLE_W: 855 return desc->channel[swizzle].type == UTIL_FORMAT_TYPE_VOID ? PIPE_SWIZZLE_1 : swizzle; 856 default: 857 break; 858 } 859 return swizzle; 860} 861 862ALWAYS_INLINE static enum pipe_swizzle 863clamp_zs_swizzle(enum pipe_swizzle swizzle) 864{ 865 switch (swizzle) { 866 case PIPE_SWIZZLE_X: 867 case PIPE_SWIZZLE_Y: 868 case PIPE_SWIZZLE_Z: 869 case PIPE_SWIZZLE_W: 870 return PIPE_SWIZZLE_X; 871 default: 872 break; 873 } 874 return swizzle; 875} 876 877ALWAYS_INLINE static bool 878viewtype_is_cube(const VkImageViewCreateInfo *ivci) 879{ 880 return ivci->viewType == VK_IMAGE_VIEW_TYPE_CUBE || 881 ivci->viewType == VK_IMAGE_VIEW_TYPE_CUBE_ARRAY; 882} 883 884static struct pipe_sampler_view * 885zink_create_sampler_view(struct pipe_context *pctx, struct pipe_resource *pres, 886 const struct pipe_sampler_view *state) 887{ 888 struct zink_screen *screen = zink_screen(pctx->screen); 889 struct zink_resource *res = zink_resource(pres); 890 struct zink_context *ctx = zink_context(pctx); 891 struct zink_sampler_view *sampler_view = CALLOC_STRUCT_CL(zink_sampler_view); 892 bool err; 893 894 sampler_view->base = *state; 895 sampler_view->base.texture = NULL; 896 pipe_resource_reference(&sampler_view->base.texture, pres); 897 sampler_view->base.reference.count = 1; 898 sampler_view->base.context = pctx; 899 900 if (state->target != PIPE_BUFFER) { 901 VkImageViewCreateInfo ivci; 902 903 struct pipe_surface templ = {0}; 904 templ.u.tex.level = state->u.tex.first_level; 905 templ.format = state->format; 906 if (state->target != PIPE_TEXTURE_3D) { 907 templ.u.tex.first_layer = state->u.tex.first_layer; 908 templ.u.tex.last_layer = state->u.tex.last_layer; 909 } 910 911 if (zink_is_swapchain(res)) { 912 if (!zink_kopper_acquire(ctx, res, UINT64_MAX)) { 913 FREE_CL(sampler_view); 914 return NULL; 915 } 916 } 917 918 ivci = create_ivci(screen, res, &templ, state->target); 919 ivci.subresourceRange.levelCount = state->u.tex.last_level - state->u.tex.first_level + 1; 920 ivci.subresourceRange.aspectMask = sampler_aspect_from_format(state->format); 921 /* samplers for stencil aspects of packed formats need to always use stencil swizzle */ 922 if (ivci.subresourceRange.aspectMask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) { 923 if (sampler_view->base.swizzle_r == PIPE_SWIZZLE_0 && 924 sampler_view->base.swizzle_g == PIPE_SWIZZLE_0 && 925 sampler_view->base.swizzle_b == PIPE_SWIZZLE_0 && 926 sampler_view->base.swizzle_a == PIPE_SWIZZLE_X) { 927 /* 928 * When the state tracker asks for 000x swizzles, this is depth mode GL_ALPHA, 929 * however with the single dref fetch this will fail, so just spam all the channels. 930 */ 931 ivci.components.r = VK_COMPONENT_SWIZZLE_R; 932 ivci.components.g = VK_COMPONENT_SWIZZLE_R; 933 ivci.components.b = VK_COMPONENT_SWIZZLE_R; 934 ivci.components.a = VK_COMPONENT_SWIZZLE_R; 935 } else { 936 ivci.components.r = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_r)); 937 ivci.components.g = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_g)); 938 ivci.components.b = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_b)); 939 ivci.components.a = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_a)); 940 } 941 } else { 942 /* if we have e.g., R8G8B8X8, then we have to ignore alpha since we're just emulating 943 * these formats 944 */ 945 if (zink_format_is_voidable_rgba_variant(state->format)) { 946 const struct util_format_description *desc = util_format_description(state->format); 947 sampler_view->base.swizzle_r = zink_clamp_void_swizzle(desc, sampler_view->base.swizzle_r); 948 sampler_view->base.swizzle_g = zink_clamp_void_swizzle(desc, sampler_view->base.swizzle_g); 949 sampler_view->base.swizzle_b = zink_clamp_void_swizzle(desc, sampler_view->base.swizzle_b); 950 sampler_view->base.swizzle_a = zink_clamp_void_swizzle(desc, sampler_view->base.swizzle_a); 951 } 952 ivci.components.r = zink_component_mapping(sampler_view->base.swizzle_r); 953 ivci.components.g = zink_component_mapping(sampler_view->base.swizzle_g); 954 ivci.components.b = zink_component_mapping(sampler_view->base.swizzle_b); 955 ivci.components.a = zink_component_mapping(sampler_view->base.swizzle_a); 956 } 957 assert(ivci.format); 958 959 sampler_view->image_view = (struct zink_surface*)zink_get_surface(ctx, pres, &templ, &ivci); 960 if (!screen->info.have_EXT_non_seamless_cube_map && viewtype_is_cube(&sampler_view->image_view->ivci)) { 961 ivci.viewType = VK_IMAGE_VIEW_TYPE_2D_ARRAY; 962 sampler_view->cube_array = (struct zink_surface*)zink_get_surface(ctx, pres, &templ, &ivci); 963 } 964 err = !sampler_view->image_view; 965 } else { 966 VkBufferViewCreateInfo bvci = create_bvci(ctx, res, state->format, state->u.buf.offset, state->u.buf.size); 967 sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci); 968 err = !sampler_view->buffer_view; 969 } 970 if (err) { 971 FREE_CL(sampler_view); 972 return NULL; 973 } 974 return &sampler_view->base; 975} 976 977void 978zink_destroy_buffer_view(struct zink_screen *screen, struct zink_buffer_view *buffer_view) 979{ 980 struct zink_resource *res = zink_resource(buffer_view->pres); 981 simple_mtx_lock(&res->bufferview_mtx); 982 if (buffer_view->reference.count) { 983 /* got a cache hit during deletion */ 984 simple_mtx_unlock(&res->bufferview_mtx); 985 return; 986 } 987 struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, buffer_view->hash, &buffer_view->bvci); 988 assert(he); 989 _mesa_hash_table_remove(&res->bufferview_cache, he); 990 simple_mtx_unlock(&res->bufferview_mtx); 991 pipe_resource_reference(&buffer_view->pres, NULL); 992 VKSCR(DestroyBufferView)(screen->dev, buffer_view->buffer_view, NULL); 993 zink_descriptor_set_refs_clear(&buffer_view->desc_set_refs, buffer_view); 994 FREE(buffer_view); 995} 996 997static void 998zink_sampler_view_destroy(struct pipe_context *pctx, 999 struct pipe_sampler_view *pview) 1000{ 1001 struct zink_sampler_view *view = zink_sampler_view(pview); 1002 if (pview->texture->target == PIPE_BUFFER) 1003 zink_buffer_view_reference(zink_screen(pctx->screen), &view->buffer_view, NULL); 1004 else { 1005 zink_surface_reference(zink_screen(pctx->screen), &view->image_view, NULL); 1006 zink_surface_reference(zink_screen(pctx->screen), &view->cube_array, NULL); 1007 } 1008 pipe_resource_reference(&pview->texture, NULL); 1009 FREE_CL(view); 1010} 1011 1012static void 1013zink_get_sample_position(struct pipe_context *ctx, 1014 unsigned sample_count, 1015 unsigned sample_index, 1016 float *out_value) 1017{ 1018 /* TODO: handle this I guess */ 1019 assert(zink_screen(ctx->screen)->info.props.limits.standardSampleLocations); 1020 /* from 26.4. Multisampling */ 1021 switch (sample_count) { 1022 case 0: 1023 case 1: { 1024 float pos[][2] = { {0.5,0.5}, }; 1025 out_value[0] = pos[sample_index][0]; 1026 out_value[1] = pos[sample_index][1]; 1027 break; 1028 } 1029 case 2: { 1030 float pos[][2] = { {0.75,0.75}, 1031 {0.25,0.25}, }; 1032 out_value[0] = pos[sample_index][0]; 1033 out_value[1] = pos[sample_index][1]; 1034 break; 1035 } 1036 case 4: { 1037 float pos[][2] = { {0.375, 0.125}, 1038 {0.875, 0.375}, 1039 {0.125, 0.625}, 1040 {0.625, 0.875}, }; 1041 out_value[0] = pos[sample_index][0]; 1042 out_value[1] = pos[sample_index][1]; 1043 break; 1044 } 1045 case 8: { 1046 float pos[][2] = { {0.5625, 0.3125}, 1047 {0.4375, 0.6875}, 1048 {0.8125, 0.5625}, 1049 {0.3125, 0.1875}, 1050 {0.1875, 0.8125}, 1051 {0.0625, 0.4375}, 1052 {0.6875, 0.9375}, 1053 {0.9375, 0.0625}, }; 1054 out_value[0] = pos[sample_index][0]; 1055 out_value[1] = pos[sample_index][1]; 1056 break; 1057 } 1058 case 16: { 1059 float pos[][2] = { {0.5625, 0.5625}, 1060 {0.4375, 0.3125}, 1061 {0.3125, 0.625}, 1062 {0.75, 0.4375}, 1063 {0.1875, 0.375}, 1064 {0.625, 0.8125}, 1065 {0.8125, 0.6875}, 1066 {0.6875, 0.1875}, 1067 {0.375, 0.875}, 1068 {0.5, 0.0625}, 1069 {0.25, 0.125}, 1070 {0.125, 0.75}, 1071 {0.0, 0.5}, 1072 {0.9375, 0.25}, 1073 {0.875, 0.9375}, 1074 {0.0625, 0.0}, }; 1075 out_value[0] = pos[sample_index][0]; 1076 out_value[1] = pos[sample_index][1]; 1077 break; 1078 } 1079 default: 1080 unreachable("unhandled sample count!"); 1081 } 1082} 1083 1084static void 1085zink_set_polygon_stipple(struct pipe_context *pctx, 1086 const struct pipe_poly_stipple *ps) 1087{ 1088} 1089 1090ALWAYS_INLINE static void 1091update_res_bind_count(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool decrement) 1092{ 1093 if (decrement) { 1094 assert(res->bind_count[is_compute]); 1095 if (!--res->bind_count[is_compute]) 1096 _mesa_set_remove_key(ctx->need_barriers[is_compute], res); 1097 check_resource_for_batch_ref(ctx, res); 1098 } else 1099 res->bind_count[is_compute]++; 1100} 1101 1102ALWAYS_INLINE static void 1103update_existing_vbo(struct zink_context *ctx, unsigned slot) 1104{ 1105 if (!ctx->vertex_buffers[slot].buffer.resource) 1106 return; 1107 struct zink_resource *res = zink_resource(ctx->vertex_buffers[slot].buffer.resource); 1108 res->vbo_bind_count--; 1109 res->vbo_bind_mask &= ~BITFIELD_BIT(slot); 1110 if (!res->vbo_bind_count) { 1111 res->gfx_barrier &= ~VK_PIPELINE_STAGE_VERTEX_INPUT_BIT; 1112 res->barrier_access[0] &= ~VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT; 1113 } 1114 update_res_bind_count(ctx, res, false, true); 1115} 1116 1117static void 1118zink_set_vertex_buffers(struct pipe_context *pctx, 1119 unsigned start_slot, 1120 unsigned num_buffers, 1121 unsigned unbind_num_trailing_slots, 1122 bool take_ownership, 1123 const struct pipe_vertex_buffer *buffers) 1124{ 1125 struct zink_context *ctx = zink_context(pctx); 1126 const bool have_input_state = zink_screen(pctx->screen)->info.have_EXT_vertex_input_dynamic_state; 1127 const bool need_state_change = !zink_screen(pctx->screen)->info.have_EXT_extended_dynamic_state && 1128 !have_input_state; 1129 uint32_t enabled_buffers = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask; 1130 enabled_buffers |= u_bit_consecutive(start_slot, num_buffers); 1131 enabled_buffers &= ~u_bit_consecutive(start_slot + num_buffers, unbind_num_trailing_slots); 1132 bool stride_changed = false; 1133 1134 if (buffers) { 1135 for (unsigned i = 0; i < num_buffers; ++i) { 1136 const struct pipe_vertex_buffer *vb = buffers + i; 1137 struct pipe_vertex_buffer *ctx_vb = &ctx->vertex_buffers[start_slot + i]; 1138 stride_changed |= ctx_vb->stride != vb->stride; 1139 update_existing_vbo(ctx, start_slot + i); 1140 if (!take_ownership) 1141 pipe_resource_reference(&ctx_vb->buffer.resource, vb->buffer.resource); 1142 else { 1143 pipe_resource_reference(&ctx_vb->buffer.resource, NULL); 1144 ctx_vb->buffer.resource = vb->buffer.resource; 1145 } 1146 if (vb->buffer.resource) { 1147 struct zink_resource *res = zink_resource(vb->buffer.resource); 1148 res->vbo_bind_mask |= BITFIELD_BIT(start_slot + i); 1149 res->vbo_bind_count++; 1150 res->gfx_barrier |= VK_PIPELINE_STAGE_VERTEX_INPUT_BIT; 1151 res->barrier_access[0] |= VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT; 1152 update_res_bind_count(ctx, res, false, false); 1153 ctx_vb->stride = vb->stride; 1154 ctx_vb->buffer_offset = vb->buffer_offset; 1155 zink_batch_resource_usage_set(&ctx->batch, res, false); 1156 /* always barrier before possible rebind */ 1157 zink_resource_buffer_barrier(ctx, res, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT, 1158 VK_PIPELINE_STAGE_VERTEX_INPUT_BIT); 1159 res->obj->unordered_read = false; 1160 } else { 1161 enabled_buffers &= ~BITFIELD_BIT(start_slot + i); 1162 } 1163 } 1164 } else { 1165 for (unsigned i = 0; i < num_buffers; ++i) { 1166 update_existing_vbo(ctx, start_slot + i); 1167 pipe_resource_reference(&ctx->vertex_buffers[start_slot + i].buffer.resource, NULL); 1168 } 1169 } 1170 for (unsigned i = 0; i < unbind_num_trailing_slots; i++) { 1171 update_existing_vbo(ctx, start_slot + i); 1172 pipe_resource_reference(&ctx->vertex_buffers[start_slot + i].buffer.resource, NULL); 1173 } 1174 if (need_state_change) 1175 ctx->vertex_state_changed = true; 1176 else if (!have_input_state && (stride_changed || ctx->gfx_pipeline_state.vertex_buffers_enabled_mask != enabled_buffers)) 1177 ctx->vertex_state_changed = true; 1178 ctx->gfx_pipeline_state.vertex_buffers_enabled_mask = enabled_buffers; 1179 ctx->vertex_buffers_dirty = num_buffers > 0; 1180#ifndef NDEBUG 1181 u_foreach_bit(b, enabled_buffers) 1182 assert(ctx->vertex_buffers[b].buffer.resource); 1183#endif 1184} 1185 1186static void 1187zink_set_viewport_states(struct pipe_context *pctx, 1188 unsigned start_slot, 1189 unsigned num_viewports, 1190 const struct pipe_viewport_state *state) 1191{ 1192 struct zink_context *ctx = zink_context(pctx); 1193 1194 for (unsigned i = 0; i < num_viewports; ++i) 1195 ctx->vp_state.viewport_states[start_slot + i] = state[i]; 1196 1197 ctx->vp_state_changed = true; 1198} 1199 1200static void 1201zink_set_scissor_states(struct pipe_context *pctx, 1202 unsigned start_slot, unsigned num_scissors, 1203 const struct pipe_scissor_state *states) 1204{ 1205 struct zink_context *ctx = zink_context(pctx); 1206 1207 for (unsigned i = 0; i < num_scissors; i++) 1208 ctx->vp_state.scissor_states[start_slot + i] = states[i]; 1209 ctx->scissor_changed = true; 1210} 1211 1212static void 1213zink_set_inlinable_constants(struct pipe_context *pctx, 1214 enum pipe_shader_type shader, 1215 uint num_values, uint32_t *values) 1216{ 1217 struct zink_context *ctx = (struct zink_context *)pctx; 1218 const uint32_t bit = BITFIELD_BIT(shader); 1219 uint32_t *inlinable_uniforms; 1220 struct zink_shader_key *key = NULL; 1221 1222 if (shader == PIPE_SHADER_COMPUTE) { 1223 key = &ctx->compute_pipeline_state.key; 1224 } else { 1225 key = &ctx->gfx_pipeline_state.shader_keys.key[shader]; 1226 } 1227 inlinable_uniforms = key->base.inlined_uniform_values; 1228 if (!(ctx->inlinable_uniforms_valid_mask & bit) || 1229 memcmp(inlinable_uniforms, values, num_values * 4)) { 1230 memcpy(inlinable_uniforms, values, num_values * 4); 1231 ctx->dirty_shader_stages |= bit; 1232 ctx->inlinable_uniforms_valid_mask |= bit; 1233 key->inline_uniforms = true; 1234 } 1235} 1236 1237ALWAYS_INLINE static void 1238unbind_descriptor_stage(struct zink_resource *res, enum pipe_shader_type pstage) 1239{ 1240 if (!res->sampler_binds[pstage] && !res->image_binds[pstage]) 1241 res->gfx_barrier &= ~zink_pipeline_flags_from_pipe_stage(pstage); 1242} 1243 1244ALWAYS_INLINE static void 1245unbind_buffer_descriptor_stage(struct zink_resource *res, enum pipe_shader_type pstage) 1246{ 1247 if (!res->ubo_bind_mask[pstage] && !res->ssbo_bind_mask[pstage]) 1248 unbind_descriptor_stage(res, pstage); 1249} 1250 1251ALWAYS_INLINE static void 1252unbind_ubo(struct zink_context *ctx, struct zink_resource *res, enum pipe_shader_type pstage, unsigned slot) 1253{ 1254 if (!res) 1255 return; 1256 res->ubo_bind_mask[pstage] &= ~BITFIELD_BIT(slot); 1257 res->ubo_bind_count[pstage == PIPE_SHADER_COMPUTE]--; 1258 unbind_buffer_descriptor_stage(res, pstage); 1259 if (!res->ubo_bind_count[pstage == PIPE_SHADER_COMPUTE]) 1260 res->barrier_access[pstage == PIPE_SHADER_COMPUTE] &= ~VK_ACCESS_UNIFORM_READ_BIT; 1261 update_res_bind_count(ctx, res, pstage == PIPE_SHADER_COMPUTE, true); 1262} 1263 1264static void 1265invalidate_inlined_uniforms(struct zink_context *ctx, enum pipe_shader_type pstage) 1266{ 1267 unsigned bit = BITFIELD_BIT(pstage); 1268 if (!(ctx->inlinable_uniforms_valid_mask & bit)) 1269 return; 1270 ctx->inlinable_uniforms_valid_mask &= ~bit; 1271 ctx->dirty_shader_stages |= bit; 1272 if (pstage == PIPE_SHADER_COMPUTE) 1273 return; 1274 1275 struct zink_shader_key *key = &ctx->gfx_pipeline_state.shader_keys.key[pstage]; 1276 key->inline_uniforms = false; 1277} 1278 1279static void 1280zink_set_constant_buffer(struct pipe_context *pctx, 1281 enum pipe_shader_type shader, uint index, 1282 bool take_ownership, 1283 const struct pipe_constant_buffer *cb) 1284{ 1285 struct zink_context *ctx = zink_context(pctx); 1286 bool update = false; 1287 1288 struct zink_resource *res = zink_resource(ctx->ubos[shader][index].buffer); 1289 if (cb) { 1290 struct pipe_resource *buffer = cb->buffer; 1291 unsigned offset = cb->buffer_offset; 1292 struct zink_screen *screen = zink_screen(pctx->screen); 1293 if (cb->user_buffer) { 1294 u_upload_data(ctx->base.const_uploader, 0, cb->buffer_size, 1295 screen->info.props.limits.minUniformBufferOffsetAlignment, 1296 cb->user_buffer, &offset, &buffer); 1297 } 1298 struct zink_resource *new_res = zink_resource(buffer); 1299 if (new_res) { 1300 if (new_res != res) { 1301 unbind_ubo(ctx, res, shader, index); 1302 new_res->ubo_bind_count[shader == PIPE_SHADER_COMPUTE]++; 1303 new_res->ubo_bind_mask[shader] |= BITFIELD_BIT(index); 1304 new_res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader); 1305 new_res->barrier_access[shader == PIPE_SHADER_COMPUTE] |= VK_ACCESS_UNIFORM_READ_BIT; 1306 update_res_bind_count(ctx, new_res, shader == PIPE_SHADER_COMPUTE, false); 1307 } 1308 zink_batch_resource_usage_set(&ctx->batch, new_res, false); 1309 zink_resource_buffer_barrier(ctx, new_res, VK_ACCESS_UNIFORM_READ_BIT, 1310 new_res->gfx_barrier); 1311 new_res->obj->unordered_read = false; 1312 } 1313 update |= ((index || zink_descriptor_mode == ZINK_DESCRIPTOR_MODE_LAZY) && ctx->ubos[shader][index].buffer_offset != offset) || 1314 !!res != !!buffer || (res && res->obj->buffer != new_res->obj->buffer) || 1315 ctx->ubos[shader][index].buffer_size != cb->buffer_size; 1316 1317 if (take_ownership) { 1318 pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL); 1319 ctx->ubos[shader][index].buffer = buffer; 1320 } else { 1321 pipe_resource_reference(&ctx->ubos[shader][index].buffer, buffer); 1322 } 1323 ctx->ubos[shader][index].buffer_offset = offset; 1324 ctx->ubos[shader][index].buffer_size = cb->buffer_size; 1325 ctx->ubos[shader][index].user_buffer = NULL; 1326 1327 if (cb->user_buffer) 1328 pipe_resource_reference(&buffer, NULL); 1329 1330 if (index + 1 >= ctx->di.num_ubos[shader]) 1331 ctx->di.num_ubos[shader] = index + 1; 1332 update_descriptor_state_ubo(ctx, shader, index, new_res); 1333 } else { 1334 ctx->ubos[shader][index].buffer_offset = 0; 1335 ctx->ubos[shader][index].buffer_size = 0; 1336 ctx->ubos[shader][index].user_buffer = NULL; 1337 if (res) { 1338 unbind_ubo(ctx, res, shader, index); 1339 update_descriptor_state_ubo(ctx, shader, index, NULL); 1340 } 1341 update = !!ctx->ubos[shader][index].buffer; 1342 1343 pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL); 1344 if (ctx->di.num_ubos[shader] == index + 1) 1345 ctx->di.num_ubos[shader]--; 1346 } 1347 if (index == 0) { 1348 /* Invalidate current inlinable uniforms. */ 1349 invalidate_inlined_uniforms(ctx, shader); 1350 } 1351 1352 if (update) 1353 zink_screen(pctx->screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, index, 1); 1354} 1355 1356ALWAYS_INLINE static void 1357unbind_descriptor_reads(struct zink_resource *res, enum pipe_shader_type pstage) 1358{ 1359 if (!res->sampler_binds[pstage] && !res->image_binds[pstage]) 1360 res->barrier_access[pstage == PIPE_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_READ_BIT; 1361} 1362 1363ALWAYS_INLINE static void 1364unbind_buffer_descriptor_reads(struct zink_resource *res, enum pipe_shader_type pstage) 1365{ 1366 if (!res->ssbo_bind_count[pstage == PIPE_SHADER_COMPUTE]) 1367 unbind_descriptor_reads(res, pstage); 1368} 1369 1370ALWAYS_INLINE static void 1371unbind_ssbo(struct zink_context *ctx, struct zink_resource *res, enum pipe_shader_type pstage, unsigned slot, bool writable) 1372{ 1373 if (!res) 1374 return; 1375 res->ssbo_bind_mask[pstage] &= ~BITFIELD_BIT(slot); 1376 res->ssbo_bind_count[pstage == PIPE_SHADER_COMPUTE]--; 1377 unbind_buffer_descriptor_stage(res, pstage); 1378 unbind_buffer_descriptor_reads(res, pstage); 1379 update_res_bind_count(ctx, res, pstage == PIPE_SHADER_COMPUTE, true); 1380 if (writable) 1381 res->write_bind_count[pstage == PIPE_SHADER_COMPUTE]--; 1382 if (!res->write_bind_count[pstage == PIPE_SHADER_COMPUTE]) 1383 res->barrier_access[pstage == PIPE_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_WRITE_BIT; 1384} 1385 1386static void 1387zink_set_shader_buffers(struct pipe_context *pctx, 1388 enum pipe_shader_type p_stage, 1389 unsigned start_slot, unsigned count, 1390 const struct pipe_shader_buffer *buffers, 1391 unsigned writable_bitmask) 1392{ 1393 struct zink_context *ctx = zink_context(pctx); 1394 bool update = false; 1395 unsigned max_slot = 0; 1396 1397 unsigned modified_bits = u_bit_consecutive(start_slot, count); 1398 unsigned old_writable_mask = ctx->writable_ssbos[p_stage]; 1399 ctx->writable_ssbos[p_stage] &= ~modified_bits; 1400 ctx->writable_ssbos[p_stage] |= writable_bitmask << start_slot; 1401 1402 for (unsigned i = 0; i < count; i++) { 1403 struct pipe_shader_buffer *ssbo = &ctx->ssbos[p_stage][start_slot + i]; 1404 struct zink_resource *res = ssbo->buffer ? zink_resource(ssbo->buffer) : NULL; 1405 bool was_writable = old_writable_mask & BITFIELD64_BIT(start_slot + i); 1406 if (buffers && buffers[i].buffer) { 1407 struct zink_resource *new_res = zink_resource(buffers[i].buffer); 1408 if (new_res != res) { 1409 unbind_ssbo(ctx, res, p_stage, i, was_writable); 1410 new_res->ssbo_bind_mask[p_stage] |= BITFIELD_BIT(i); 1411 new_res->ssbo_bind_count[p_stage == PIPE_SHADER_COMPUTE]++; 1412 new_res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(p_stage); 1413 update_res_bind_count(ctx, new_res, p_stage == PIPE_SHADER_COMPUTE, false); 1414 } 1415 VkAccessFlags access = VK_ACCESS_SHADER_READ_BIT; 1416 if (ctx->writable_ssbos[p_stage] & BITFIELD64_BIT(start_slot + i)) { 1417 new_res->write_bind_count[p_stage == PIPE_SHADER_COMPUTE]++; 1418 access |= VK_ACCESS_SHADER_WRITE_BIT; 1419 } 1420 pipe_resource_reference(&ssbo->buffer, &new_res->base.b); 1421 new_res->barrier_access[p_stage == PIPE_SHADER_COMPUTE] |= access; 1422 zink_batch_resource_usage_set(&ctx->batch, new_res, access & VK_ACCESS_SHADER_WRITE_BIT); 1423 ssbo->buffer_offset = buffers[i].buffer_offset; 1424 ssbo->buffer_size = MIN2(buffers[i].buffer_size, new_res->base.b.width0 - ssbo->buffer_offset); 1425 util_range_add(&new_res->base.b, &new_res->valid_buffer_range, ssbo->buffer_offset, 1426 ssbo->buffer_offset + ssbo->buffer_size); 1427 zink_resource_buffer_barrier(ctx, new_res, access, 1428 new_res->gfx_barrier); 1429 update = true; 1430 max_slot = MAX2(max_slot, start_slot + i); 1431 update_descriptor_state_ssbo(ctx, p_stage, start_slot + i, new_res); 1432 if (zink_resource_access_is_write(access)) 1433 new_res->obj->unordered_read = new_res->obj->unordered_write = false; 1434 else 1435 new_res->obj->unordered_read = false; 1436 } else { 1437 update = !!res; 1438 ssbo->buffer_offset = 0; 1439 ssbo->buffer_size = 0; 1440 if (res) { 1441 unbind_ssbo(ctx, res, p_stage, i, was_writable); 1442 update_descriptor_state_ssbo(ctx, p_stage, start_slot + i, NULL); 1443 } 1444 pipe_resource_reference(&ssbo->buffer, NULL); 1445 } 1446 } 1447 if (start_slot + count >= ctx->di.num_ssbos[p_stage]) 1448 ctx->di.num_ssbos[p_stage] = max_slot + 1; 1449 if (update) 1450 zink_screen(pctx->screen)->context_invalidate_descriptor_state(ctx, p_stage, ZINK_DESCRIPTOR_TYPE_SSBO, start_slot, count); 1451} 1452 1453static void 1454update_binds_for_samplerviews(struct zink_context *ctx, struct zink_resource *res, bool is_compute) 1455{ 1456 VkImageLayout layout = get_layout_for_binding(ctx, res, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, is_compute); 1457 if (is_compute) { 1458 u_foreach_bit(slot, res->sampler_binds[PIPE_SHADER_COMPUTE]) { 1459 if (ctx->di.textures[PIPE_SHADER_COMPUTE][slot].imageLayout != layout) { 1460 update_descriptor_state_sampler(ctx, PIPE_SHADER_COMPUTE, slot, res); 1461 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, PIPE_SHADER_COMPUTE, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1); 1462 } 1463 } 1464 } else { 1465 for (unsigned i = 0; i < ZINK_SHADER_COUNT; i++) { 1466 u_foreach_bit(slot, res->sampler_binds[i]) { 1467 if (ctx->di.textures[i][slot].imageLayout != layout) { 1468 update_descriptor_state_sampler(ctx, i, slot, res); 1469 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1); 1470 } 1471 } 1472 } 1473 } 1474} 1475 1476static void 1477flush_pending_clears(struct zink_context *ctx, struct zink_resource *res) 1478{ 1479 if (res->fb_binds && ctx->clears_enabled) 1480 zink_fb_clears_apply(ctx, &res->base.b); 1481} 1482 1483static inline void 1484unbind_shader_image_counts(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool writable) 1485{ 1486 update_res_bind_count(ctx, res, is_compute, true); 1487 if (writable) 1488 res->write_bind_count[is_compute]--; 1489 res->image_bind_count[is_compute]--; 1490 /* if this was the last image bind, the sampler bind layouts must be updated */ 1491 if (!res->obj->is_buffer && !res->image_bind_count[is_compute] && res->bind_count[is_compute]) 1492 update_binds_for_samplerviews(ctx, res, is_compute); 1493} 1494 1495ALWAYS_INLINE static void 1496check_for_layout_update(struct zink_context *ctx, struct zink_resource *res, bool is_compute) 1497{ 1498 VkImageLayout layout = res->bind_count[is_compute] ? zink_descriptor_util_image_layout_eval(ctx, res, is_compute) : VK_IMAGE_LAYOUT_UNDEFINED; 1499 VkImageLayout other_layout = res->bind_count[!is_compute] ? zink_descriptor_util_image_layout_eval(ctx, res, !is_compute) : VK_IMAGE_LAYOUT_UNDEFINED; 1500 if (res->bind_count[is_compute] && layout && res->layout != layout) 1501 _mesa_set_add(ctx->need_barriers[is_compute], res); 1502 if (res->bind_count[!is_compute] && other_layout && (layout != other_layout || res->layout != other_layout)) 1503 _mesa_set_add(ctx->need_barriers[!is_compute], res); 1504} 1505 1506static void 1507unbind_shader_image(struct zink_context *ctx, enum pipe_shader_type stage, unsigned slot) 1508{ 1509 struct zink_image_view *image_view = &ctx->image_views[stage][slot]; 1510 bool is_compute = stage == PIPE_SHADER_COMPUTE; 1511 if (!image_view->base.resource) 1512 return; 1513 1514 struct zink_resource *res = zink_resource(image_view->base.resource); 1515 res->image_binds[stage] &= ~BITFIELD_BIT(slot); 1516 unbind_shader_image_counts(ctx, res, is_compute, image_view->base.access & PIPE_IMAGE_ACCESS_WRITE); 1517 if (!res->write_bind_count[is_compute]) 1518 res->barrier_access[stage == PIPE_SHADER_COMPUTE] &= ~VK_ACCESS_SHADER_WRITE_BIT; 1519 1520 if (image_view->base.resource->target == PIPE_BUFFER) { 1521 unbind_buffer_descriptor_stage(res, stage); 1522 unbind_buffer_descriptor_reads(res, stage); 1523 if (zink_batch_usage_exists(image_view->buffer_view->batch_uses)) 1524 zink_batch_reference_bufferview(&ctx->batch, image_view->buffer_view); 1525 zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL); 1526 } else { 1527 unbind_descriptor_stage(res, stage); 1528 if (!res->image_bind_count[is_compute]) 1529 check_for_layout_update(ctx, res, is_compute); 1530 if (zink_batch_usage_exists(image_view->surface->batch_uses)) 1531 zink_batch_reference_surface(&ctx->batch, image_view->surface); 1532 zink_surface_reference(zink_screen(ctx->base.screen), &image_view->surface, NULL); 1533 } 1534 image_view->base.resource = NULL; 1535 image_view->surface = NULL; 1536} 1537 1538static struct zink_buffer_view * 1539create_image_bufferview(struct zink_context *ctx, const struct pipe_image_view *view) 1540{ 1541 struct zink_resource *res = zink_resource(view->resource); 1542 VkBufferViewCreateInfo bvci = create_bvci(ctx, res, view->format, view->u.buf.offset, view->u.buf.size); 1543 struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci); 1544 if (!buffer_view) 1545 return NULL; 1546 util_range_add(&res->base.b, &res->valid_buffer_range, view->u.buf.offset, 1547 view->u.buf.offset + view->u.buf.size); 1548 return buffer_view; 1549} 1550 1551static void 1552finalize_image_bind(struct zink_context *ctx, struct zink_resource *res, bool is_compute) 1553{ 1554 /* if this is the first image bind and there are sampler binds, the image's sampler layout 1555 * must be updated to GENERAL 1556 */ 1557 if (res->image_bind_count[is_compute] == 1 && 1558 res->bind_count[is_compute] > 1) 1559 update_binds_for_samplerviews(ctx, res, is_compute); 1560 check_for_layout_update(ctx, res, is_compute); 1561} 1562 1563static struct zink_surface * 1564create_image_surface(struct zink_context *ctx, const struct pipe_image_view *view, bool is_compute) 1565{ 1566 struct zink_screen *screen = zink_screen(ctx->base.screen); 1567 struct zink_resource *res = zink_resource(view->resource); 1568 struct pipe_surface tmpl = {0}; 1569 enum pipe_texture_target target = res->base.b.target; 1570 tmpl.format = view->format; 1571 tmpl.u.tex.level = view->u.tex.level; 1572 tmpl.u.tex.first_layer = view->u.tex.first_layer; 1573 tmpl.u.tex.last_layer = view->u.tex.last_layer; 1574 unsigned depth = 1 + tmpl.u.tex.last_layer - tmpl.u.tex.first_layer; 1575 switch (target) { 1576 case PIPE_TEXTURE_3D: 1577 if (depth < u_minify(res->base.b.depth0, view->u.tex.level)) { 1578 assert(depth == 1); 1579 target = PIPE_TEXTURE_2D; 1580 if (!screen->info.have_EXT_image_2d_view_of_3d || 1581 !screen->info.view2d_feats.image2DViewOf3D) { 1582 static bool warned = false; 1583 warn_missing_feature(warned, "image2DViewOf3D"); 1584 } 1585 } else { 1586 assert(tmpl.u.tex.first_layer == 0); 1587 tmpl.u.tex.last_layer = 0; 1588 } 1589 break; 1590 case PIPE_TEXTURE_2D_ARRAY: 1591 case PIPE_TEXTURE_1D_ARRAY: 1592 if (depth < res->base.b.array_size && depth == 1) 1593 target = target == PIPE_TEXTURE_2D_ARRAY ? PIPE_TEXTURE_2D : PIPE_TEXTURE_1D; 1594 break; 1595 default: break; 1596 } 1597 VkImageViewCreateInfo ivci = create_ivci(screen, res, &tmpl, target); 1598 struct pipe_surface *psurf = zink_get_surface(ctx, view->resource, &tmpl, &ivci); 1599 if (!psurf) 1600 return NULL; 1601 struct zink_surface *surface = zink_surface(psurf); 1602 if (is_compute) 1603 flush_pending_clears(ctx, res); 1604 return surface; 1605} 1606 1607static void 1608zink_set_shader_images(struct pipe_context *pctx, 1609 enum pipe_shader_type p_stage, 1610 unsigned start_slot, unsigned count, 1611 unsigned unbind_num_trailing_slots, 1612 const struct pipe_image_view *images) 1613{ 1614 struct zink_context *ctx = zink_context(pctx); 1615 bool update = false; 1616 for (unsigned i = 0; i < count; i++) { 1617 struct zink_image_view *image_view = &ctx->image_views[p_stage][start_slot + i]; 1618 if (images && images[i].resource) { 1619 struct zink_resource *res = zink_resource(images[i].resource); 1620 if (!zink_resource_object_init_storage(ctx, res)) { 1621 debug_printf("couldn't create storage image!"); 1622 continue; 1623 } 1624 /* no refs */ 1625 VkAccessFlags access = 0; 1626 if (images[i].access & PIPE_IMAGE_ACCESS_WRITE) { 1627 res->write_bind_count[p_stage == PIPE_SHADER_COMPUTE]++; 1628 access |= VK_ACCESS_SHADER_WRITE_BIT; 1629 } 1630 if (images[i].access & PIPE_IMAGE_ACCESS_READ) { 1631 access |= VK_ACCESS_SHADER_READ_BIT; 1632 } 1633 res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(p_stage); 1634 res->barrier_access[p_stage == PIPE_SHADER_COMPUTE] |= access; 1635 if (images[i].resource->target == PIPE_BUFFER) { 1636 struct zink_buffer_view *bv = create_image_bufferview(ctx, &images[i]); 1637 assert(bv); 1638 if (image_view->buffer_view != bv) { 1639 update_res_bind_count(ctx, res, p_stage == PIPE_SHADER_COMPUTE, false); 1640 res->image_bind_count[p_stage == PIPE_SHADER_COMPUTE]++; 1641 unbind_shader_image(ctx, p_stage, start_slot + i); 1642 } 1643 image_view->buffer_view = bv; 1644 zink_batch_usage_set(&image_view->buffer_view->batch_uses, ctx->batch.state); 1645 zink_resource_buffer_barrier(ctx, res, access, 1646 res->gfx_barrier); 1647 } else { 1648 struct zink_surface *surface = create_image_surface(ctx, &images[i], p_stage == PIPE_SHADER_COMPUTE); 1649 assert(surface); 1650 if (image_view->surface != surface) { 1651 res->image_bind_count[p_stage == PIPE_SHADER_COMPUTE]++; 1652 update_res_bind_count(ctx, res, p_stage == PIPE_SHADER_COMPUTE, false); 1653 unbind_shader_image(ctx, p_stage, start_slot + i); 1654 } 1655 image_view->surface = surface; 1656 finalize_image_bind(ctx, res, p_stage == PIPE_SHADER_COMPUTE); 1657 zink_batch_usage_set(&image_view->surface->batch_uses, ctx->batch.state); 1658 } 1659 memcpy(&image_view->base, images + i, sizeof(struct pipe_image_view)); 1660 zink_batch_resource_usage_set(&ctx->batch, res, 1661 zink_resource_access_is_write(access)); 1662 update = true; 1663 update_descriptor_state_image(ctx, p_stage, start_slot + i, res); 1664 if (zink_resource_access_is_write(access)) 1665 res->obj->unordered_read = res->obj->unordered_write = false; 1666 else 1667 res->obj->unordered_read = false; 1668 res->image_binds[p_stage] |= BITFIELD_BIT(start_slot + i); 1669 } else if (image_view->base.resource) { 1670 update = true; 1671 1672 unbind_shader_image(ctx, p_stage, start_slot + i); 1673 update_descriptor_state_image(ctx, p_stage, start_slot + i, NULL); 1674 } 1675 } 1676 for (unsigned i = 0; i < unbind_num_trailing_slots; i++) { 1677 update |= !!ctx->image_views[p_stage][start_slot + count + i].base.resource; 1678 unbind_shader_image(ctx, p_stage, start_slot + count + i); 1679 update_descriptor_state_image(ctx, p_stage, start_slot + count + i, NULL); 1680 } 1681 ctx->di.num_images[p_stage] = start_slot + count; 1682 if (update) 1683 zink_screen(pctx->screen)->context_invalidate_descriptor_state(ctx, p_stage, ZINK_DESCRIPTOR_TYPE_IMAGE, start_slot, count); 1684} 1685 1686ALWAYS_INLINE static void 1687check_samplerview_for_batch_ref(struct zink_context *ctx, struct zink_sampler_view *sv) 1688{ 1689 const struct zink_resource *res = zink_resource(sv->base.texture); 1690 if ((res->obj->is_buffer && zink_batch_usage_exists(sv->buffer_view->batch_uses)) || 1691 (!res->obj->is_buffer && zink_batch_usage_exists(sv->image_view->batch_uses))) 1692 zink_batch_reference_sampler_view(&ctx->batch, sv); 1693} 1694 1695ALWAYS_INLINE static void 1696unbind_samplerview(struct zink_context *ctx, enum pipe_shader_type stage, unsigned slot) 1697{ 1698 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[stage][slot]); 1699 if (!sv || !sv->base.texture) 1700 return; 1701 struct zink_resource *res = zink_resource(sv->base.texture); 1702 res->sampler_bind_count[stage == PIPE_SHADER_COMPUTE]--; 1703 check_samplerview_for_batch_ref(ctx, sv); 1704 update_res_bind_count(ctx, res, stage == PIPE_SHADER_COMPUTE, true); 1705 res->sampler_binds[stage] &= ~BITFIELD_BIT(slot); 1706 if (res->obj->is_buffer) { 1707 unbind_buffer_descriptor_stage(res, stage); 1708 unbind_buffer_descriptor_reads(res, stage); 1709 } else { 1710 unbind_descriptor_stage(res, stage); 1711 unbind_descriptor_reads(res, stage); 1712 } 1713} 1714 1715static void 1716zink_set_sampler_views(struct pipe_context *pctx, 1717 enum pipe_shader_type shader_type, 1718 unsigned start_slot, 1719 unsigned num_views, 1720 unsigned unbind_num_trailing_slots, 1721 bool take_ownership, 1722 struct pipe_sampler_view **views) 1723{ 1724 struct zink_context *ctx = zink_context(pctx); 1725 unsigned i; 1726 1727 const uint32_t mask = BITFIELD_RANGE(start_slot, num_views); 1728 ctx->di.cubes[shader_type] &= ~mask; 1729 1730 bool update = false; 1731 for (i = 0; i < num_views; ++i) { 1732 struct pipe_sampler_view *pview = views ? views[i] : NULL; 1733 struct zink_sampler_view *a = zink_sampler_view(ctx->sampler_views[shader_type][start_slot + i]); 1734 struct zink_sampler_view *b = zink_sampler_view(pview); 1735 struct zink_resource *res = b ? zink_resource(b->base.texture) : NULL; 1736 if (b && b->base.texture) { 1737 if (!a || zink_resource(a->base.texture) != res) { 1738 if (a) 1739 unbind_samplerview(ctx, shader_type, start_slot + i); 1740 update_res_bind_count(ctx, res, shader_type == PIPE_SHADER_COMPUTE, false); 1741 res->sampler_bind_count[shader_type == PIPE_SHADER_COMPUTE]++; 1742 res->gfx_barrier |= zink_pipeline_flags_from_pipe_stage(shader_type); 1743 res->barrier_access[shader_type == PIPE_SHADER_COMPUTE] |= VK_ACCESS_SHADER_READ_BIT; 1744 } else if (a != b) { 1745 check_samplerview_for_batch_ref(ctx, a); 1746 } 1747 if (res->base.b.target == PIPE_BUFFER) { 1748 if (b->buffer_view->bvci.buffer != res->obj->buffer) { 1749 /* if this resource has been rebound while it wasn't set here, 1750 * its backing resource will have changed and thus we need to update 1751 * the bufferview 1752 */ 1753 VkBufferViewCreateInfo bvci = b->buffer_view->bvci; 1754 bvci.buffer = res->obj->buffer; 1755 struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci); 1756 assert(buffer_view != b->buffer_view); 1757 if (zink_batch_usage_exists(b->buffer_view->batch_uses)) 1758 zink_batch_reference_bufferview(&ctx->batch, b->buffer_view); 1759 zink_buffer_view_reference(zink_screen(ctx->base.screen), &b->buffer_view, NULL); 1760 b->buffer_view = buffer_view; 1761 update = true; 1762 } 1763 zink_batch_usage_set(&b->buffer_view->batch_uses, ctx->batch.state); 1764 zink_resource_buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT, 1765 res->gfx_barrier); 1766 if (!a || a->buffer_view->buffer_view != b->buffer_view->buffer_view) 1767 update = true; 1768 } else if (!res->obj->is_buffer) { 1769 if (res->obj != b->image_view->obj) { 1770 struct pipe_surface *psurf = &b->image_view->base; 1771 VkImageView iv = b->image_view->image_view; 1772 zink_rebind_surface(ctx, &psurf); 1773 b->image_view = zink_surface(psurf); 1774 update |= iv != b->image_view->image_view; 1775 } else if (a != b) 1776 update = true; 1777 if (shader_type == PIPE_SHADER_COMPUTE) 1778 flush_pending_clears(ctx, res); 1779 if (b->cube_array) { 1780 ctx->di.cubes[shader_type] |= BITFIELD_BIT(start_slot + i); 1781 zink_batch_usage_set(&b->cube_array->batch_uses, ctx->batch.state); 1782 } 1783 check_for_layout_update(ctx, res, shader_type == PIPE_SHADER_COMPUTE); 1784 zink_batch_usage_set(&b->image_view->batch_uses, ctx->batch.state); 1785 if (!a) 1786 update = true; 1787 } 1788 res->sampler_binds[shader_type] |= BITFIELD_BIT(start_slot + i); 1789 zink_batch_resource_usage_set(&ctx->batch, res, false); 1790 res->obj->unordered_read = false; 1791 } else if (a) { 1792 unbind_samplerview(ctx, shader_type, start_slot + i); 1793 update = true; 1794 } 1795 if (take_ownership) { 1796 pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], NULL); 1797 ctx->sampler_views[shader_type][start_slot + i] = pview; 1798 } else { 1799 pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], pview); 1800 } 1801 update_descriptor_state_sampler(ctx, shader_type, start_slot + i, res); 1802 } 1803 for (; i < num_views + unbind_num_trailing_slots; ++i) { 1804 update |= !!ctx->sampler_views[shader_type][start_slot + i]; 1805 unbind_samplerview(ctx, shader_type, start_slot + i); 1806 pipe_sampler_view_reference( 1807 &ctx->sampler_views[shader_type][start_slot + i], 1808 NULL); 1809 update_descriptor_state_sampler(ctx, shader_type, start_slot + i, NULL); 1810 } 1811 ctx->di.num_sampler_views[shader_type] = start_slot + num_views; 1812 if (update) { 1813 struct zink_screen *screen = zink_screen(pctx->screen); 1814 screen->context_invalidate_descriptor_state(ctx, shader_type, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, num_views); 1815 if (!screen->info.have_EXT_non_seamless_cube_map) 1816 update_nonseamless_shader_key(ctx, shader_type); 1817 } 1818} 1819 1820static uint64_t 1821zink_create_texture_handle(struct pipe_context *pctx, struct pipe_sampler_view *view, const struct pipe_sampler_state *state) 1822{ 1823 struct zink_context *ctx = zink_context(pctx); 1824 struct zink_resource *res = zink_resource(view->texture); 1825 struct zink_sampler_view *sv = zink_sampler_view(view); 1826 struct zink_bindless_descriptor *bd; 1827 bd = calloc(1, sizeof(struct zink_bindless_descriptor)); 1828 if (!bd) 1829 return 0; 1830 1831 bd->sampler = pctx->create_sampler_state(pctx, state); 1832 if (!bd->sampler) { 1833 free(bd); 1834 return 0; 1835 } 1836 1837 bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER; 1838 if (res->base.b.target == PIPE_BUFFER) 1839 zink_buffer_view_reference(zink_screen(pctx->screen), &bd->ds.bufferview, sv->buffer_view); 1840 else 1841 zink_surface_reference(zink_screen(pctx->screen), &bd->ds.surface, sv->image_view); 1842 uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].tex_slots); 1843 if (bd->ds.is_buffer) 1844 handle += ZINK_MAX_BINDLESS_HANDLES; 1845 bd->handle = handle; 1846 _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].tex_handles, (void*)(uintptr_t)handle, bd); 1847 return handle; 1848} 1849 1850static void 1851zink_delete_texture_handle(struct pipe_context *pctx, uint64_t handle) 1852{ 1853 struct zink_context *ctx = zink_context(pctx); 1854 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle); 1855 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle); 1856 assert(he); 1857 struct zink_bindless_descriptor *bd = he->data; 1858 struct zink_descriptor_surface *ds = &bd->ds; 1859 _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].tex_handles, he); 1860 uint32_t h = handle; 1861 util_dynarray_append(&ctx->batch.state->bindless_releases[0], uint32_t, h); 1862 1863 struct zink_resource *res = zink_descriptor_surface_resource(ds); 1864 if (ds->is_buffer) { 1865 if (zink_resource_has_usage(res)) 1866 zink_batch_reference_bufferview(&ctx->batch, ds->bufferview); 1867 zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL); 1868 } else { 1869 if (zink_resource_has_usage(res)) 1870 zink_batch_reference_surface(&ctx->batch, ds->surface); 1871 zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL); 1872 pctx->delete_sampler_state(pctx, bd->sampler); 1873 } 1874 free(ds); 1875} 1876 1877static void 1878rebind_bindless_bufferview(struct zink_context *ctx, struct zink_resource *res, struct zink_descriptor_surface *ds) 1879{ 1880 /* if this resource has been rebound while it wasn't set here, 1881 * its backing resource will have changed and thus we need to update 1882 * the bufferview 1883 */ 1884 VkBufferViewCreateInfo bvci = ds->bufferview->bvci; 1885 bvci.buffer = res->obj->buffer; 1886 struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci); 1887 assert(buffer_view != ds->bufferview); 1888 if (zink_resource_has_usage(res)) 1889 zink_batch_reference_bufferview(&ctx->batch, ds->bufferview); 1890 zink_buffer_view_reference(zink_screen(ctx->base.screen), &ds->bufferview, NULL); 1891 ds->bufferview = buffer_view; 1892} 1893 1894static void 1895zero_bindless_descriptor(struct zink_context *ctx, uint32_t handle, bool is_buffer, bool is_image) 1896{ 1897 if (likely(zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor)) { 1898 if (is_buffer) { 1899 VkBufferView *bv = &ctx->di.bindless[is_image].buffer_infos[handle]; 1900 *bv = VK_NULL_HANDLE; 1901 } else { 1902 VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle]; 1903 memset(ii, 0, sizeof(*ii)); 1904 } 1905 } else { 1906 if (is_buffer) { 1907 VkBufferView *bv = &ctx->di.bindless[is_image].buffer_infos[handle]; 1908 struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview; 1909 *bv = null_bufferview->buffer_view; 1910 } else { 1911 struct zink_surface *null_surface = zink_csurface(ctx->dummy_surface[is_image]); 1912 VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle]; 1913 ii->sampler = VK_NULL_HANDLE; 1914 ii->imageView = null_surface->image_view; 1915 ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL; 1916 } 1917 } 1918} 1919 1920static void 1921zink_make_texture_handle_resident(struct pipe_context *pctx, uint64_t handle, bool resident) 1922{ 1923 struct zink_context *ctx = zink_context(pctx); 1924 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle); 1925 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle); 1926 assert(he); 1927 struct zink_bindless_descriptor *bd = he->data; 1928 struct zink_descriptor_surface *ds = &bd->ds; 1929 struct zink_resource *res = zink_descriptor_surface_resource(ds); 1930 if (is_buffer) 1931 handle -= ZINK_MAX_BINDLESS_HANDLES; 1932 if (resident) { 1933 update_res_bind_count(ctx, res, false, false); 1934 update_res_bind_count(ctx, res, true, false); 1935 res->bindless[0]++; 1936 if (is_buffer) { 1937 if (ds->bufferview->bvci.buffer != res->obj->buffer) 1938 rebind_bindless_bufferview(ctx, res, ds); 1939 VkBufferView *bv = &ctx->di.bindless[0].buffer_infos[handle]; 1940 *bv = ds->bufferview->buffer_view; 1941 zink_resource_buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); 1942 } else { 1943 VkDescriptorImageInfo *ii = &ctx->di.bindless[0].img_infos[handle]; 1944 ii->sampler = bd->sampler->sampler; 1945 ii->imageView = ds->surface->image_view; 1946 ii->imageLayout = zink_descriptor_util_image_layout_eval(ctx, res, false); 1947 flush_pending_clears(ctx, res); 1948 check_for_layout_update(ctx, res, false); 1949 check_for_layout_update(ctx, res, true); 1950 } 1951 zink_batch_resource_usage_set(&ctx->batch, res, false); 1952 util_dynarray_append(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd); 1953 uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle; 1954 util_dynarray_append(&ctx->di.bindless[0].updates, uint32_t, h); 1955 res->obj->unordered_read = false; 1956 } else { 1957 zero_bindless_descriptor(ctx, handle, is_buffer, false); 1958 util_dynarray_delete_unordered(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd); 1959 update_res_bind_count(ctx, res, false, true); 1960 update_res_bind_count(ctx, res, true, true); 1961 res->bindless[0]--; 1962 for (unsigned i = 0; i < 2; i++) { 1963 if (!res->image_bind_count[i]) 1964 check_for_layout_update(ctx, res, i); 1965 } 1966 } 1967 ctx->di.bindless_dirty[0] = true; 1968} 1969 1970static uint64_t 1971zink_create_image_handle(struct pipe_context *pctx, const struct pipe_image_view *view) 1972{ 1973 struct zink_context *ctx = zink_context(pctx); 1974 struct zink_resource *res = zink_resource(view->resource); 1975 struct zink_bindless_descriptor *bd; 1976 if (!zink_resource_object_init_storage(ctx, res)) { 1977 debug_printf("couldn't create storage image!"); 1978 return 0; 1979 } 1980 bd = malloc(sizeof(struct zink_bindless_descriptor)); 1981 if (!bd) 1982 return 0; 1983 bd->sampler = NULL; 1984 1985 bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER; 1986 if (res->base.b.target == PIPE_BUFFER) 1987 bd->ds.bufferview = create_image_bufferview(ctx, view); 1988 else 1989 bd->ds.surface = create_image_surface(ctx, view, false); 1990 uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].img_slots); 1991 if (bd->ds.is_buffer) 1992 handle += ZINK_MAX_BINDLESS_HANDLES; 1993 bd->handle = handle; 1994 _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].img_handles, (void*)(uintptr_t)handle, bd); 1995 return handle; 1996} 1997 1998static void 1999zink_delete_image_handle(struct pipe_context *pctx, uint64_t handle) 2000{ 2001 struct zink_context *ctx = zink_context(pctx); 2002 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle); 2003 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle); 2004 assert(he); 2005 struct zink_descriptor_surface *ds = he->data; 2006 _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].img_handles, he); 2007 uint32_t h = handle; 2008 util_dynarray_append(&ctx->batch.state->bindless_releases[1], uint32_t, h); 2009 2010 struct zink_resource *res = zink_descriptor_surface_resource(ds); 2011 if (ds->is_buffer) { 2012 if (zink_resource_has_usage(res)) 2013 zink_batch_reference_bufferview(&ctx->batch, ds->bufferview); 2014 zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL); 2015 } else { 2016 if (zink_resource_has_usage(res)) 2017 zink_batch_reference_surface(&ctx->batch, ds->surface); 2018 zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL); 2019 } 2020 free(ds); 2021} 2022 2023static void 2024zink_make_image_handle_resident(struct pipe_context *pctx, uint64_t handle, unsigned paccess, bool resident) 2025{ 2026 struct zink_context *ctx = zink_context(pctx); 2027 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle); 2028 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle); 2029 assert(he); 2030 struct zink_bindless_descriptor *bd = he->data; 2031 struct zink_descriptor_surface *ds = &bd->ds; 2032 bd->access = paccess; 2033 struct zink_resource *res = zink_descriptor_surface_resource(ds); 2034 VkAccessFlags access = 0; 2035 if (paccess & PIPE_IMAGE_ACCESS_WRITE) { 2036 if (resident) { 2037 res->write_bind_count[0]++; 2038 res->write_bind_count[1]++; 2039 } else { 2040 res->write_bind_count[0]--; 2041 res->write_bind_count[1]--; 2042 } 2043 access |= VK_ACCESS_SHADER_WRITE_BIT; 2044 } 2045 if (paccess & PIPE_IMAGE_ACCESS_READ) { 2046 access |= VK_ACCESS_SHADER_READ_BIT; 2047 } 2048 if (is_buffer) 2049 handle -= ZINK_MAX_BINDLESS_HANDLES; 2050 if (resident) { 2051 update_res_bind_count(ctx, res, false, false); 2052 update_res_bind_count(ctx, res, true, false); 2053 res->image_bind_count[0]++; 2054 res->image_bind_count[1]++; 2055 res->bindless[1]++; 2056 if (is_buffer) { 2057 if (ds->bufferview->bvci.buffer != res->obj->buffer) 2058 rebind_bindless_bufferview(ctx, res, ds); 2059 VkBufferView *bv = &ctx->di.bindless[1].buffer_infos[handle]; 2060 *bv = ds->bufferview->buffer_view; 2061 zink_resource_buffer_barrier(ctx, res, access, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT); 2062 } else { 2063 VkDescriptorImageInfo *ii = &ctx->di.bindless[1].img_infos[handle]; 2064 ii->sampler = VK_NULL_HANDLE; 2065 ii->imageView = ds->surface->image_view; 2066 ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL; 2067 finalize_image_bind(ctx, res, false); 2068 finalize_image_bind(ctx, res, true); 2069 } 2070 zink_batch_resource_usage_set(&ctx->batch, res, zink_resource_access_is_write(access)); 2071 util_dynarray_append(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd); 2072 uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle; 2073 util_dynarray_append(&ctx->di.bindless[1].updates, uint32_t, h); 2074 if (zink_resource_access_is_write(access)) 2075 res->obj->unordered_read = res->obj->unordered_write = false; 2076 else 2077 res->obj->unordered_read = false; 2078 } else { 2079 zero_bindless_descriptor(ctx, handle, is_buffer, true); 2080 util_dynarray_delete_unordered(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd); 2081 unbind_shader_image_counts(ctx, res, false, false); 2082 unbind_shader_image_counts(ctx, res, true, false); 2083 res->bindless[1]--; 2084 for (unsigned i = 0; i < 2; i++) { 2085 if (!res->image_bind_count[i]) 2086 check_for_layout_update(ctx, res, i); 2087 } 2088 } 2089 ctx->di.bindless_dirty[1] = true; 2090} 2091 2092static void 2093zink_set_stencil_ref(struct pipe_context *pctx, 2094 const struct pipe_stencil_ref ref) 2095{ 2096 struct zink_context *ctx = zink_context(pctx); 2097 ctx->stencil_ref = ref; 2098 ctx->stencil_ref_changed = true; 2099} 2100 2101static void 2102zink_set_clip_state(struct pipe_context *pctx, 2103 const struct pipe_clip_state *pcs) 2104{ 2105} 2106 2107static void 2108zink_set_tess_state(struct pipe_context *pctx, 2109 const float default_outer_level[4], 2110 const float default_inner_level[2]) 2111{ 2112 struct zink_context *ctx = zink_context(pctx); 2113 memcpy(&ctx->default_inner_level, default_inner_level, sizeof(ctx->default_inner_level)); 2114 memcpy(&ctx->default_outer_level, default_outer_level, sizeof(ctx->default_outer_level)); 2115} 2116 2117static void 2118zink_set_patch_vertices(struct pipe_context *pctx, uint8_t patch_vertices) 2119{ 2120 struct zink_context *ctx = zink_context(pctx); 2121 if (zink_set_tcs_key_patches(ctx, patch_vertices)) { 2122 ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch = patch_vertices; 2123 if (zink_screen(ctx->base.screen)->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints) 2124 VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->cmdbuf, patch_vertices); 2125 else 2126 ctx->gfx_pipeline_state.dirty = true; 2127 } 2128} 2129 2130void 2131zink_update_fbfetch(struct zink_context *ctx) 2132{ 2133 const bool had_fbfetch = ctx->di.fbfetch.imageLayout == VK_IMAGE_LAYOUT_GENERAL; 2134 if (!ctx->gfx_stages[PIPE_SHADER_FRAGMENT] || 2135 !ctx->gfx_stages[PIPE_SHADER_FRAGMENT]->nir->info.fs.uses_fbfetch_output) { 2136 if (!had_fbfetch) 2137 return; 2138 ctx->rp_changed = true; 2139 zink_batch_no_rp(ctx); 2140 ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_UNDEFINED; 2141 ctx->di.fbfetch.imageView = zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor ? 2142 VK_NULL_HANDLE : 2143 zink_csurface(ctx->dummy_surface[0])->image_view; 2144 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, PIPE_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1); 2145 return; 2146 } 2147 2148 bool changed = !had_fbfetch; 2149 if (ctx->fb_state.cbufs[0]) { 2150 VkImageView fbfetch = zink_csurface(ctx->fb_state.cbufs[0])->image_view; 2151 if (!fbfetch) 2152 /* swapchain image: retry later */ 2153 return; 2154 changed |= fbfetch != ctx->di.fbfetch.imageView; 2155 ctx->di.fbfetch.imageView = zink_csurface(ctx->fb_state.cbufs[0])->image_view; 2156 2157 bool fbfetch_ms = ctx->fb_state.cbufs[0]->texture->nr_samples > 1; 2158 if (zink_get_fs_key(ctx)->fbfetch_ms != fbfetch_ms) 2159 zink_set_fs_key(ctx)->fbfetch_ms = fbfetch_ms; 2160 } 2161 ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_GENERAL; 2162 if (changed) { 2163 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, PIPE_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1); 2164 if (!had_fbfetch) { 2165 ctx->rp_changed = true; 2166 zink_batch_no_rp(ctx); 2167 } 2168 } 2169} 2170 2171void 2172zink_update_vk_sample_locations(struct zink_context *ctx) 2173{ 2174 if (ctx->gfx_pipeline_state.sample_locations_enabled && ctx->sample_locations_changed) { 2175 unsigned samples = ctx->gfx_pipeline_state.rast_samples + 1; 2176 unsigned idx = util_logbase2_ceil(MAX2(samples, 1)); 2177 VkExtent2D grid_size = zink_screen(ctx->base.screen)->maxSampleLocationGridSize[idx]; 2178 2179 for (unsigned pixel = 0; pixel < grid_size.width * grid_size.height; pixel++) { 2180 for (unsigned sample = 0; sample < samples; sample++) { 2181 unsigned pixel_x = pixel % grid_size.width; 2182 unsigned pixel_y = pixel / grid_size.width; 2183 unsigned wi = pixel * samples + sample; 2184 unsigned ri = (pixel_y * grid_size.width + pixel_x % grid_size.width); 2185 ri = ri * samples + sample; 2186 ctx->vk_sample_locations[wi].x = (ctx->sample_locations[ri] & 0xf) / 16.0f; 2187 ctx->vk_sample_locations[wi].y = (16 - (ctx->sample_locations[ri] >> 4)) / 16.0f; 2188 } 2189 } 2190 } 2191} 2192 2193static unsigned 2194find_rp_state(struct zink_context *ctx) 2195{ 2196 bool found = false; 2197 struct set_entry *he = _mesa_set_search_or_add(&ctx->rendering_state_cache, &ctx->gfx_pipeline_state.rendering_info, &found); 2198 struct zink_rendering_info *info; 2199 if (found) { 2200 info = (void*)he->key; 2201 return info->id; 2202 } 2203 info = ralloc(ctx, struct zink_rendering_info); 2204 memcpy(info, &ctx->gfx_pipeline_state.rendering_info, sizeof(VkPipelineRenderingCreateInfo)); 2205 info->id = ctx->rendering_state_cache.entries; 2206 he->key = info; 2207 return info->id; 2208} 2209 2210static unsigned 2211begin_rendering(struct zink_context *ctx) 2212{ 2213 unsigned clear_buffers = 0; 2214 ctx->gfx_pipeline_state.render_pass = NULL; 2215 zink_update_vk_sample_locations(ctx); 2216 zink_render_update_swapchain(ctx); 2217 bool has_depth = false; 2218 bool has_stencil = false; 2219 bool changed_layout = false; 2220 bool changed_size = false; 2221 if (ctx->rp_changed || ctx->rp_layout_changed || ctx->rp_loadop_changed) { 2222 /* init imageviews, base loadOp, formats */ 2223 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) { 2224 struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]); 2225 2226 if (!surf || !zink_resource(surf->base.texture)->valid || (surf->is_swapchain && ctx->new_swapchain)) 2227 ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; 2228 else 2229 ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; 2230 ctx->gfx_pipeline_state.rendering_formats[i] = surf ? surf->info.format[0] : VK_FORMAT_R8G8B8A8_UNORM; 2231 /* use dummy fb size of 1024 if no surf exists */ 2232 unsigned width = surf ? surf->base.texture->width0 : 1024; 2233 unsigned height = surf ? surf->base.texture->height0 : 1024; 2234 unsigned prev_width = ctx->dynamic_fb.info.renderArea.extent.width; 2235 unsigned prev_height = ctx->dynamic_fb.info.renderArea.extent.height; 2236 ctx->dynamic_fb.info.renderArea.extent.width = MIN2(ctx->dynamic_fb.info.renderArea.extent.width, width); 2237 ctx->dynamic_fb.info.renderArea.extent.height = MIN2(ctx->dynamic_fb.info.renderArea.extent.height, height); 2238 changed_size |= ctx->dynamic_fb.info.renderArea.extent.width != prev_width; 2239 changed_size |= ctx->dynamic_fb.info.renderArea.extent.height != prev_height; 2240 } 2241 2242 /* unset depth and stencil info: reset below */ 2243 VkImageLayout zlayout = ctx->dynamic_fb.info.pDepthAttachment ? ctx->dynamic_fb.info.pDepthAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED; 2244 VkImageLayout slayout = ctx->dynamic_fb.info.pStencilAttachment ? ctx->dynamic_fb.info.pStencilAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED; 2245 ctx->dynamic_fb.info.pDepthAttachment = NULL; 2246 ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = VK_FORMAT_UNDEFINED; 2247 ctx->dynamic_fb.info.pStencilAttachment = NULL; 2248 ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat = VK_FORMAT_UNDEFINED; 2249 2250 if (ctx->fb_state.zsbuf) { 2251 struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf); 2252 has_depth = util_format_has_depth(util_format_description(ctx->fb_state.zsbuf->format)); 2253 has_stencil = util_format_has_stencil(util_format_description(ctx->fb_state.zsbuf->format)); 2254 2255 /* depth may or may not be used but init it anyway */ 2256 if (zink_resource(surf->base.texture)->valid) 2257 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; 2258 else 2259 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; 2260 2261 /* stencil may or may not be used but init it anyway */ 2262 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].loadOp = ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp; 2263 2264 if (has_depth) { 2265 ctx->dynamic_fb.info.pDepthAttachment = &ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS]; 2266 ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = surf->info.format[0]; 2267 /* stencil info only set for clears below */ 2268 } 2269 if (has_stencil) { 2270 /* must be stencil-only */ 2271 ctx->dynamic_fb.info.pStencilAttachment = &ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS + 1]; 2272 ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat = surf->info.format[0]; 2273 } 2274 } else { 2275 ctx->dynamic_fb.info.pDepthAttachment = NULL; 2276 ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat = VK_FORMAT_UNDEFINED; 2277 } 2278 if (zlayout != (ctx->dynamic_fb.info.pDepthAttachment ? ctx->dynamic_fb.info.pDepthAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED)) 2279 changed_layout = true; 2280 if (slayout != (ctx->dynamic_fb.info.pStencilAttachment ? ctx->dynamic_fb.info.pStencilAttachment->imageLayout : VK_IMAGE_LAYOUT_UNDEFINED)) 2281 changed_layout = true; 2282 2283 /* similar to begin_render_pass(), but just filling in VkRenderingInfo */ 2284 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) { 2285 /* these are no-ops */ 2286 if (!ctx->fb_state.cbufs[i] || !zink_fb_clear_enabled(ctx, i)) 2287 continue; 2288 /* these need actual clear calls inside the rp */ 2289 struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(&ctx->fb_clears[i], 0); 2290 if (zink_fb_clear_needs_explicit(&ctx->fb_clears[i])) { 2291 clear_buffers |= (PIPE_CLEAR_COLOR0 << i); 2292 if (zink_fb_clear_count(&ctx->fb_clears[i]) < 2 || 2293 zink_fb_clear_element_needs_explicit(clear)) 2294 continue; 2295 } 2296 /* we now know there's one clear that can be done here */ 2297 memcpy(&ctx->dynamic_fb.attachments[i].clearValue, &clear->color, sizeof(float) * 4); 2298 ctx->dynamic_fb.attachments[i].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; 2299 } 2300 if (ctx->fb_state.zsbuf && zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS)) { 2301 struct zink_framebuffer_clear *fb_clear = &ctx->fb_clears[PIPE_MAX_COLOR_BUFS]; 2302 struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(fb_clear, 0); 2303 if (!zink_fb_clear_element_needs_explicit(clear)) { 2304 /* base zs clear info */ 2305 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].clearValue.depthStencil.depth = clear->zs.depth; 2306 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].clearValue.depthStencil.stencil = clear->zs.stencil; 2307 /* always init separate stencil attachment */ 2308 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].clearValue.depthStencil.stencil = clear->zs.stencil; 2309 if ((zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_DEPTH)) 2310 /* initiate a depth clear */ 2311 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; 2312 if ((zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_STENCIL)) { 2313 /* use a stencil clear, also set stencil attachment */ 2314 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; 2315 } 2316 } 2317 if (zink_fb_clear_needs_explicit(fb_clear)) { 2318 for (int j = !zink_fb_clear_element_needs_explicit(clear); 2319 (clear_buffers & PIPE_CLEAR_DEPTHSTENCIL) != PIPE_CLEAR_DEPTHSTENCIL && j < zink_fb_clear_count(fb_clear); 2320 j++) 2321 clear_buffers |= zink_fb_clear_element(fb_clear, j)->zs.bits; 2322 } 2323 } 2324 if (changed_size || changed_layout) 2325 ctx->rp_changed = true; 2326 ctx->rp_loadop_changed = false; 2327 ctx->rp_layout_changed = false; 2328 } 2329 /* validate zs VUs: attachment must be null or format must be valid */ 2330 assert(!ctx->dynamic_fb.info.pDepthAttachment || ctx->gfx_pipeline_state.rendering_info.depthAttachmentFormat); 2331 assert(!ctx->dynamic_fb.info.pStencilAttachment || ctx->gfx_pipeline_state.rendering_info.stencilAttachmentFormat); 2332 2333 if (!ctx->rp_changed && ctx->batch.in_rp) 2334 return 0; 2335 ctx->rp_changed = false; 2336 /* update pipeline info id for compatibility VUs */ 2337 unsigned rp_state = find_rp_state(ctx); 2338 bool rp_changed = ctx->gfx_pipeline_state.rp_state != rp_state; 2339 if (!rp_changed && ctx->batch.in_rp) 2340 return 0; 2341 zink_batch_no_rp(ctx); 2342 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) { 2343 struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]); 2344 VkImageView iv = zink_prep_fb_attachment(ctx, surf, i); 2345 if (!iv) 2346 /* dead swapchain */ 2347 return 0; 2348 ctx->dynamic_fb.attachments[i].imageView = iv; 2349 } 2350 if (ctx->fb_state.zsbuf) { 2351 struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf); 2352 VkImageView iv = zink_prep_fb_attachment(ctx, surf, ctx->fb_state.nr_cbufs); 2353 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageView = iv; 2354 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS].imageLayout = zink_resource(surf->base.texture)->layout; 2355 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageView = iv; 2356 ctx->dynamic_fb.attachments[PIPE_MAX_COLOR_BUFS+1].imageLayout = zink_resource(surf->base.texture)->layout; 2357 } 2358 ctx->gfx_pipeline_state.dirty |= rp_changed; 2359 ctx->gfx_pipeline_state.rp_state = rp_state; 2360 2361 VKCTX(CmdBeginRendering)(ctx->batch.state->cmdbuf, &ctx->dynamic_fb.info); 2362 ctx->batch.in_rp = true; 2363 ctx->new_swapchain = false; 2364 return clear_buffers; 2365} 2366 2367void 2368zink_batch_rp(struct zink_context *ctx) 2369{ 2370 assert(!(ctx->batch.in_rp && ctx->rp_changed)); 2371 if (ctx->batch.in_rp && !ctx->rp_layout_changed) 2372 return; 2373 bool in_rp = ctx->batch.in_rp; 2374 if (!in_rp && ctx->void_clears) { 2375 union pipe_color_union color; 2376 color.f[0] = color.f[1] = color.f[2] = 0; 2377 color.f[3] = 1.0; 2378 ctx->base.clear(&ctx->base, ctx->void_clears, NULL, &color, 0, 0); 2379 ctx->void_clears = 0; 2380 } 2381 unsigned clear_buffers; 2382 /* use renderpass for multisample-to-singlesample or fbfetch: 2383 * - msrtss is TODO 2384 * - dynamic rendering doesn't have input attachments 2385 */ 2386 if (!zink_screen(ctx->base.screen)->info.have_KHR_dynamic_rendering || ctx->transient_attachments || ctx->fbfetch_outputs) 2387 clear_buffers = zink_begin_render_pass(ctx); 2388 else 2389 clear_buffers = begin_rendering(ctx); 2390 assert(!ctx->rp_changed); 2391 if (in_rp || !ctx->batch.in_rp) 2392 return; //dead swapchain or continued renderpass 2393 if (ctx->render_condition.query) 2394 zink_start_conditional_render(ctx); 2395 zink_clear_framebuffer(ctx, clear_buffers); 2396} 2397 2398void 2399zink_batch_no_rp(struct zink_context *ctx) 2400{ 2401 if (!ctx->batch.in_rp) 2402 return; 2403 if (ctx->render_condition.query) 2404 zink_stop_conditional_render(ctx); 2405 if (ctx->gfx_pipeline_state.render_pass) 2406 zink_end_render_pass(ctx); 2407 else { 2408 VKCTX(CmdEndRendering)(ctx->batch.state->cmdbuf); 2409 ctx->batch.in_rp = false; 2410 } 2411 assert(!ctx->batch.in_rp); 2412} 2413 2414ALWAYS_INLINE static void 2415update_res_sampler_layouts(struct zink_context *ctx, struct zink_resource *res) 2416{ 2417 unsigned find = res->sampler_bind_count[0]; 2418 for (unsigned i = 0; find && i < PIPE_SHADER_COMPUTE; i++) { 2419 u_foreach_bit(slot, res->sampler_binds[i]) { 2420 /* only set layout, skip rest of update */ 2421 if (ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW][i][slot] == res) 2422 ctx->di.textures[i][slot].imageLayout = zink_descriptor_util_image_layout_eval(ctx, res, false); 2423 find--; 2424 if (!find) break; 2425 } 2426 } 2427} 2428 2429VkImageView 2430zink_prep_fb_attachment(struct zink_context *ctx, struct zink_surface *surf, unsigned i) 2431{ 2432 struct zink_resource *res; 2433 if (!surf || (i < ctx->fb_state.nr_cbufs && zink_use_dummy_attachments(ctx))) { 2434 surf = zink_csurface(ctx->dummy_surface[util_logbase2_ceil(ctx->fb_state.samples)]); 2435 res = zink_resource(surf->base.texture); 2436 } else { 2437 res = zink_resource(surf->base.texture); 2438 zink_batch_resource_usage_set(&ctx->batch, res, true); 2439 zink_batch_usage_set(&surf->batch_uses, ctx->batch.state); 2440 } 2441 2442 VkAccessFlags access; 2443 VkPipelineStageFlags pipeline; 2444 if (zink_is_swapchain(res)) { 2445 if (!zink_kopper_acquire(ctx, res, UINT64_MAX)) 2446 return VK_NULL_HANDLE; 2447 zink_surface_swapchain_update(ctx, surf); 2448 if (!i) 2449 zink_update_fbfetch(ctx); 2450 } 2451 VkImageLayout layout; 2452 if (ctx->gfx_pipeline_state.render_pass) { 2453 layout = zink_render_pass_attachment_get_barrier_info(&ctx->gfx_pipeline_state.render_pass->state.rts[i], 2454 i < ctx->fb_state.nr_cbufs, &pipeline, &access); 2455 } else { 2456 struct zink_rt_attrib rt; 2457 if (i < ctx->fb_state.nr_cbufs) 2458 zink_init_color_attachment(ctx, i, &rt); 2459 else 2460 zink_init_zs_attachment(ctx, &rt); 2461 layout = zink_render_pass_attachment_get_barrier_info(&rt, i < ctx->fb_state.nr_cbufs, &pipeline, &access); 2462 } 2463 zink_resource_image_barrier(ctx, res, layout, access, pipeline); 2464 res->obj->unordered_read = res->obj->unordered_write = false; 2465 if (i == ctx->fb_state.nr_cbufs && res->sampler_bind_count[0]) 2466 update_res_sampler_layouts(ctx, res); 2467 return surf->image_view; 2468} 2469 2470static uint32_t 2471hash_rendering_state(const void *key) 2472{ 2473 const VkPipelineRenderingCreateInfo *info = key; 2474 uint32_t hash = 0; 2475 /* 2476 uint32_t colorAttachmentCount; 2477 const VkFormat* pColorAttachmentFormats; 2478 VkFormat depthAttachmentFormat; 2479 VkFormat stencilAttachmentFormat; 2480 * this data is not optimally arranged, so it must be manually hashed 2481 */ 2482 hash = XXH32(&info->colorAttachmentCount, sizeof(uint32_t), hash); 2483 hash = XXH32(&info->depthAttachmentFormat, sizeof(uint32_t), hash); 2484 hash = XXH32(&info->stencilAttachmentFormat, sizeof(VkFormat), hash); 2485 return XXH32(info->pColorAttachmentFormats, sizeof(VkFormat) * info->colorAttachmentCount, hash); 2486} 2487 2488static bool 2489equals_rendering_state(const void *a, const void *b) 2490{ 2491 const VkPipelineRenderingCreateInfo *ai = a; 2492 const VkPipelineRenderingCreateInfo *bi = b; 2493 return ai->colorAttachmentCount == bi->colorAttachmentCount && 2494 ai->depthAttachmentFormat == bi->depthAttachmentFormat && 2495 ai->stencilAttachmentFormat == bi->stencilAttachmentFormat && 2496 !memcmp(ai->pColorAttachmentFormats, bi->pColorAttachmentFormats, sizeof(VkFormat) * ai->colorAttachmentCount); 2497} 2498 2499static uint32_t 2500hash_framebuffer_imageless(const void *key) 2501{ 2502 struct zink_framebuffer_state* s = (struct zink_framebuffer_state*)key; 2503 return _mesa_hash_data(key, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments); 2504} 2505 2506static bool 2507equals_framebuffer_imageless(const void *a, const void *b) 2508{ 2509 struct zink_framebuffer_state *s = (struct zink_framebuffer_state*)a; 2510 return memcmp(a, b, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments) == 0; 2511} 2512 2513void 2514zink_init_vk_sample_locations(struct zink_context *ctx, VkSampleLocationsInfoEXT *loc) 2515{ 2516 struct zink_screen *screen = zink_screen(ctx->base.screen); 2517 unsigned idx = util_logbase2_ceil(MAX2(ctx->gfx_pipeline_state.rast_samples + 1, 1)); 2518 loc->sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT; 2519 loc->pNext = NULL; 2520 loc->sampleLocationsPerPixel = 1 << idx; 2521 loc->sampleLocationsCount = ctx->gfx_pipeline_state.rast_samples + 1; 2522 loc->sampleLocationGridSize = screen->maxSampleLocationGridSize[idx]; 2523 loc->pSampleLocations = ctx->vk_sample_locations; 2524} 2525 2526static void 2527zink_evaluate_depth_buffer(struct pipe_context *pctx) 2528{ 2529 struct zink_context *ctx = zink_context(pctx); 2530 2531 if (!ctx->fb_state.zsbuf) 2532 return; 2533 2534 struct zink_resource *res = zink_resource(ctx->fb_state.zsbuf->texture); 2535 res->obj->needs_zs_evaluate = true; 2536 zink_init_vk_sample_locations(ctx, &res->obj->zs_evaluate); 2537 zink_batch_no_rp(ctx); 2538} 2539 2540static void 2541sync_flush(struct zink_context *ctx, struct zink_batch_state *bs) 2542{ 2543 if (zink_screen(ctx->base.screen)->threaded) 2544 util_queue_fence_wait(&bs->flush_completed); 2545} 2546 2547static inline VkAccessFlags 2548get_access_flags_for_binding(struct zink_context *ctx, enum zink_descriptor_type type, enum pipe_shader_type stage, unsigned idx) 2549{ 2550 VkAccessFlags flags = 0; 2551 switch (type) { 2552 case ZINK_DESCRIPTOR_TYPE_UBO: 2553 return VK_ACCESS_UNIFORM_READ_BIT; 2554 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: 2555 return VK_ACCESS_SHADER_READ_BIT; 2556 case ZINK_DESCRIPTOR_TYPE_SSBO: { 2557 flags = VK_ACCESS_SHADER_READ_BIT; 2558 if (ctx->writable_ssbos[stage] & (1 << idx)) 2559 flags |= VK_ACCESS_SHADER_WRITE_BIT; 2560 return flags; 2561 } 2562 case ZINK_DESCRIPTOR_TYPE_IMAGE: { 2563 struct zink_image_view *image_view = &ctx->image_views[stage][idx]; 2564 if (image_view->base.access & PIPE_IMAGE_ACCESS_READ) 2565 flags |= VK_ACCESS_SHADER_READ_BIT; 2566 if (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE) 2567 flags |= VK_ACCESS_SHADER_WRITE_BIT; 2568 return flags; 2569 } 2570 default: 2571 break; 2572 } 2573 unreachable("ACK"); 2574 return 0; 2575} 2576 2577static void 2578update_resource_refs_for_stage(struct zink_context *ctx, enum pipe_shader_type stage) 2579{ 2580 struct zink_batch *batch = &ctx->batch; 2581 unsigned max_slot[] = { 2582 [ZINK_DESCRIPTOR_TYPE_UBO] = ctx->di.num_ubos[stage], 2583 [ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW] = ctx->di.num_samplers[stage], 2584 [ZINK_DESCRIPTOR_TYPE_SSBO] = ctx->di.num_ssbos[stage], 2585 [ZINK_DESCRIPTOR_TYPE_IMAGE] = ctx->di.num_images[stage] 2586 }; 2587 for (unsigned i = 0; i < ZINK_DESCRIPTOR_TYPES; i++) { 2588 for (unsigned j = 0; j < max_slot[i]; j++) { 2589 if (ctx->di.descriptor_res[i][stage][j]) { 2590 struct zink_resource *res = ctx->di.descriptor_res[i][stage][j]; 2591 if (!res) 2592 continue; 2593 bool is_write = zink_resource_access_is_write(get_access_flags_for_binding(ctx, i, stage, j)); 2594 zink_batch_resource_usage_set(batch, res, is_write); 2595 if (is_write) 2596 res->obj->unordered_read = res->obj->unordered_write = false; 2597 else 2598 res->obj->unordered_read = false; 2599 2600 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[stage][j]); 2601 struct zink_sampler_state *sampler_state = ctx->sampler_states[stage][j]; 2602 struct zink_image_view *iv = &ctx->image_views[stage][j]; 2603 if (sampler_state && i == ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW && j <= ctx->di.num_samplers[stage]) 2604 zink_batch_usage_set(&sampler_state->batch_uses, ctx->batch.state); 2605 if (sv && i == ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW && j <= ctx->di.num_sampler_views[stage]) { 2606 if (res->obj->is_buffer) { 2607 zink_batch_usage_set(&sv->buffer_view->batch_uses, ctx->batch.state); 2608 } else { 2609 zink_batch_usage_set(&sv->image_view->batch_uses, ctx->batch.state); 2610 if (sv->cube_array) 2611 zink_batch_usage_set(&sv->cube_array->batch_uses, ctx->batch.state); 2612 } 2613 zink_batch_reference_sampler_view(batch, sv); 2614 } else if (i == ZINK_DESCRIPTOR_TYPE_IMAGE && j <= ctx->di.num_images[stage]) { 2615 if (res->obj->is_buffer) 2616 zink_batch_usage_set(&iv->buffer_view->batch_uses, ctx->batch.state); 2617 else 2618 zink_batch_usage_set(&iv->surface->batch_uses, ctx->batch.state); 2619 zink_batch_reference_image_view(batch, iv); 2620 } 2621 } 2622 } 2623 } 2624} 2625 2626void 2627zink_update_descriptor_refs(struct zink_context *ctx, bool compute) 2628{ 2629 struct zink_batch *batch = &ctx->batch; 2630 if (compute) { 2631 update_resource_refs_for_stage(ctx, PIPE_SHADER_COMPUTE); 2632 if (ctx->curr_compute) 2633 zink_batch_reference_program(batch, &ctx->curr_compute->base); 2634 } else { 2635 for (unsigned i = 0; i < ZINK_SHADER_COUNT; i++) 2636 update_resource_refs_for_stage(ctx, i); 2637 unsigned vertex_buffers_enabled_mask = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask; 2638 unsigned last_vbo = util_last_bit(vertex_buffers_enabled_mask); 2639 for (unsigned i = 0; i < last_vbo + 1; i++) { 2640 struct zink_resource *res = zink_resource(ctx->vertex_buffers[i].buffer.resource); 2641 if (res) { 2642 zink_batch_resource_usage_set(batch, res, false); 2643 res->obj->unordered_read = false; 2644 } 2645 } 2646 if (ctx->curr_program) 2647 zink_batch_reference_program(batch, &ctx->curr_program->base); 2648 } 2649 if (ctx->di.bindless_refs_dirty) { 2650 ctx->di.bindless_refs_dirty = false; 2651 for (unsigned i = 0; i < 2; i++) { 2652 util_dynarray_foreach(&ctx->di.bindless[i].resident, struct zink_bindless_descriptor*, bd) { 2653 struct zink_resource *res = zink_descriptor_surface_resource(&(*bd)->ds); 2654 zink_batch_resource_usage_set(&ctx->batch, res, (*bd)->access & PIPE_IMAGE_ACCESS_WRITE); 2655 if ((*bd)->access & PIPE_IMAGE_ACCESS_WRITE) 2656 res->obj->unordered_read = res->obj->unordered_write = false; 2657 else 2658 res->obj->unordered_read = false; 2659 } 2660 } 2661 } 2662} 2663 2664static void 2665reapply_color_write(struct zink_context *ctx) 2666{ 2667 struct zink_screen *screen = zink_screen(ctx->base.screen); 2668 if (!screen->driver_workarounds.color_write_missing) { 2669 const VkBool32 enables[PIPE_MAX_COLOR_BUFS] = {1, 1, 1, 1, 1, 1, 1, 1}; 2670 const VkBool32 disables[PIPE_MAX_COLOR_BUFS] = {0}; 2671 const unsigned max_att = MIN2(PIPE_MAX_COLOR_BUFS, screen->info.props.limits.maxColorAttachments); 2672 VKCTX(CmdSetColorWriteEnableEXT)(ctx->batch.state->cmdbuf, max_att, ctx->disable_color_writes ? disables : enables); 2673 } 2674 if (screen->info.have_EXT_extended_dynamic_state && ctx->dsa_state) 2675 VKCTX(CmdSetDepthWriteEnableEXT)(ctx->batch.state->cmdbuf, ctx->disable_color_writes ? VK_FALSE : ctx->dsa_state->hw_state.depth_write); 2676} 2677 2678static void 2679stall(struct zink_context *ctx) 2680{ 2681 struct zink_screen *screen = zink_screen(ctx->base.screen); 2682 sync_flush(ctx, zink_batch_state(ctx->last_fence)); 2683 zink_screen_timeline_wait(screen, ctx->last_fence->batch_id, PIPE_TIMEOUT_INFINITE); 2684 zink_batch_reset_all(ctx); 2685} 2686 2687static void 2688flush_batch(struct zink_context *ctx, bool sync) 2689{ 2690 struct zink_batch *batch = &ctx->batch; 2691 if (ctx->clears_enabled) 2692 /* start rp to do all the clears */ 2693 zink_batch_rp(ctx); 2694 bool conditional_render_active = ctx->render_condition.active; 2695 zink_stop_conditional_render(ctx); 2696 zink_batch_no_rp(ctx); 2697 zink_end_batch(ctx, batch); 2698 ctx->deferred_fence = NULL; 2699 2700 if (sync) 2701 sync_flush(ctx, ctx->batch.state); 2702 2703 if (ctx->batch.state->is_device_lost) { 2704 check_device_lost(ctx); 2705 } else { 2706 zink_start_batch(ctx, batch); 2707 if (zink_screen(ctx->base.screen)->info.have_EXT_transform_feedback && ctx->num_so_targets) 2708 ctx->dirty_so_targets = true; 2709 ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true; 2710 zink_select_draw_vbo(ctx); 2711 zink_select_launch_grid(ctx); 2712 2713 if (ctx->oom_stall) 2714 stall(ctx); 2715 ctx->oom_flush = false; 2716 ctx->oom_stall = false; 2717 if (ctx->dd) //copy context 2718 ctx->dd->bindless_bound = false; 2719 ctx->di.bindless_refs_dirty = true; 2720 ctx->sample_locations_changed = ctx->gfx_pipeline_state.sample_locations_enabled; 2721 if (zink_screen(ctx->base.screen)->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints) 2722 VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->cmdbuf, ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch); 2723 if (conditional_render_active) 2724 zink_start_conditional_render(ctx); 2725 reapply_color_write(ctx); 2726 } 2727} 2728 2729void 2730zink_flush_queue(struct zink_context *ctx) 2731{ 2732 flush_batch(ctx, true); 2733} 2734 2735static bool 2736rebind_fb_surface(struct zink_context *ctx, struct pipe_surface **surf, struct zink_resource *match_res) 2737{ 2738 if (!*surf) 2739 return false; 2740 struct zink_resource *surf_res = zink_resource((*surf)->texture); 2741 if ((match_res == surf_res) || surf_res->obj != zink_csurface(*surf)->obj) 2742 return zink_rebind_ctx_surface(ctx, surf); 2743 return false; 2744} 2745 2746static bool 2747rebind_fb_state(struct zink_context *ctx, struct zink_resource *match_res, bool from_set_fb) 2748{ 2749 bool rebind = false; 2750 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) { 2751 rebind |= rebind_fb_surface(ctx, &ctx->fb_state.cbufs[i], match_res); 2752 if (from_set_fb && ctx->fb_state.cbufs[i] && ctx->fb_state.cbufs[i]->texture->bind & PIPE_BIND_SCANOUT) 2753 ctx->new_swapchain = true; 2754 } 2755 rebind |= rebind_fb_surface(ctx, &ctx->fb_state.zsbuf, match_res); 2756 return rebind; 2757} 2758 2759static void 2760unbind_fb_surface(struct zink_context *ctx, struct pipe_surface *surf, unsigned idx, bool changed) 2761{ 2762 ctx->dynamic_fb.attachments[idx].imageView = VK_NULL_HANDLE; 2763 if (!surf) 2764 return; 2765 struct zink_surface *transient = zink_transient_surface(surf); 2766 struct zink_resource *res = zink_resource(surf->texture); 2767 if (changed) { 2768 if (zink_batch_usage_exists(zink_csurface(surf)->batch_uses)) { 2769 zink_batch_reference_surface(&ctx->batch, zink_csurface(surf)); 2770 if (transient) 2771 zink_batch_reference_surface(&ctx->batch, transient); 2772 } 2773 ctx->rp_changed = true; 2774 } 2775 res->fb_binds--; 2776 if (!res->fb_binds) { 2777 check_resource_for_batch_ref(ctx, res); 2778 if (res->sampler_bind_count[0]) { 2779 update_res_sampler_layouts(ctx, res); 2780 _mesa_set_add(ctx->need_barriers[0], res); 2781 } 2782 } 2783} 2784 2785void 2786zink_set_color_write_enables(struct zink_context *ctx) 2787{ 2788 bool disable_color_writes = ctx->rast_state && ctx->rast_state->base.rasterizer_discard && ctx->primitives_generated_active; 2789 if (ctx->disable_color_writes == disable_color_writes) 2790 return; 2791 /* flush all pending clears: these have already occurred */ 2792 if (disable_color_writes && ctx->clears_enabled) 2793 zink_batch_rp(ctx); 2794 ctx->disable_color_writes = disable_color_writes; 2795 if (zink_screen(ctx->base.screen)->driver_workarounds.color_write_missing) { 2796 /* use dummy color buffers instead of the more sane option */ 2797 zink_batch_no_rp(ctx); 2798 ctx->rp_changed = true; 2799 zink_update_framebuffer_state(ctx); 2800 } else { 2801 reapply_color_write(ctx); 2802 } 2803} 2804 2805static void 2806zink_set_framebuffer_state(struct pipe_context *pctx, 2807 const struct pipe_framebuffer_state *state) 2808{ 2809 struct zink_context *ctx = zink_context(pctx); 2810 unsigned samples = state->nr_cbufs || state->zsbuf ? 0 : state->samples; 2811 unsigned w = ctx->fb_state.width; 2812 unsigned h = ctx->fb_state.height; 2813 unsigned layers = MAX2(zink_framebuffer_get_num_layers(state), 1); 2814 2815 bool flush_clears = ctx->clears_enabled && 2816 (ctx->dynamic_fb.info.layerCount != layers || 2817 state->width != w || state->height != h); 2818 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) { 2819 if (i >= state->nr_cbufs || ctx->fb_state.cbufs[i] != state->cbufs[i]) 2820 flush_clears |= zink_fb_clear_enabled(ctx, i); 2821 } 2822 if (ctx->fb_state.zsbuf != state->zsbuf) 2823 flush_clears |= zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS); 2824 if (flush_clears) 2825 zink_batch_rp(ctx); 2826 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) { 2827 struct pipe_surface *psurf = ctx->fb_state.cbufs[i]; 2828 if (i < state->nr_cbufs) 2829 ctx->rp_changed |= !!zink_transient_surface(psurf) != !!zink_transient_surface(state->cbufs[i]); 2830 unbind_fb_surface(ctx, psurf, i, i >= state->nr_cbufs || psurf != state->cbufs[i]); 2831 if (psurf && ctx->needs_present == zink_resource(psurf->texture)) 2832 ctx->needs_present = NULL; 2833 } 2834 if (ctx->fb_state.zsbuf) { 2835 struct pipe_surface *psurf = ctx->fb_state.zsbuf; 2836 struct zink_resource *res = zink_resource(psurf->texture); 2837 bool changed = psurf != state->zsbuf; 2838 unbind_fb_surface(ctx, psurf, PIPE_MAX_COLOR_BUFS, changed); 2839 if (!changed) 2840 ctx->rp_changed |= !!zink_transient_surface(psurf) != !!zink_transient_surface(state->zsbuf); 2841 if (changed && unlikely(res->obj->needs_zs_evaluate)) 2842 /* have to flush zs eval while the sample location data still exists, 2843 * so just throw some random barrier */ 2844 zink_resource_image_barrier(ctx, res, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, 2845 VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT); 2846 } 2847 /* renderpass changes if the number or types of attachments change */ 2848 ctx->rp_changed |= ctx->fb_state.nr_cbufs != state->nr_cbufs; 2849 ctx->rp_changed |= !!ctx->fb_state.zsbuf != !!state->zsbuf; 2850 2851 util_copy_framebuffer_state(&ctx->fb_state, state); 2852 zink_update_fbfetch(ctx); 2853 unsigned prev_void_alpha_attachments = ctx->gfx_pipeline_state.void_alpha_attachments; 2854 ctx->gfx_pipeline_state.void_alpha_attachments = 0; 2855 ctx->transient_attachments = 0; 2856 ctx->fb_layer_mismatch = 0; 2857 2858 ctx->dynamic_fb.info.renderArea.offset.x = 0; 2859 ctx->dynamic_fb.info.renderArea.offset.y = 0; 2860 ctx->dynamic_fb.info.renderArea.extent.width = state->width; 2861 ctx->dynamic_fb.info.renderArea.extent.height = state->height; 2862 ctx->dynamic_fb.info.colorAttachmentCount = ctx->fb_state.nr_cbufs; 2863 ctx->rp_changed |= ctx->dynamic_fb.info.layerCount != layers; 2864 ctx->dynamic_fb.info.layerCount = layers; 2865 ctx->gfx_pipeline_state.rendering_info.colorAttachmentCount = ctx->fb_state.nr_cbufs; 2866 2867 ctx->void_clears = 0; 2868 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) { 2869 struct pipe_surface *psurf = ctx->fb_state.cbufs[i]; 2870 if (psurf) { 2871 struct zink_surface *transient = zink_transient_surface(psurf); 2872 if (transient) 2873 ctx->transient_attachments |= BITFIELD_BIT(i); 2874 if (!samples) 2875 samples = MAX3(transient ? transient->base.nr_samples : 1, psurf->texture->nr_samples, 1); 2876 struct zink_resource *res = zink_resource(psurf->texture); 2877 if (zink_csurface(psurf)->info.layerCount > layers) 2878 ctx->fb_layer_mismatch |= BITFIELD_BIT(i); 2879 if (res->modifiers) { 2880 assert(!ctx->needs_present || ctx->needs_present == res); 2881 ctx->needs_present = res; 2882 } 2883 if (res->obj->dt) { 2884 /* #6274 */ 2885 if (!zink_screen(ctx->base.screen)->info.have_KHR_swapchain_mutable_format && 2886 psurf->format != res->base.b.format) { 2887 static bool warned = false; 2888 if (!warned) { 2889 mesa_loge("zink: SRGB framebuffer unsupported without KHR_swapchain_mutable_format"); 2890 warned = true; 2891 } 2892 } 2893 } 2894 res->fb_binds++; 2895 if (util_format_has_alpha1(psurf->format)) { 2896 ctx->gfx_pipeline_state.void_alpha_attachments |= BITFIELD_BIT(i); 2897 if (!res->valid) 2898 ctx->void_clears |= (PIPE_CLEAR_COLOR0 << i); 2899 } 2900 } 2901 } 2902 if (ctx->gfx_pipeline_state.void_alpha_attachments != prev_void_alpha_attachments) 2903 ctx->gfx_pipeline_state.dirty = true; 2904 unsigned depth_bias_scale_factor = ctx->depth_bias_scale_factor; 2905 if (ctx->fb_state.zsbuf) { 2906 struct pipe_surface *psurf = ctx->fb_state.zsbuf; 2907 struct zink_surface *transient = zink_transient_surface(psurf); 2908 if (transient) 2909 ctx->transient_attachments |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS); 2910 if (!samples) 2911 samples = MAX3(transient ? transient->base.nr_samples : 1, psurf->texture->nr_samples, 1); 2912 if (zink_csurface(psurf)->info.layerCount > layers) 2913 ctx->fb_layer_mismatch |= BITFIELD_BIT(PIPE_MAX_COLOR_BUFS); 2914 zink_resource(psurf->texture)->fb_binds++; 2915 switch (psurf->format) { 2916 case PIPE_FORMAT_Z16_UNORM: 2917 case PIPE_FORMAT_Z16_UNORM_S8_UINT: 2918 ctx->depth_bias_scale_factor = zink_screen(ctx->base.screen)->driver_workarounds.z16_unscaled_bias; 2919 break; 2920 case PIPE_FORMAT_Z24X8_UNORM: 2921 case PIPE_FORMAT_Z24_UNORM_S8_UINT: 2922 case PIPE_FORMAT_X24S8_UINT: 2923 case PIPE_FORMAT_X8Z24_UNORM: 2924 ctx->depth_bias_scale_factor = zink_screen(ctx->base.screen)->driver_workarounds.z24_unscaled_bias; 2925 break; 2926 case PIPE_FORMAT_Z32_FLOAT: 2927 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: 2928 case PIPE_FORMAT_Z32_UNORM: 2929 ctx->depth_bias_scale_factor = 1<<23; 2930 break; 2931 default: 2932 ctx->depth_bias_scale_factor = 0; 2933 } 2934 } else { 2935 ctx->depth_bias_scale_factor = 0; 2936 } 2937 if (depth_bias_scale_factor != ctx->depth_bias_scale_factor && 2938 ctx->rast_state && ctx->rast_state->base.offset_units_unscaled) 2939 ctx->rast_state_changed = true; 2940 rebind_fb_state(ctx, NULL, true); 2941 ctx->fb_state.samples = MAX2(samples, 1); 2942 zink_update_framebuffer_state(ctx); 2943 if (ctx->fb_state.width != w || ctx->fb_state.height != h) 2944 ctx->scissor_changed = true; 2945 2946 uint8_t rast_samples = ctx->fb_state.samples - 1; 2947 if (rast_samples != ctx->gfx_pipeline_state.rast_samples) 2948 zink_update_fs_key_samples(ctx); 2949 if (ctx->gfx_pipeline_state.rast_samples != rast_samples) { 2950 ctx->sample_locations_changed |= ctx->gfx_pipeline_state.sample_locations_enabled; 2951 ctx->gfx_pipeline_state.dirty = true; 2952 } 2953 ctx->gfx_pipeline_state.rast_samples = rast_samples; 2954 2955 /* need to ensure we start a new rp on next draw */ 2956 zink_batch_no_rp(ctx); 2957 /* this is an ideal time to oom flush since it won't split a renderpass */ 2958 if (ctx->oom_flush) 2959 flush_batch(ctx, false); 2960} 2961 2962static void 2963zink_set_blend_color(struct pipe_context *pctx, 2964 const struct pipe_blend_color *color) 2965{ 2966 struct zink_context *ctx = zink_context(pctx); 2967 memcpy(ctx->blend_constants, color->color, sizeof(float) * 4); 2968} 2969 2970static void 2971zink_set_sample_mask(struct pipe_context *pctx, unsigned sample_mask) 2972{ 2973 struct zink_context *ctx = zink_context(pctx); 2974 ctx->gfx_pipeline_state.sample_mask = sample_mask; 2975 ctx->gfx_pipeline_state.dirty = true; 2976} 2977 2978static void 2979zink_set_sample_locations(struct pipe_context *pctx, size_t size, const uint8_t *locations) 2980{ 2981 struct zink_context *ctx = zink_context(pctx); 2982 2983 ctx->gfx_pipeline_state.sample_locations_enabled = size && locations; 2984 ctx->sample_locations_changed = ctx->gfx_pipeline_state.sample_locations_enabled; 2985 if (size > sizeof(ctx->sample_locations)) 2986 size = sizeof(ctx->sample_locations); 2987 2988 if (locations) 2989 memcpy(ctx->sample_locations, locations, size); 2990} 2991 2992static VkAccessFlags 2993access_src_flags(VkImageLayout layout) 2994{ 2995 switch (layout) { 2996 case VK_IMAGE_LAYOUT_UNDEFINED: 2997 return 0; 2998 2999 case VK_IMAGE_LAYOUT_GENERAL: 3000 return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; 3001 3002 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: 3003 return VK_ACCESS_COLOR_ATTACHMENT_READ_BIT; 3004 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: 3005 return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT; 3006 3007 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL: 3008 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: 3009 return VK_ACCESS_SHADER_READ_BIT; 3010 3011 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: 3012 return VK_ACCESS_TRANSFER_READ_BIT; 3013 3014 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: 3015 return VK_ACCESS_TRANSFER_WRITE_BIT; 3016 3017 case VK_IMAGE_LAYOUT_PREINITIALIZED: 3018 return VK_ACCESS_HOST_WRITE_BIT; 3019 3020 case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR: 3021 return 0; 3022 3023 default: 3024 unreachable("unexpected layout"); 3025 } 3026} 3027 3028static VkAccessFlags 3029access_dst_flags(VkImageLayout layout) 3030{ 3031 switch (layout) { 3032 case VK_IMAGE_LAYOUT_UNDEFINED: 3033 return 0; 3034 3035 case VK_IMAGE_LAYOUT_GENERAL: 3036 return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; 3037 3038 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: 3039 return VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; 3040 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: 3041 return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT; 3042 3043 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: 3044 return VK_ACCESS_SHADER_READ_BIT; 3045 3046 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: 3047 return VK_ACCESS_TRANSFER_READ_BIT; 3048 3049 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL: 3050 return VK_ACCESS_SHADER_READ_BIT; 3051 3052 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: 3053 return VK_ACCESS_TRANSFER_WRITE_BIT; 3054 3055 case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR: 3056 return 0; 3057 3058 default: 3059 unreachable("unexpected layout"); 3060 } 3061} 3062 3063static VkPipelineStageFlags 3064pipeline_dst_stage(VkImageLayout layout) 3065{ 3066 switch (layout) { 3067 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL: 3068 return VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; 3069 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL: 3070 return VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT; 3071 3072 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL: 3073 return VK_PIPELINE_STAGE_TRANSFER_BIT; 3074 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL: 3075 return VK_PIPELINE_STAGE_TRANSFER_BIT; 3076 3077 case VK_IMAGE_LAYOUT_GENERAL: 3078 return VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; 3079 3080 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL: 3081 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL: 3082 return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; 3083 3084 default: 3085 return VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; 3086 } 3087} 3088 3089#define ALL_READ_ACCESS_FLAGS \ 3090 (VK_ACCESS_INDIRECT_COMMAND_READ_BIT | \ 3091 VK_ACCESS_INDEX_READ_BIT | \ 3092 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | \ 3093 VK_ACCESS_UNIFORM_READ_BIT | \ 3094 VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | \ 3095 VK_ACCESS_SHADER_READ_BIT | \ 3096 VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | \ 3097 VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | \ 3098 VK_ACCESS_TRANSFER_READ_BIT |\ 3099 VK_ACCESS_HOST_READ_BIT |\ 3100 VK_ACCESS_MEMORY_READ_BIT |\ 3101 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT |\ 3102 VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT |\ 3103 VK_ACCESS_COLOR_ATTACHMENT_READ_NONCOHERENT_BIT_EXT |\ 3104 VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\ 3105 VK_ACCESS_FRAGMENT_SHADING_RATE_ATTACHMENT_READ_BIT_KHR |\ 3106 VK_ACCESS_FRAGMENT_DENSITY_MAP_READ_BIT_EXT |\ 3107 VK_ACCESS_COMMAND_PREPROCESS_READ_BIT_NV |\ 3108 VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\ 3109 VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_KHR) 3110 3111 3112bool 3113zink_resource_access_is_write(VkAccessFlags flags) 3114{ 3115 return (flags & ALL_READ_ACCESS_FLAGS) != flags; 3116} 3117 3118bool 3119zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) 3120{ 3121 if (!pipeline) 3122 pipeline = pipeline_dst_stage(new_layout); 3123 if (!flags) 3124 flags = access_dst_flags(new_layout); 3125 return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline || 3126 (res->obj->access & flags) != flags || 3127 zink_resource_access_is_write(res->obj->access) || 3128 zink_resource_access_is_write(flags); 3129} 3130 3131bool 3132zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) 3133{ 3134 if (!pipeline) 3135 pipeline = pipeline_dst_stage(new_layout); 3136 if (!flags) 3137 flags = access_dst_flags(new_layout); 3138 3139 VkImageSubresourceRange isr = { 3140 res->aspect, 3141 0, VK_REMAINING_MIP_LEVELS, 3142 0, VK_REMAINING_ARRAY_LAYERS 3143 }; 3144 *imb = (VkImageMemoryBarrier){ 3145 VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER, 3146 NULL, 3147 res->obj->access ? res->obj->access : access_src_flags(res->layout), 3148 flags, 3149 res->layout, 3150 new_layout, 3151 VK_QUEUE_FAMILY_IGNORED, 3152 VK_QUEUE_FAMILY_IGNORED, 3153 res->obj->image, 3154 isr 3155 }; 3156 return res->obj->needs_zs_evaluate || zink_resource_image_needs_barrier(res, new_layout, flags, pipeline); 3157} 3158 3159static inline bool 3160is_shader_pipline_stage(VkPipelineStageFlags pipeline) 3161{ 3162 return pipeline & GFX_SHADER_BITS; 3163} 3164 3165static void 3166resource_check_defer_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkPipelineStageFlags pipeline) 3167{ 3168 assert(res->obj->is_buffer); 3169 if (res->bind_count[0] - res->so_bind_count > 0) { 3170 if ((res->obj->is_buffer && res->vbo_bind_mask && !(pipeline & VK_PIPELINE_STAGE_VERTEX_INPUT_BIT)) || 3171 ((!res->obj->is_buffer || util_bitcount(res->vbo_bind_mask) != res->bind_count[0]) && !is_shader_pipline_stage(pipeline))) 3172 /* gfx rebind */ 3173 _mesa_set_add(ctx->need_barriers[0], res); 3174 } 3175 if (res->bind_count[1] && !(pipeline & VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT)) 3176 /* compute rebind */ 3177 _mesa_set_add(ctx->need_barriers[1], res); 3178} 3179 3180static inline bool 3181unordered_res_exec(const struct zink_context *ctx, const struct zink_resource *res, bool is_write) 3182{ 3183 /* if all usage is unordered, keep unordered */ 3184 if (res->obj->unordered_read && res->obj->unordered_write) 3185 return true; 3186 /* if testing write access but have any ordered read access, cannot promote */ 3187 if (is_write && zink_batch_usage_matches(res->obj->bo->reads, ctx->batch.state) && !res->obj->unordered_read) 3188 return false; 3189 /* if write access is unordered or nonexistent, always promote */ 3190 return res->obj->unordered_write || !zink_batch_usage_matches(res->obj->bo->writes, ctx->batch.state); 3191} 3192 3193VkCommandBuffer 3194zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst) 3195{ 3196 bool unordered_exec = (zink_debug & ZINK_DEBUG_NOREORDER) == 0; 3197 if (src) 3198 unordered_exec &= unordered_res_exec(ctx, src, false); 3199 if (dst) 3200 unordered_exec &= unordered_res_exec(ctx, dst, true); 3201 if (src) 3202 src->obj->unordered_read = unordered_exec; 3203 if (dst) 3204 dst->obj->unordered_write = unordered_exec; 3205 if (unordered_exec) { 3206 ctx->batch.state->has_barriers = true; 3207 return ctx->batch.state->barrier_cmdbuf; 3208 } 3209 zink_batch_no_rp(ctx); 3210 return ctx->batch.state->cmdbuf; 3211} 3212 3213static void 3214resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout layout, VkPipelineStageFlags pipeline) 3215{ 3216 assert(!res->obj->is_buffer); 3217 3218 bool is_compute = pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; 3219 /* if this is a non-shader barrier and there are binds, always queue a shader barrier */ 3220 bool is_shader = is_shader_pipline_stage(pipeline); 3221 if ((is_shader || !res->bind_count[is_compute]) && 3222 /* if no layout change is needed between gfx and compute, do nothing */ 3223 !res->bind_count[!is_compute] && (!is_compute || !res->fb_binds)) 3224 return; 3225 3226 if (res->bind_count[!is_compute] && is_shader) { 3227 /* if the layout is the same between gfx and compute, do nothing */ 3228 if (layout == zink_descriptor_util_image_layout_eval(ctx, res, !is_compute)) 3229 return; 3230 } 3231 /* queue a layout change if a layout change will be needed */ 3232 if (res->bind_count[!is_compute]) 3233 _mesa_set_add(ctx->need_barriers[!is_compute], res); 3234 /* also queue a layout change if this is a non-shader layout */ 3235 if (res->bind_count[is_compute] && !is_shader) 3236 _mesa_set_add(ctx->need_barriers[is_compute], res); 3237} 3238 3239void 3240zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res, 3241 VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) 3242{ 3243 VkImageMemoryBarrier imb; 3244 if (!pipeline) 3245 pipeline = pipeline_dst_stage(new_layout); 3246 3247 if (!zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline)) 3248 return; 3249 /* only barrier if we're changing layout or doing something besides read -> read */ 3250 bool is_write = zink_resource_access_is_write(imb.dstAccessMask); 3251 VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); 3252 assert(new_layout); 3253 if (!res->obj->access_stage) 3254 imb.srcAccessMask = 0; 3255 if (res->obj->needs_zs_evaluate) 3256 imb.pNext = &res->obj->zs_evaluate; 3257 res->obj->needs_zs_evaluate = false; 3258 if (res->dmabuf_acquire) { 3259 imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT; 3260 imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue; 3261 res->dmabuf_acquire = false; 3262 } 3263 VKCTX(CmdPipelineBarrier)( 3264 cmdbuf, 3265 res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, 3266 pipeline, 3267 0, 3268 0, NULL, 3269 0, NULL, 3270 1, &imb 3271 ); 3272 3273 resource_check_defer_image_barrier(ctx, res, new_layout, pipeline); 3274 3275 res->obj->access = imb.dstAccessMask; 3276 res->obj->access_stage = pipeline; 3277 res->layout = new_layout; 3278} 3279 3280 3281VkPipelineStageFlags 3282zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage) 3283{ 3284 switch (stage) { 3285 case VK_SHADER_STAGE_VERTEX_BIT: 3286 return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT; 3287 case VK_SHADER_STAGE_FRAGMENT_BIT: 3288 return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; 3289 case VK_SHADER_STAGE_GEOMETRY_BIT: 3290 return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT; 3291 case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT: 3292 return VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT; 3293 case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT: 3294 return VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT; 3295 case VK_SHADER_STAGE_COMPUTE_BIT: 3296 return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; 3297 default: 3298 unreachable("unknown shader stage bit"); 3299 } 3300} 3301 3302ALWAYS_INLINE static VkPipelineStageFlags 3303pipeline_access_stage(VkAccessFlags flags) 3304{ 3305 if (flags & (VK_ACCESS_UNIFORM_READ_BIT | 3306 VK_ACCESS_SHADER_READ_BIT | 3307 VK_ACCESS_SHADER_WRITE_BIT)) 3308 return VK_PIPELINE_STAGE_TASK_SHADER_BIT_NV | 3309 VK_PIPELINE_STAGE_MESH_SHADER_BIT_NV | 3310 VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR | 3311 VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | 3312 VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT | 3313 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT | 3314 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT | 3315 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | 3316 VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; 3317 return VK_PIPELINE_STAGE_TRANSFER_BIT; 3318} 3319 3320ALWAYS_INLINE static bool 3321zink_resource_buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline) 3322{ 3323 if (!res->obj->access || !res->obj->access_stage) 3324 return true; 3325 if (!pipeline) 3326 pipeline = pipeline_access_stage(flags); 3327 return zink_resource_access_is_write(res->obj->access) || 3328 zink_resource_access_is_write(flags) || 3329 (res->obj->access_stage & pipeline) != pipeline || 3330 (res->obj->access & flags) != flags; 3331} 3332 3333void 3334zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline) 3335{ 3336 VkMemoryBarrier bmb; 3337 if (!pipeline) 3338 pipeline = pipeline_access_stage(flags); 3339 if (!zink_resource_buffer_needs_barrier(res, flags, pipeline)) 3340 return; 3341 3342 bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; 3343 bmb.pNext = NULL; 3344 bmb.srcAccessMask = res->obj->access; 3345 bmb.dstAccessMask = flags; 3346 if (!res->obj->access_stage) 3347 bmb.srcAccessMask = 0; 3348 bool is_write = zink_resource_access_is_write(flags); 3349 VkCommandBuffer cmdbuf = is_write ? zink_get_cmdbuf(ctx, NULL, res) : zink_get_cmdbuf(ctx, res, NULL); 3350 /* only barrier if we're changing layout or doing something besides read -> read */ 3351 VKCTX(CmdPipelineBarrier)( 3352 cmdbuf, 3353 res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access), 3354 pipeline, 3355 0, 3356 1, &bmb, 3357 0, NULL, 3358 0, NULL 3359 ); 3360 3361 resource_check_defer_buffer_barrier(ctx, res, pipeline); 3362 3363 res->obj->access = bmb.dstAccessMask; 3364 res->obj->access_stage = pipeline; 3365} 3366 3367bool 3368zink_resource_needs_barrier(struct zink_resource *res, VkImageLayout layout, VkAccessFlags flags, VkPipelineStageFlags pipeline) 3369{ 3370 if (res->base.b.target == PIPE_BUFFER) 3371 return zink_resource_buffer_needs_barrier(res, flags, pipeline); 3372 return zink_resource_image_needs_barrier(res, layout, flags, pipeline); 3373} 3374 3375VkShaderStageFlagBits 3376zink_shader_stage(enum pipe_shader_type type) 3377{ 3378 VkShaderStageFlagBits stages[] = { 3379 [PIPE_SHADER_VERTEX] = VK_SHADER_STAGE_VERTEX_BIT, 3380 [PIPE_SHADER_FRAGMENT] = VK_SHADER_STAGE_FRAGMENT_BIT, 3381 [PIPE_SHADER_GEOMETRY] = VK_SHADER_STAGE_GEOMETRY_BIT, 3382 [PIPE_SHADER_TESS_CTRL] = VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT, 3383 [PIPE_SHADER_TESS_EVAL] = VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT, 3384 [PIPE_SHADER_COMPUTE] = VK_SHADER_STAGE_COMPUTE_BIT, 3385 }; 3386 return stages[type]; 3387} 3388 3389static void 3390zink_flush(struct pipe_context *pctx, 3391 struct pipe_fence_handle **pfence, 3392 unsigned flags) 3393{ 3394 struct zink_context *ctx = zink_context(pctx); 3395 bool deferred = flags & PIPE_FLUSH_DEFERRED; 3396 bool deferred_fence = false; 3397 struct zink_batch *batch = &ctx->batch; 3398 struct zink_fence *fence = NULL; 3399 struct zink_screen *screen = zink_screen(ctx->base.screen); 3400 unsigned submit_count = 0; 3401 3402 /* triggering clears will force has_work */ 3403 if (!deferred && ctx->clears_enabled) { 3404 /* if fbfetch outputs are active, disable them when flushing clears */ 3405 unsigned fbfetch_outputs = ctx->fbfetch_outputs; 3406 if (fbfetch_outputs) { 3407 ctx->fbfetch_outputs = 0; 3408 ctx->rp_changed = true; 3409 } 3410 /* start rp to do all the clears */ 3411 zink_batch_rp(ctx); 3412 ctx->fbfetch_outputs = fbfetch_outputs; 3413 ctx->rp_changed |= fbfetch_outputs > 0; 3414 } 3415 3416 if (ctx->needs_present && (flags & PIPE_FLUSH_END_OF_FRAME)) { 3417 if (ctx->needs_present->obj->image) 3418 zink_resource_image_barrier(ctx, ctx->needs_present, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT); 3419 ctx->needs_present = NULL; 3420 } 3421 3422 if (!batch->has_work) { 3423 if (pfence) { 3424 /* reuse last fence */ 3425 fence = ctx->last_fence; 3426 } 3427 if (!deferred) { 3428 struct zink_batch_state *last = zink_batch_state(ctx->last_fence); 3429 if (last) { 3430 sync_flush(ctx, last); 3431 if (last->is_device_lost) 3432 check_device_lost(ctx); 3433 } 3434 } 3435 tc_driver_internal_flush_notify(ctx->tc); 3436 } else { 3437 fence = &batch->state->fence; 3438 submit_count = batch->state->submit_count; 3439 if (deferred && !(flags & PIPE_FLUSH_FENCE_FD) && pfence) 3440 deferred_fence = true; 3441 else 3442 flush_batch(ctx, true); 3443 } 3444 3445 if (pfence) { 3446 struct zink_tc_fence *mfence; 3447 3448 if (flags & TC_FLUSH_ASYNC) { 3449 mfence = zink_tc_fence(*pfence); 3450 assert(mfence); 3451 } else { 3452 mfence = zink_create_tc_fence(); 3453 3454 screen->base.fence_reference(&screen->base, pfence, NULL); 3455 *pfence = (struct pipe_fence_handle *)mfence; 3456 } 3457 3458 mfence->fence = fence; 3459 if (fence) 3460 mfence->submit_count = submit_count; 3461 3462 if (deferred_fence) { 3463 assert(fence); 3464 mfence->deferred_ctx = pctx; 3465 assert(!ctx->deferred_fence || ctx->deferred_fence == fence); 3466 ctx->deferred_fence = fence; 3467 } 3468 3469 if (!fence || flags & TC_FLUSH_ASYNC) { 3470 if (!util_queue_fence_is_signalled(&mfence->ready)) 3471 util_queue_fence_signal(&mfence->ready); 3472 } 3473 } 3474 if (fence) { 3475 if (!(flags & (PIPE_FLUSH_DEFERRED | PIPE_FLUSH_ASYNC))) 3476 sync_flush(ctx, zink_batch_state(fence)); 3477 } 3478} 3479 3480void 3481zink_fence_wait(struct pipe_context *pctx) 3482{ 3483 struct zink_context *ctx = zink_context(pctx); 3484 3485 if (ctx->batch.has_work) 3486 pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH); 3487 if (ctx->last_fence) 3488 stall(ctx); 3489} 3490 3491void 3492zink_wait_on_batch(struct zink_context *ctx, uint64_t batch_id) 3493{ 3494 struct zink_batch_state *bs; 3495 if (!batch_id) { 3496 /* not submitted yet */ 3497 flush_batch(ctx, true); 3498 bs = zink_batch_state(ctx->last_fence); 3499 assert(bs); 3500 batch_id = bs->fence.batch_id; 3501 } 3502 assert(batch_id); 3503 if (!zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, UINT64_MAX)) 3504 check_device_lost(ctx); 3505} 3506 3507bool 3508zink_check_batch_completion(struct zink_context *ctx, uint64_t batch_id) 3509{ 3510 assert(ctx->batch.state); 3511 if (!batch_id) 3512 /* not submitted yet */ 3513 return false; 3514 3515 if (zink_screen_check_last_finished(zink_screen(ctx->base.screen), batch_id)) 3516 return true; 3517 3518 bool success = zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, 0); 3519 if (!success) 3520 check_device_lost(ctx); 3521 return success; 3522} 3523 3524static void 3525zink_texture_barrier(struct pipe_context *pctx, unsigned flags) 3526{ 3527 struct zink_context *ctx = zink_context(pctx); 3528 VkAccessFlags dst = flags == PIPE_TEXTURE_BARRIER_FRAMEBUFFER ? 3529 VK_ACCESS_INPUT_ATTACHMENT_READ_BIT : 3530 VK_ACCESS_SHADER_READ_BIT; 3531 3532 if (!ctx->framebuffer || !ctx->framebuffer->state.num_attachments) 3533 return; 3534 3535 /* if this is a fb barrier, flush all pending clears */ 3536 if (ctx->rp_clears_enabled && dst == VK_ACCESS_INPUT_ATTACHMENT_READ_BIT) 3537 zink_batch_rp(ctx); 3538 3539 /* this is not an in-renderpass barrier */ 3540 if (!ctx->fbfetch_outputs) 3541 zink_batch_no_rp(ctx); 3542 3543 if (zink_screen(ctx->base.screen)->info.have_KHR_synchronization2) { 3544 VkDependencyInfo dep = {0}; 3545 dep.sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO; 3546 dep.dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT; 3547 dep.memoryBarrierCount = 1; 3548 3549 VkMemoryBarrier2 dmb = {0}; 3550 dmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2; 3551 dmb.pNext = NULL; 3552 dmb.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; 3553 dmb.dstAccessMask = dst; 3554 dmb.srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT; 3555 dmb.dstStageMask = VK_PIPELINE_STAGE_2_FRAGMENT_SHADER_BIT; 3556 dep.pMemoryBarriers = &dmb; 3557 3558 /* if zs fbfetch is a thing? 3559 if (ctx->fb_state.zsbuf) { 3560 const VkPipelineStageFlagBits2 depth_flags = VK_PIPELINE_STAGE_2_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_2_LATE_FRAGMENT_TESTS_BIT; 3561 dmb.dstAccessMask |= VK_ACCESS_2_DEPTH_STENCIL_ATTACHMENT_READ_BIT; 3562 dmb.srcStageMask |= depth_flags; 3563 dmb.dstStageMask |= depth_flags; 3564 } 3565 */ 3566 VKCTX(CmdPipelineBarrier2)(ctx->batch.state->cmdbuf, &dep); 3567 } else { 3568 VkMemoryBarrier bmb = {0}; 3569 bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; 3570 bmb.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; 3571 bmb.dstAccessMask = dst; 3572 VKCTX(CmdPipelineBarrier)( 3573 ctx->batch.state->cmdbuf, 3574 VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, 3575 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, 3576 0, 3577 1, &bmb, 3578 0, NULL, 3579 0, NULL 3580 ); 3581 } 3582} 3583 3584static inline void 3585mem_barrier(struct zink_context *ctx, VkPipelineStageFlags src_stage, VkPipelineStageFlags dst_stage, VkAccessFlags src, VkAccessFlags dst) 3586{ 3587 struct zink_batch *batch = &ctx->batch; 3588 VkMemoryBarrier mb; 3589 mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER; 3590 mb.pNext = NULL; 3591 mb.srcAccessMask = src; 3592 mb.dstAccessMask = dst; 3593 zink_batch_no_rp(ctx); 3594 VKCTX(CmdPipelineBarrier)(batch->state->cmdbuf, src_stage, dst_stage, 0, 1, &mb, 0, NULL, 0, NULL); 3595} 3596 3597void 3598zink_flush_memory_barrier(struct zink_context *ctx, bool is_compute) 3599{ 3600 const VkPipelineStageFlags gfx_flags = VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | 3601 VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT | 3602 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT | 3603 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT | 3604 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; 3605 const VkPipelineStageFlags cs_flags = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; 3606 VkPipelineStageFlags src = ctx->batch.last_was_compute ? cs_flags : gfx_flags; 3607 VkPipelineStageFlags dst = is_compute ? cs_flags : gfx_flags; 3608 3609 if (ctx->memory_barrier & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_SHADER_BUFFER | PIPE_BARRIER_IMAGE)) 3610 mem_barrier(ctx, src, dst, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); 3611 3612 if (ctx->memory_barrier & PIPE_BARRIER_CONSTANT_BUFFER) 3613 mem_barrier(ctx, src, dst, 3614 VK_ACCESS_SHADER_WRITE_BIT, 3615 VK_ACCESS_UNIFORM_READ_BIT); 3616 3617 if (!is_compute) { 3618 if (ctx->memory_barrier & PIPE_BARRIER_INDIRECT_BUFFER) 3619 mem_barrier(ctx, src, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT, 3620 VK_ACCESS_SHADER_WRITE_BIT, 3621 VK_ACCESS_INDIRECT_COMMAND_READ_BIT); 3622 if (ctx->memory_barrier & PIPE_BARRIER_VERTEX_BUFFER) 3623 mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 3624 VK_ACCESS_SHADER_WRITE_BIT, 3625 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT); 3626 3627 if (ctx->memory_barrier & PIPE_BARRIER_INDEX_BUFFER) 3628 mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT, 3629 VK_ACCESS_SHADER_WRITE_BIT, 3630 VK_ACCESS_INDEX_READ_BIT); 3631 if (ctx->memory_barrier & PIPE_BARRIER_FRAMEBUFFER) 3632 zink_texture_barrier(&ctx->base, 0); 3633 if (ctx->memory_barrier & PIPE_BARRIER_STREAMOUT_BUFFER) 3634 mem_barrier(ctx, VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | 3635 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT | 3636 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT, 3637 VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT, 3638 VK_ACCESS_SHADER_READ_BIT, 3639 VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT | 3640 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT); 3641 } 3642 ctx->memory_barrier = 0; 3643} 3644 3645static void 3646zink_memory_barrier(struct pipe_context *pctx, unsigned flags) 3647{ 3648 struct zink_context *ctx = zink_context(pctx); 3649 3650 flags &= ~PIPE_BARRIER_UPDATE; 3651 if (!flags) 3652 return; 3653 3654 if (flags & PIPE_BARRIER_MAPPED_BUFFER) { 3655 /* TODO: this should flush all persistent buffers in use as I think */ 3656 flags &= ~PIPE_BARRIER_MAPPED_BUFFER; 3657 } 3658 ctx->memory_barrier = flags; 3659} 3660 3661static void 3662zink_flush_resource(struct pipe_context *pctx, 3663 struct pipe_resource *pres) 3664{ 3665 struct zink_context *ctx = zink_context(pctx); 3666 struct zink_resource *res = zink_resource(pres); 3667 if (res->obj->dt) { 3668 if (zink_kopper_acquired(res->obj->dt, res->obj->dt_idx)) { 3669 zink_resource_image_barrier(ctx, res, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, 0, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT); 3670 zink_batch_reference_resource_rw(&ctx->batch, res, true); 3671 } else { 3672 ctx->needs_present = res; 3673 } 3674 ctx->batch.swapchain = res; 3675 } else if (res->dmabuf) 3676 res->dmabuf_acquire = true; 3677} 3678 3679static struct pipe_stream_output_target * 3680zink_create_stream_output_target(struct pipe_context *pctx, 3681 struct pipe_resource *pres, 3682 unsigned buffer_offset, 3683 unsigned buffer_size) 3684{ 3685 struct zink_so_target *t; 3686 t = CALLOC_STRUCT(zink_so_target); 3687 if (!t) 3688 return NULL; 3689 3690 /* using PIPE_BIND_CUSTOM here lets us create a custom pipe buffer resource, 3691 * which allows us to differentiate and use VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT 3692 * as we must for this case 3693 */ 3694 t->counter_buffer = pipe_buffer_create(pctx->screen, PIPE_BIND_STREAM_OUTPUT | PIPE_BIND_CUSTOM, PIPE_USAGE_DEFAULT, 4); 3695 if (!t->counter_buffer) { 3696 FREE(t); 3697 return NULL; 3698 } 3699 3700 t->base.reference.count = 1; 3701 t->base.context = pctx; 3702 pipe_resource_reference(&t->base.buffer, pres); 3703 t->base.buffer_offset = buffer_offset; 3704 t->base.buffer_size = buffer_size; 3705 3706 zink_resource(t->base.buffer)->so_valid = true; 3707 3708 return &t->base; 3709} 3710 3711static void 3712zink_stream_output_target_destroy(struct pipe_context *pctx, 3713 struct pipe_stream_output_target *psot) 3714{ 3715 struct zink_so_target *t = (struct zink_so_target *)psot; 3716 pipe_resource_reference(&t->counter_buffer, NULL); 3717 pipe_resource_reference(&t->base.buffer, NULL); 3718 FREE(t); 3719} 3720 3721static void 3722zink_set_stream_output_targets(struct pipe_context *pctx, 3723 unsigned num_targets, 3724 struct pipe_stream_output_target **targets, 3725 const unsigned *offsets) 3726{ 3727 struct zink_context *ctx = zink_context(pctx); 3728 3729 /* always set counter_buffer_valid=false on unbind: 3730 * - on resume (indicated by offset==-1), set counter_buffer_valid=true 3731 * - otherwise the counter buffer is invalidated 3732 */ 3733 3734 if (num_targets == 0) { 3735 for (unsigned i = 0; i < ctx->num_so_targets; i++) { 3736 if (ctx->so_targets[i]) { 3737 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer); 3738 if (so) { 3739 so->so_bind_count--; 3740 update_res_bind_count(ctx, so, false, true); 3741 } 3742 } 3743 pipe_so_target_reference(&ctx->so_targets[i], NULL); 3744 } 3745 ctx->num_so_targets = 0; 3746 } else { 3747 for (unsigned i = 0; i < num_targets; i++) { 3748 struct zink_so_target *t = zink_so_target(targets[i]); 3749 pipe_so_target_reference(&ctx->so_targets[i], targets[i]); 3750 if (!t) 3751 continue; 3752 if (offsets[0] != (unsigned)-1) 3753 t->counter_buffer_valid = false; 3754 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer); 3755 if (so) { 3756 so->so_bind_count++; 3757 update_res_bind_count(ctx, so, false, false); 3758 } 3759 } 3760 for (unsigned i = num_targets; i < ctx->num_so_targets; i++) { 3761 if (ctx->so_targets[i]) { 3762 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer); 3763 if (so) { 3764 so->so_bind_count--; 3765 update_res_bind_count(ctx, so, false, true); 3766 } 3767 } 3768 pipe_so_target_reference(&ctx->so_targets[i], NULL); 3769 } 3770 ctx->num_so_targets = num_targets; 3771 3772 /* TODO: possibly avoid rebinding on resume if resuming from same buffers? */ 3773 ctx->dirty_so_targets = true; 3774 } 3775} 3776 3777void 3778zink_rebind_framebuffer(struct zink_context *ctx, struct zink_resource *res) 3779{ 3780 if (!ctx->framebuffer) 3781 return; 3782 bool did_rebind = false; 3783 if (res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) { 3784 for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++) { 3785 if (!ctx->fb_state.cbufs[i] || 3786 zink_resource(ctx->fb_state.cbufs[i]->texture) != res) 3787 continue; 3788 zink_rebind_ctx_surface(ctx, &ctx->fb_state.cbufs[i]); 3789 did_rebind = true; 3790 } 3791 } else { 3792 if (ctx->fb_state.zsbuf && zink_resource(ctx->fb_state.zsbuf->texture) != res) { 3793 zink_rebind_ctx_surface(ctx, &ctx->fb_state.zsbuf); 3794 did_rebind = true; 3795 } 3796 } 3797 3798 did_rebind |= rebind_fb_state(ctx, res, false); 3799 3800 if (!did_rebind) 3801 return; 3802 3803 zink_batch_no_rp(ctx); 3804 struct zink_framebuffer *fb = zink_get_framebuffer(ctx); 3805 ctx->fb_changed |= ctx->framebuffer != fb; 3806 ctx->framebuffer = fb; 3807} 3808 3809ALWAYS_INLINE static struct zink_resource * 3810rebind_ubo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot) 3811{ 3812 struct zink_resource *res = update_descriptor_state_ubo(ctx, shader, slot, 3813 ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_UBO][shader][slot]); 3814 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, slot, 1); 3815 return res; 3816} 3817 3818ALWAYS_INLINE static struct zink_resource * 3819rebind_ssbo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot) 3820{ 3821 const struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot]; 3822 struct zink_resource *res = zink_resource(ssbo->buffer); 3823 if (!res) 3824 return NULL; 3825 util_range_add(&res->base.b, &res->valid_buffer_range, ssbo->buffer_offset, 3826 ssbo->buffer_offset + ssbo->buffer_size); 3827 update_descriptor_state_ssbo(ctx, shader, slot, res); 3828 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SSBO, slot, 1); 3829 return res; 3830} 3831 3832ALWAYS_INLINE static struct zink_resource * 3833rebind_tbo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot) 3834{ 3835 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]); 3836 if (!sampler_view || sampler_view->base.texture->target != PIPE_BUFFER) 3837 return NULL; 3838 struct zink_resource *res = zink_resource(sampler_view->base.texture); 3839 if (zink_batch_usage_exists(sampler_view->buffer_view->batch_uses)) 3840 zink_batch_reference_bufferview(&ctx->batch, sampler_view->buffer_view); 3841 VkBufferViewCreateInfo bvci = sampler_view->buffer_view->bvci; 3842 bvci.buffer = res->obj->buffer; 3843 zink_buffer_view_reference(zink_screen(ctx->base.screen), &sampler_view->buffer_view, NULL); 3844 sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci); 3845 update_descriptor_state_sampler(ctx, shader, slot, res); 3846 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1); 3847 return res; 3848} 3849 3850ALWAYS_INLINE static struct zink_resource * 3851rebind_ibo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot) 3852{ 3853 struct zink_image_view *image_view = &ctx->image_views[shader][slot]; 3854 struct zink_resource *res = zink_resource(image_view->base.resource); 3855 if (!res || res->base.b.target != PIPE_BUFFER) 3856 return NULL; 3857 zink_descriptor_set_refs_clear(&image_view->buffer_view->desc_set_refs, image_view->buffer_view); 3858 if (zink_batch_usage_exists(image_view->buffer_view->batch_uses)) 3859 zink_batch_reference_bufferview(&ctx->batch, image_view->buffer_view); 3860 VkBufferViewCreateInfo bvci = image_view->buffer_view->bvci; 3861 bvci.buffer = res->obj->buffer; 3862 zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL); 3863 if (!zink_resource_object_init_storage(ctx, res)) { 3864 debug_printf("couldn't create storage image!"); 3865 return NULL; 3866 } 3867 image_view->buffer_view = get_buffer_view(ctx, res, &bvci); 3868 assert(image_view->buffer_view); 3869 util_range_add(&res->base.b, &res->valid_buffer_range, image_view->base.u.buf.offset, 3870 image_view->base.u.buf.offset + image_view->base.u.buf.size); 3871 update_descriptor_state_image(ctx, shader, slot, res); 3872 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_IMAGE, slot, 1); 3873 return res; 3874} 3875 3876static unsigned 3877rebind_buffer(struct zink_context *ctx, struct zink_resource *res, uint32_t rebind_mask, const unsigned expected_num_rebinds) 3878{ 3879 unsigned num_rebinds = 0; 3880 bool has_write = false; 3881 3882 if (!zink_resource_has_binds(res)) 3883 return 0; 3884 3885 assert(!res->bindless[1]); //TODO 3886 if ((rebind_mask & BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER)) || (!rebind_mask && res->so_bind_count && ctx->num_so_targets)) { 3887 for (unsigned i = 0; i < ctx->num_so_targets; i++) { 3888 if (ctx->so_targets[i]) { 3889 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer); 3890 if (so && so == res) { 3891 ctx->dirty_so_targets = true; 3892 num_rebinds++; 3893 } 3894 } 3895 } 3896 rebind_mask &= ~BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER); 3897 } 3898 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask) 3899 goto end; 3900 3901 if ((rebind_mask & BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER)) || (!rebind_mask && res->vbo_bind_mask)) { 3902 u_foreach_bit(slot, res->vbo_bind_mask) { 3903 if (ctx->vertex_buffers[slot].buffer.resource != &res->base.b) //wrong context 3904 goto end; 3905 num_rebinds++; 3906 } 3907 rebind_mask &= ~BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER); 3908 ctx->vertex_buffers_dirty = true; 3909 } 3910 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask) 3911 goto end; 3912 3913 const uint32_t ubo_mask = rebind_mask ? 3914 rebind_mask & BITFIELD_RANGE(TC_BINDING_UBO_VS, PIPE_SHADER_TYPES) : 3915 ((res->ubo_bind_count[0] ? BITFIELD_RANGE(TC_BINDING_UBO_VS, (PIPE_SHADER_TYPES - 1)) : 0) | 3916 (res->ubo_bind_count[1] ? BITFIELD_BIT(TC_BINDING_UBO_CS) : 0)); 3917 u_foreach_bit(shader, ubo_mask >> TC_BINDING_UBO_VS) { 3918 u_foreach_bit(slot, res->ubo_bind_mask[shader]) { 3919 if (&res->base.b != ctx->ubos[shader][slot].buffer) //wrong context 3920 goto end; 3921 rebind_ubo(ctx, shader, slot); 3922 num_rebinds++; 3923 } 3924 } 3925 rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_UBO_VS, PIPE_SHADER_TYPES); 3926 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask) 3927 goto end; 3928 3929 const unsigned ssbo_mask = rebind_mask ? 3930 rebind_mask & BITFIELD_RANGE(TC_BINDING_SSBO_VS, PIPE_SHADER_TYPES) : 3931 BITFIELD_RANGE(TC_BINDING_SSBO_VS, PIPE_SHADER_TYPES); 3932 u_foreach_bit(shader, ssbo_mask >> TC_BINDING_SSBO_VS) { 3933 u_foreach_bit(slot, res->ssbo_bind_mask[shader]) { 3934 struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot]; 3935 if (&res->base.b != ssbo->buffer) //wrong context 3936 goto end; 3937 rebind_ssbo(ctx, shader, slot); 3938 has_write |= (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0; 3939 num_rebinds++; 3940 } 3941 } 3942 rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SSBO_VS, PIPE_SHADER_TYPES); 3943 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask) 3944 goto end; 3945 const unsigned sampler_mask = rebind_mask ? 3946 rebind_mask & BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, PIPE_SHADER_TYPES) : 3947 BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, PIPE_SHADER_TYPES); 3948 u_foreach_bit(shader, sampler_mask >> TC_BINDING_SAMPLERVIEW_VS) { 3949 u_foreach_bit(slot, res->sampler_binds[shader]) { 3950 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]); 3951 if (&res->base.b != sampler_view->base.texture) //wrong context 3952 goto end; 3953 rebind_tbo(ctx, shader, slot); 3954 num_rebinds++; 3955 } 3956 } 3957 rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, PIPE_SHADER_TYPES); 3958 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask) 3959 goto end; 3960 3961 const unsigned image_mask = rebind_mask ? 3962 rebind_mask & BITFIELD_RANGE(TC_BINDING_IMAGE_VS, PIPE_SHADER_TYPES) : 3963 BITFIELD_RANGE(TC_BINDING_IMAGE_VS, PIPE_SHADER_TYPES); 3964 unsigned num_image_rebinds_remaining = rebind_mask ? expected_num_rebinds - num_rebinds : res->image_bind_count[0] + res->image_bind_count[1]; 3965 u_foreach_bit(shader, image_mask >> TC_BINDING_IMAGE_VS) { 3966 for (unsigned slot = 0; num_image_rebinds_remaining && slot < ctx->di.num_images[shader]; slot++) { 3967 struct zink_resource *cres = ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_IMAGE][shader][slot]; 3968 if (res != cres) 3969 continue; 3970 3971 rebind_ibo(ctx, shader, slot); 3972 const struct zink_image_view *image_view = &ctx->image_views[shader][slot]; 3973 has_write |= (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE) != 0; 3974 num_image_rebinds_remaining--; 3975 num_rebinds++; 3976 } 3977 } 3978end: 3979 if (num_rebinds) 3980 zink_batch_resource_usage_set(&ctx->batch, res, has_write); 3981 return num_rebinds; 3982} 3983 3984void 3985zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src, 3986 unsigned dst_offset, unsigned src_offset, unsigned size) 3987{ 3988 VkBufferCopy region; 3989 region.srcOffset = src_offset; 3990 region.dstOffset = dst_offset; 3991 region.size = size; 3992 3993 struct zink_batch *batch = &ctx->batch; 3994 util_range_add(&dst->base.b, &dst->valid_buffer_range, dst_offset, dst_offset + size); 3995 zink_resource_buffer_barrier(ctx, src, VK_ACCESS_TRANSFER_READ_BIT, 0); 3996 zink_resource_buffer_barrier(ctx, dst, VK_ACCESS_TRANSFER_WRITE_BIT, 0); 3997 VkCommandBuffer cmdbuf = zink_get_cmdbuf(ctx, src, dst); 3998 zink_batch_reference_resource_rw(batch, src, false); 3999 zink_batch_reference_resource_rw(batch, dst, true); 4000 VKCTX(CmdCopyBuffer)(cmdbuf, src->obj->buffer, dst->obj->buffer, 1, ®ion); 4001} 4002 4003void 4004zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src, 4005 unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz, 4006 unsigned src_level, const struct pipe_box *src_box, enum pipe_map_flags map_flags) 4007{ 4008 struct zink_resource *img = dst->base.b.target == PIPE_BUFFER ? src : dst; 4009 struct zink_resource *buf = dst->base.b.target == PIPE_BUFFER ? dst : src; 4010 struct zink_batch *batch = &ctx->batch; 4011 bool needs_present_readback = false; 4012 4013 bool buf2img = buf == src; 4014 4015 if (buf2img) { 4016 if (zink_is_swapchain(img)) { 4017 if (!zink_kopper_acquire(ctx, img, UINT64_MAX)) 4018 return; 4019 } 4020 zink_resource_image_barrier(ctx, img, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 0, 0); 4021 zink_resource_buffer_barrier(ctx, buf, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); 4022 } else { 4023 if (zink_is_swapchain(img)) 4024 needs_present_readback = zink_kopper_acquire_readback(ctx, img); 4025 zink_resource_image_barrier(ctx, img, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 0); 4026 zink_resource_buffer_barrier(ctx, buf, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); 4027 util_range_add(&dst->base.b, &dst->valid_buffer_range, dstx, dstx + src_box->width); 4028 } 4029 4030 VkBufferImageCopy region = {0}; 4031 region.bufferOffset = buf2img ? src_box->x : dstx; 4032 region.bufferRowLength = 0; 4033 region.bufferImageHeight = 0; 4034 region.imageSubresource.mipLevel = buf2img ? dst_level : src_level; 4035 enum pipe_texture_target img_target = img->base.b.target; 4036 if (img->need_2D) 4037 img_target = img_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY; 4038 switch (img_target) { 4039 case PIPE_TEXTURE_CUBE: 4040 case PIPE_TEXTURE_CUBE_ARRAY: 4041 case PIPE_TEXTURE_2D_ARRAY: 4042 case PIPE_TEXTURE_1D_ARRAY: 4043 /* these use layer */ 4044 region.imageSubresource.baseArrayLayer = buf2img ? dstz : src_box->z; 4045 region.imageSubresource.layerCount = src_box->depth; 4046 region.imageOffset.z = 0; 4047 region.imageExtent.depth = 1; 4048 break; 4049 case PIPE_TEXTURE_3D: 4050 /* this uses depth */ 4051 region.imageSubresource.baseArrayLayer = 0; 4052 region.imageSubresource.layerCount = 1; 4053 region.imageOffset.z = buf2img ? dstz : src_box->z; 4054 region.imageExtent.depth = src_box->depth; 4055 break; 4056 default: 4057 /* these must only copy one layer */ 4058 region.imageSubresource.baseArrayLayer = 0; 4059 region.imageSubresource.layerCount = 1; 4060 region.imageOffset.z = 0; 4061 region.imageExtent.depth = 1; 4062 } 4063 region.imageOffset.x = buf2img ? dstx : src_box->x; 4064 region.imageOffset.y = buf2img ? dsty : src_box->y; 4065 4066 region.imageExtent.width = src_box->width; 4067 region.imageExtent.height = src_box->height; 4068 4069 /* never promote to unordered if swapchain was acquired */ 4070 VkCommandBuffer cmdbuf = needs_present_readback ? 4071 ctx->batch.state->cmdbuf : 4072 buf2img ? zink_get_cmdbuf(ctx, buf, img) : zink_get_cmdbuf(ctx, img, buf); 4073 zink_batch_reference_resource_rw(batch, img, buf2img); 4074 zink_batch_reference_resource_rw(batch, buf, !buf2img); 4075 4076 /* we're using u_transfer_helper_deinterleave, which means we'll be getting PIPE_MAP_* usage 4077 * to indicate whether to copy either the depth or stencil aspects 4078 */ 4079 unsigned aspects = 0; 4080 if (map_flags) { 4081 assert((map_flags & (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY)) != 4082 (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY)); 4083 if (map_flags & PIPE_MAP_DEPTH_ONLY) 4084 aspects = VK_IMAGE_ASPECT_DEPTH_BIT; 4085 else if (map_flags & PIPE_MAP_STENCIL_ONLY) 4086 aspects = VK_IMAGE_ASPECT_STENCIL_BIT; 4087 } 4088 if (!aspects) 4089 aspects = img->aspect; 4090 while (aspects) { 4091 int aspect = 1 << u_bit_scan(&aspects); 4092 region.imageSubresource.aspectMask = aspect; 4093 4094 /* this may or may not work with multisampled depth/stencil buffers depending on the driver implementation: 4095 * 4096 * srcImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT 4097 * - vkCmdCopyImageToBuffer spec 4098 * 4099 * dstImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT 4100 * - vkCmdCopyBufferToImage spec 4101 */ 4102 if (buf2img) 4103 VKCTX(CmdCopyBufferToImage)(cmdbuf, buf->obj->buffer, img->obj->image, img->layout, 1, ®ion); 4104 else 4105 VKCTX(CmdCopyImageToBuffer)(cmdbuf, img->obj->image, img->layout, buf->obj->buffer, 1, ®ion); 4106 } 4107 if (needs_present_readback) 4108 zink_kopper_present_readback(ctx, img); 4109} 4110 4111static void 4112zink_resource_copy_region(struct pipe_context *pctx, 4113 struct pipe_resource *pdst, 4114 unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz, 4115 struct pipe_resource *psrc, 4116 unsigned src_level, const struct pipe_box *src_box) 4117{ 4118 struct zink_resource *dst = zink_resource(pdst); 4119 struct zink_resource *src = zink_resource(psrc); 4120 struct zink_context *ctx = zink_context(pctx); 4121 if (dst->base.b.target != PIPE_BUFFER && src->base.b.target != PIPE_BUFFER) { 4122 VkImageCopy region = {0}; 4123 if (util_format_get_num_planes(src->base.b.format) == 1 && 4124 util_format_get_num_planes(dst->base.b.format) == 1) { 4125 /* If neither the calling command’s srcImage nor the calling command’s dstImage 4126 * has a multi-planar image format then the aspectMask member of srcSubresource 4127 * and dstSubresource must match 4128 * 4129 * -VkImageCopy spec 4130 */ 4131 assert(src->aspect == dst->aspect); 4132 } else 4133 unreachable("planar formats not yet handled"); 4134 4135 zink_fb_clears_apply_or_discard(ctx, pdst, (struct u_rect){dstx, dstx + src_box->width, dsty, dsty + src_box->height}, false); 4136 zink_fb_clears_apply_region(ctx, psrc, zink_rect_from_box(src_box)); 4137 4138 region.srcSubresource.aspectMask = src->aspect; 4139 region.srcSubresource.mipLevel = src_level; 4140 enum pipe_texture_target src_target = src->base.b.target; 4141 if (src->need_2D) 4142 src_target = src_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY; 4143 switch (src_target) { 4144 case PIPE_TEXTURE_CUBE: 4145 case PIPE_TEXTURE_CUBE_ARRAY: 4146 case PIPE_TEXTURE_2D_ARRAY: 4147 case PIPE_TEXTURE_1D_ARRAY: 4148 /* these use layer */ 4149 region.srcSubresource.baseArrayLayer = src_box->z; 4150 region.srcSubresource.layerCount = src_box->depth; 4151 region.srcOffset.z = 0; 4152 region.extent.depth = 1; 4153 break; 4154 case PIPE_TEXTURE_3D: 4155 /* this uses depth */ 4156 region.srcSubresource.baseArrayLayer = 0; 4157 region.srcSubresource.layerCount = 1; 4158 region.srcOffset.z = src_box->z; 4159 region.extent.depth = src_box->depth; 4160 break; 4161 default: 4162 /* these must only copy one layer */ 4163 region.srcSubresource.baseArrayLayer = 0; 4164 region.srcSubresource.layerCount = 1; 4165 region.srcOffset.z = 0; 4166 region.extent.depth = 1; 4167 } 4168 4169 region.srcOffset.x = src_box->x; 4170 region.srcOffset.y = src_box->y; 4171 4172 region.dstSubresource.aspectMask = dst->aspect; 4173 region.dstSubresource.mipLevel = dst_level; 4174 enum pipe_texture_target dst_target = dst->base.b.target; 4175 if (dst->need_2D) 4176 dst_target = dst_target == PIPE_TEXTURE_1D ? PIPE_TEXTURE_2D : PIPE_TEXTURE_2D_ARRAY; 4177 switch (dst_target) { 4178 case PIPE_TEXTURE_CUBE: 4179 case PIPE_TEXTURE_CUBE_ARRAY: 4180 case PIPE_TEXTURE_2D_ARRAY: 4181 case PIPE_TEXTURE_1D_ARRAY: 4182 /* these use layer */ 4183 region.dstSubresource.baseArrayLayer = dstz; 4184 region.dstSubresource.layerCount = src_box->depth; 4185 region.dstOffset.z = 0; 4186 break; 4187 case PIPE_TEXTURE_3D: 4188 /* this uses depth */ 4189 region.dstSubresource.baseArrayLayer = 0; 4190 region.dstSubresource.layerCount = 1; 4191 region.dstOffset.z = dstz; 4192 break; 4193 default: 4194 /* these must only copy one layer */ 4195 region.dstSubresource.baseArrayLayer = 0; 4196 region.dstSubresource.layerCount = 1; 4197 region.dstOffset.z = 0; 4198 } 4199 4200 region.dstOffset.x = dstx; 4201 region.dstOffset.y = dsty; 4202 region.extent.width = src_box->width; 4203 region.extent.height = src_box->height; 4204 4205 struct zink_batch *batch = &ctx->batch; 4206 zink_resource_setup_transfer_layouts(ctx, src, dst); 4207 VkCommandBuffer cmdbuf = zink_get_cmdbuf(ctx, src, dst); 4208 zink_batch_reference_resource_rw(batch, src, false); 4209 zink_batch_reference_resource_rw(batch, dst, true); 4210 4211 VKCTX(CmdCopyImage)(cmdbuf, src->obj->image, src->layout, 4212 dst->obj->image, dst->layout, 4213 1, ®ion); 4214 } else if (dst->base.b.target == PIPE_BUFFER && 4215 src->base.b.target == PIPE_BUFFER) { 4216 zink_copy_buffer(ctx, dst, src, dstx, src_box->x, src_box->width); 4217 } else 4218 zink_copy_image_buffer(ctx, dst, src, dst_level, dstx, dsty, dstz, src_level, src_box, 0); 4219} 4220 4221static bool 4222zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsigned level, struct pipe_box *box, bool commit) 4223{ 4224 struct zink_context *ctx = zink_context(pctx); 4225 struct zink_resource *res = zink_resource(pres); 4226 struct zink_screen *screen = zink_screen(pctx->screen); 4227 4228 /* if any current usage exists, flush the queue */ 4229 if (zink_resource_has_unflushed_usage(res)) 4230 zink_flush_queue(ctx); 4231 4232 VkSemaphore sem = VK_NULL_HANDLE; 4233 bool ret = zink_bo_commit(screen, res, level, box, commit, &sem); 4234 if (ret) { 4235 if (sem) 4236 zink_batch_add_wait_semaphore(&ctx->batch, sem); 4237 } else { 4238 check_device_lost(ctx); 4239 } 4240 4241 return ret; 4242} 4243 4244static void 4245rebind_image(struct zink_context *ctx, struct zink_resource *res) 4246{ 4247 zink_rebind_framebuffer(ctx, res); 4248 if (!zink_resource_has_binds(res)) 4249 return; 4250 for (unsigned i = 0; i < PIPE_SHADER_TYPES; i++) { 4251 if (res->sampler_binds[i]) { 4252 for (unsigned j = 0; j < ctx->di.num_sampler_views[i]; j++) { 4253 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[i][j]); 4254 if (sv && sv->base.texture == &res->base.b) { 4255 struct pipe_surface *psurf = &sv->image_view->base; 4256 zink_rebind_surface(ctx, &psurf); 4257 sv->image_view = zink_surface(psurf); 4258 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j, 1); 4259 update_descriptor_state_sampler(ctx, i, j, res); 4260 } 4261 } 4262 } 4263 if (!res->image_bind_count[i == PIPE_SHADER_COMPUTE]) 4264 continue; 4265 for (unsigned j = 0; j < ctx->di.num_images[i]; j++) { 4266 if (zink_resource(ctx->image_views[i][j].base.resource) == res) { 4267 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_IMAGE, j, 1); 4268 update_descriptor_state_image(ctx, i, j, res); 4269 _mesa_set_add(ctx->need_barriers[i == PIPE_SHADER_COMPUTE], res); 4270 } 4271 } 4272 } 4273} 4274 4275bool 4276zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res) 4277{ 4278 if (res->base.b.target == PIPE_BUFFER) { 4279 /* force counter buffer reset */ 4280 res->so_valid = false; 4281 return rebind_buffer(ctx, res, 0, 0) == res->bind_count[0] + res->bind_count[1]; 4282 } 4283 rebind_image(ctx, res); 4284 return false; 4285} 4286 4287void 4288zink_rebind_all_buffers(struct zink_context *ctx) 4289{ 4290 struct zink_batch *batch = &ctx->batch; 4291 ctx->vertex_buffers_dirty = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask > 0; 4292 ctx->dirty_so_targets = ctx->num_so_targets > 0; 4293 if (ctx->num_so_targets) 4294 zink_resource_buffer_barrier(ctx, zink_resource(ctx->dummy_xfb_buffer), 4295 VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT, VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT); 4296 for (unsigned shader = PIPE_SHADER_VERTEX; shader < PIPE_SHADER_TYPES; shader++) { 4297 for (unsigned slot = 0; slot < ctx->di.num_ubos[shader]; slot++) { 4298 struct zink_resource *res = rebind_ubo(ctx, shader, slot); 4299 if (res) 4300 zink_batch_resource_usage_set(batch, res, false); 4301 } 4302 for (unsigned slot = 0; slot < ctx->di.num_sampler_views[shader]; slot++) { 4303 struct zink_resource *res = rebind_tbo(ctx, shader, slot); 4304 if (res) 4305 zink_batch_resource_usage_set(batch, res, false); 4306 } 4307 for (unsigned slot = 0; slot < ctx->di.num_ssbos[shader]; slot++) { 4308 struct zink_resource *res = rebind_ssbo(ctx, shader, slot); 4309 if (res) 4310 zink_batch_resource_usage_set(batch, res, (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0); 4311 } 4312 for (unsigned slot = 0; slot < ctx->di.num_images[shader]; slot++) { 4313 struct zink_resource *res = rebind_ibo(ctx, shader, slot); 4314 if (res) 4315 zink_batch_resource_usage_set(batch, res, (ctx->image_views[shader][slot].base.access & PIPE_IMAGE_ACCESS_WRITE) != 0); 4316 } 4317 } 4318} 4319 4320void 4321zink_rebind_all_images(struct zink_context *ctx) 4322{ 4323 rebind_fb_state(ctx, NULL, false); 4324 for (unsigned i = 0; i < PIPE_SHADER_TYPES; i++) { 4325 for (unsigned j = 0; j < ctx->di.num_sampler_views[i]; j++) { 4326 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[i][j]); 4327 if (!sv) 4328 continue; 4329 struct zink_resource *res = zink_resource(sv->image_view->base.texture); 4330 if (res->obj != sv->image_view->obj) { 4331 struct pipe_surface *psurf = &sv->image_view->base; 4332 zink_rebind_surface(ctx, &psurf); 4333 sv->image_view = zink_surface(psurf); 4334 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j, 1); 4335 update_descriptor_state_sampler(ctx, i, j, res); 4336 } 4337 } 4338 for (unsigned j = 0; j < ctx->di.num_images[i]; j++) { 4339 struct zink_image_view *image_view = &ctx->image_views[i][j]; 4340 struct zink_resource *res = zink_resource(image_view->base.resource); 4341 if (!res) 4342 continue; 4343 if (ctx->image_views[i][j].surface->obj != res->obj) { 4344 zink_surface_reference(zink_screen(ctx->base.screen), &image_view->surface, NULL); 4345 image_view->surface = create_image_surface(ctx, &image_view->base, i == PIPE_SHADER_COMPUTE); 4346 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_IMAGE, j, 1); 4347 update_descriptor_state_image(ctx, i, j, res); 4348 _mesa_set_add(ctx->need_barriers[i == PIPE_SHADER_COMPUTE], res); 4349 } 4350 } 4351 } 4352} 4353 4354static void 4355zink_context_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resource *dst, 4356 struct pipe_resource *src, unsigned num_rebinds, 4357 uint32_t rebind_mask, uint32_t delete_buffer_id) 4358{ 4359 struct zink_resource *d = zink_resource(dst); 4360 struct zink_resource *s = zink_resource(src); 4361 struct zink_context *ctx = zink_context(pctx); 4362 struct zink_screen *screen = zink_screen(pctx->screen); 4363 4364 assert(d->internal_format == s->internal_format); 4365 assert(d->obj); 4366 assert(s->obj); 4367 util_idalloc_mt_free(&screen->buffer_ids, delete_buffer_id); 4368 zink_descriptor_set_refs_clear(&d->obj->desc_set_refs, d->obj); 4369 /* add a ref just like check_resource_for_batch_ref() would've */ 4370 if (zink_resource_has_binds(d) && zink_resource_has_usage(d)) 4371 zink_batch_reference_resource(&ctx->batch, d); 4372 /* don't be too creative */ 4373 zink_resource_object_reference(screen, &d->obj, s->obj); 4374 /* force counter buffer reset */ 4375 d->so_valid = false; 4376 if (num_rebinds && rebind_buffer(ctx, d, rebind_mask, num_rebinds) < num_rebinds) 4377 ctx->buffer_rebind_counter = p_atomic_inc_return(&screen->buffer_rebind_counter); 4378} 4379 4380static bool 4381zink_context_is_resource_busy(struct pipe_screen *pscreen, struct pipe_resource *pres, unsigned usage) 4382{ 4383 struct zink_screen *screen = zink_screen(pscreen); 4384 struct zink_resource *res = zink_resource(pres); 4385 uint32_t check_usage = 0; 4386 if (usage & PIPE_MAP_READ) 4387 check_usage |= ZINK_RESOURCE_ACCESS_WRITE; 4388 if (usage & PIPE_MAP_WRITE) 4389 check_usage |= ZINK_RESOURCE_ACCESS_RW; 4390 return !zink_resource_usage_check_completion(screen, res, check_usage); 4391} 4392 4393static void 4394zink_emit_string_marker(struct pipe_context *pctx, 4395 const char *string, int len) 4396{ 4397 struct zink_screen *screen = zink_screen(pctx->screen); 4398 struct zink_batch *batch = &zink_context(pctx)->batch; 4399 4400 /* make sure string is nul-terminated */ 4401 char buf[512], *temp = NULL; 4402 if (len < ARRAY_SIZE(buf)) { 4403 memcpy(buf, string, len); 4404 buf[len] = '\0'; 4405 string = buf; 4406 } else 4407 string = temp = strndup(string, len); 4408 4409 VkDebugUtilsLabelEXT label = { 4410 VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT, NULL, 4411 string, 4412 { 0 } 4413 }; 4414 screen->vk.CmdInsertDebugUtilsLabelEXT(batch->state->cmdbuf, &label); 4415 free(temp); 4416} 4417 4418struct pipe_context * 4419zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags) 4420{ 4421 struct zink_screen *screen = zink_screen(pscreen); 4422 struct zink_context *ctx = rzalloc(NULL, struct zink_context); 4423 bool is_copy_only = (flags & ZINK_CONTEXT_COPY_ONLY) > 0; 4424 if (!ctx) 4425 goto fail; 4426 4427 ctx->flags = flags; 4428 ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true; 4429 ctx->gfx_pipeline_state.dirty = true; 4430 ctx->gfx_pipeline_state.dyn_state2.vertices_per_patch = 1; 4431 ctx->gfx_pipeline_state.uses_dynamic_stride = screen->info.have_EXT_extended_dynamic_state || 4432 screen->info.have_EXT_vertex_input_dynamic_state; 4433 ctx->compute_pipeline_state.dirty = true; 4434 ctx->fb_changed = ctx->rp_changed = true; 4435 ctx->gfx_pipeline_state.gfx_prim_mode = PIPE_PRIM_MAX; 4436 4437 zink_init_draw_functions(ctx, screen); 4438 zink_init_grid_functions(ctx); 4439 4440 ctx->base.screen = pscreen; 4441 ctx->base.priv = priv; 4442 4443 ctx->base.destroy = zink_context_destroy; 4444 ctx->base.get_device_reset_status = zink_get_device_reset_status; 4445 ctx->base.set_device_reset_callback = zink_set_device_reset_callback; 4446 4447 zink_context_state_init(&ctx->base); 4448 4449 ctx->base.create_sampler_state = zink_create_sampler_state; 4450 ctx->base.bind_sampler_states = zink_bind_sampler_states; 4451 ctx->base.delete_sampler_state = zink_delete_sampler_state; 4452 4453 ctx->base.create_sampler_view = zink_create_sampler_view; 4454 ctx->base.set_sampler_views = zink_set_sampler_views; 4455 ctx->base.sampler_view_destroy = zink_sampler_view_destroy; 4456 ctx->base.get_sample_position = zink_get_sample_position; 4457 ctx->base.set_sample_locations = zink_set_sample_locations; 4458 4459 zink_program_init(ctx); 4460 4461 ctx->base.set_polygon_stipple = zink_set_polygon_stipple; 4462 ctx->base.set_vertex_buffers = zink_set_vertex_buffers; 4463 ctx->base.set_viewport_states = zink_set_viewport_states; 4464 ctx->base.set_scissor_states = zink_set_scissor_states; 4465 ctx->base.set_inlinable_constants = zink_set_inlinable_constants; 4466 ctx->base.set_constant_buffer = zink_set_constant_buffer; 4467 ctx->base.set_shader_buffers = zink_set_shader_buffers; 4468 ctx->base.set_shader_images = zink_set_shader_images; 4469 ctx->base.set_framebuffer_state = zink_set_framebuffer_state; 4470 ctx->base.set_stencil_ref = zink_set_stencil_ref; 4471 ctx->base.set_clip_state = zink_set_clip_state; 4472 ctx->base.set_blend_color = zink_set_blend_color; 4473 ctx->base.set_tess_state = zink_set_tess_state; 4474 ctx->base.set_patch_vertices = zink_set_patch_vertices; 4475 4476 ctx->base.set_sample_mask = zink_set_sample_mask; 4477 ctx->gfx_pipeline_state.sample_mask = UINT32_MAX; 4478 4479 ctx->base.clear = zink_clear; 4480 ctx->base.clear_texture = zink_clear_texture; 4481 ctx->base.clear_buffer = zink_clear_buffer; 4482 ctx->base.clear_render_target = zink_clear_render_target; 4483 ctx->base.clear_depth_stencil = zink_clear_depth_stencil; 4484 4485 ctx->base.create_fence_fd = zink_create_fence_fd; 4486 ctx->base.fence_server_sync = zink_fence_server_sync; 4487 ctx->base.fence_server_signal = zink_fence_server_signal; 4488 ctx->base.flush = zink_flush; 4489 ctx->base.memory_barrier = zink_memory_barrier; 4490 ctx->base.texture_barrier = zink_texture_barrier; 4491 ctx->base.evaluate_depth_buffer = zink_evaluate_depth_buffer; 4492 4493 ctx->base.resource_commit = zink_resource_commit; 4494 ctx->base.resource_copy_region = zink_resource_copy_region; 4495 ctx->base.blit = zink_blit; 4496 ctx->base.create_stream_output_target = zink_create_stream_output_target; 4497 ctx->base.stream_output_target_destroy = zink_stream_output_target_destroy; 4498 4499 ctx->base.set_stream_output_targets = zink_set_stream_output_targets; 4500 ctx->base.flush_resource = zink_flush_resource; 4501 4502 ctx->base.emit_string_marker = zink_emit_string_marker; 4503 4504 zink_context_surface_init(&ctx->base); 4505 zink_context_resource_init(&ctx->base); 4506 zink_context_query_init(&ctx->base); 4507 4508 list_inithead(&ctx->query_pools); 4509 _mesa_set_init(&ctx->update_barriers[0][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal); 4510 _mesa_set_init(&ctx->update_barriers[1][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal); 4511 _mesa_set_init(&ctx->update_barriers[0][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal); 4512 _mesa_set_init(&ctx->update_barriers[1][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal); 4513 ctx->need_barriers[0] = &ctx->update_barriers[0][0]; 4514 ctx->need_barriers[1] = &ctx->update_barriers[1][0]; 4515 4516 util_dynarray_init(&ctx->free_batch_states, ctx); 4517 4518 ctx->gfx_pipeline_state.have_EXT_extended_dynamic_state = screen->info.have_EXT_extended_dynamic_state; 4519 ctx->gfx_pipeline_state.have_EXT_extended_dynamic_state2 = screen->info.have_EXT_extended_dynamic_state2; 4520 ctx->gfx_pipeline_state.extendedDynamicState2PatchControlPoints = screen->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints; 4521 4522 slab_create_child(&ctx->transfer_pool, &screen->transfer_pool); 4523 slab_create_child(&ctx->transfer_pool_unsync, &screen->transfer_pool); 4524 4525 ctx->base.stream_uploader = u_upload_create_default(&ctx->base); 4526 ctx->base.const_uploader = u_upload_create_default(&ctx->base); 4527 for (int i = 0; i < ARRAY_SIZE(ctx->fb_clears); i++) 4528 util_dynarray_init(&ctx->fb_clears[i].clears, ctx); 4529 4530 if (!is_copy_only) { 4531 ctx->blitter = util_blitter_create(&ctx->base); 4532 if (!ctx->blitter) 4533 goto fail; 4534 } 4535 4536 ctx->gfx_pipeline_state.shader_keys.last_vertex.key.vs_base.last_vertex_stage = true; 4537 ctx->last_vertex_stage_dirty = true; 4538 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_TESS_CTRL].key.tcs.patch_vertices = 1; 4539 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_VERTEX].size = sizeof(struct zink_vs_key_base); 4540 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_TESS_EVAL].size = sizeof(struct zink_vs_key_base); 4541 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_TESS_CTRL].size = sizeof(struct zink_tcs_key); 4542 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_GEOMETRY].size = sizeof(struct zink_vs_key_base); 4543 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_FRAGMENT].size = sizeof(struct zink_fs_key); 4544 _mesa_hash_table_init(&ctx->compute_program_cache, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal); 4545 _mesa_hash_table_init(&ctx->framebuffer_cache, ctx, hash_framebuffer_imageless, equals_framebuffer_imageless); 4546 if (!zink_init_render_pass(ctx)) 4547 goto fail; 4548 _mesa_set_init(&ctx->rendering_state_cache, ctx, hash_rendering_state, equals_rendering_state); 4549 ctx->dynamic_fb.info.pColorAttachments = ctx->dynamic_fb.attachments; 4550 ctx->dynamic_fb.info.sType = VK_STRUCTURE_TYPE_RENDERING_INFO; 4551 for (unsigned i = 0; i < ARRAY_SIZE(ctx->dynamic_fb.attachments); i++) { 4552 VkRenderingAttachmentInfo *att = &ctx->dynamic_fb.attachments[i]; 4553 att->sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO; 4554 att->imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; 4555 att->storeOp = VK_ATTACHMENT_STORE_OP_STORE; 4556 } 4557 ctx->gfx_pipeline_state.rendering_info.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO; 4558 ctx->gfx_pipeline_state.rendering_info.pColorAttachmentFormats = ctx->gfx_pipeline_state.rendering_formats; 4559 4560 const uint32_t data[] = {0}; 4561 if (!is_copy_only) { 4562 ctx->dummy_vertex_buffer = pipe_buffer_create(&screen->base, 4563 PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_SHADER_IMAGE, PIPE_USAGE_IMMUTABLE, sizeof(data)); 4564 if (!ctx->dummy_vertex_buffer) 4565 goto fail; 4566 ctx->dummy_xfb_buffer = pipe_buffer_create(&screen->base, 4567 PIPE_BIND_STREAM_OUTPUT, PIPE_USAGE_IMMUTABLE, sizeof(data)); 4568 if (!ctx->dummy_xfb_buffer) 4569 goto fail; 4570 for (unsigned i = 0; i < ARRAY_SIZE(ctx->dummy_surface); i++) { 4571 if (!(screen->info.props.limits.framebufferDepthSampleCounts & BITFIELD_BIT(i))) 4572 continue; 4573 ctx->dummy_surface[i] = zink_surface_create_null(ctx, PIPE_TEXTURE_2D, 1024, 1024, BITFIELD_BIT(i)); 4574 if (!ctx->dummy_surface[i]) 4575 goto fail; 4576 } 4577 VkBufferViewCreateInfo bvci = create_bvci(ctx, zink_resource(ctx->dummy_vertex_buffer), PIPE_FORMAT_R8G8B8A8_UNORM, 0, sizeof(data)); 4578 ctx->dummy_bufferview = get_buffer_view(ctx, zink_resource(ctx->dummy_vertex_buffer), &bvci); 4579 if (!ctx->dummy_bufferview) 4580 goto fail; 4581 4582 if (!zink_descriptor_layouts_init(ctx)) 4583 goto fail; 4584 4585 if (!screen->descriptors_init(ctx)) { 4586 zink_screen_init_descriptor_funcs(screen, true); 4587 if (!screen->descriptors_init(ctx)) 4588 goto fail; 4589 } 4590 4591 ctx->base.create_texture_handle = zink_create_texture_handle; 4592 ctx->base.delete_texture_handle = zink_delete_texture_handle; 4593 ctx->base.make_texture_handle_resident = zink_make_texture_handle_resident; 4594 ctx->base.create_image_handle = zink_create_image_handle; 4595 ctx->base.delete_image_handle = zink_delete_image_handle; 4596 ctx->base.make_image_handle_resident = zink_make_image_handle_resident; 4597 for (unsigned i = 0; i < 2; i++) { 4598 _mesa_hash_table_init(&ctx->di.bindless[i].img_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal); 4599 _mesa_hash_table_init(&ctx->di.bindless[i].tex_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal); 4600 4601 /* allocate 1024 slots and reserve slot 0 */ 4602 util_idalloc_init(&ctx->di.bindless[i].tex_slots, ZINK_MAX_BINDLESS_HANDLES); 4603 util_idalloc_alloc(&ctx->di.bindless[i].tex_slots); 4604 util_idalloc_init(&ctx->di.bindless[i].img_slots, ZINK_MAX_BINDLESS_HANDLES); 4605 util_idalloc_alloc(&ctx->di.bindless[i].img_slots); 4606 ctx->di.bindless[i].buffer_infos = malloc(sizeof(VkBufferView) * ZINK_MAX_BINDLESS_HANDLES); 4607 ctx->di.bindless[i].img_infos = malloc(sizeof(VkDescriptorImageInfo) * ZINK_MAX_BINDLESS_HANDLES); 4608 util_dynarray_init(&ctx->di.bindless[i].updates, NULL); 4609 util_dynarray_init(&ctx->di.bindless[i].resident, NULL); 4610 } 4611 } 4612 4613 zink_start_batch(ctx, &ctx->batch); 4614 if (!ctx->batch.state) 4615 goto fail; 4616 4617 if (!is_copy_only) { 4618 pipe_buffer_write_nooverlap(&ctx->base, ctx->dummy_vertex_buffer, 0, sizeof(data), data); 4619 pipe_buffer_write_nooverlap(&ctx->base, ctx->dummy_xfb_buffer, 0, sizeof(data), data); 4620 4621 for (unsigned i = 0; i < PIPE_SHADER_TYPES; i++) { 4622 /* need to update these based on screen config for null descriptors */ 4623 for (unsigned j = 0; j < 32; j++) { 4624 update_descriptor_state_ubo(ctx, i, j, NULL); 4625 update_descriptor_state_sampler(ctx, i, j, NULL); 4626 update_descriptor_state_ssbo(ctx, i, j, NULL); 4627 update_descriptor_state_image(ctx, i, j, NULL); 4628 } 4629 } 4630 if (!screen->info.rb2_feats.nullDescriptor) 4631 ctx->di.fbfetch.imageView = zink_csurface(ctx->dummy_surface[0])->image_view; 4632 4633 reapply_color_write(ctx); 4634 p_atomic_inc(&screen->base.num_contexts); 4635 } 4636 4637 zink_select_draw_vbo(ctx); 4638 zink_select_launch_grid(ctx); 4639 4640 /* set on startup just to avoid validation errors if a draw comes through without 4641 * a tess shader later 4642 */ 4643 if (screen->info.dynamic_state2_feats.extendedDynamicState2PatchControlPoints) 4644 VKCTX(CmdSetPatchControlPointsEXT)(ctx->batch.state->cmdbuf, 1); 4645 4646 if (!(flags & PIPE_CONTEXT_PREFER_THREADED) || flags & PIPE_CONTEXT_COMPUTE_ONLY) { 4647 return &ctx->base; 4648 } 4649 4650 struct threaded_context *tc = (struct threaded_context*)threaded_context_create(&ctx->base, &screen->transfer_pool, 4651 zink_context_replace_buffer_storage, 4652 &(struct threaded_context_options){ 4653 .create_fence = zink_create_tc_fence_for_tc, 4654 .is_resource_busy = zink_context_is_resource_busy, 4655 .driver_calls_flush_notify = true, 4656 .unsynchronized_get_device_reset_status = true, 4657 }, 4658 &ctx->tc); 4659 4660 if (tc && (struct zink_context*)tc != ctx) { 4661 threaded_context_init_bytes_mapped_limit(tc, 4); 4662 ctx->base.set_context_param = zink_set_context_param; 4663 } 4664 4665 return (struct pipe_context*)tc; 4666 4667fail: 4668 if (ctx) 4669 zink_context_destroy(&ctx->base); 4670 return NULL; 4671} 4672 4673struct zink_context * 4674zink_tc_context_unwrap(struct pipe_context *pctx) 4675{ 4676 struct zink_context *ctx = zink_context(pctx); 4677 struct zink_screen *screen = zink_screen(ctx->base.screen); 4678 /* need to get the actual zink_context, not the threaded context */ 4679 if (screen->threaded) 4680 pctx = threaded_context_unwrap_sync(pctx); 4681 pctx = trace_get_possibly_threaded_context(pctx); 4682 return zink_context(pctx); 4683} 4684