1/* 2 * Copyright © 2016 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 */ 23 24#include <assert.h> 25#include <stdbool.h> 26 27#include "nir/nir_builder.h" 28#include "radv_meta.h" 29#include "radv_private.h" 30#include "sid.h" 31#include "vk_format.h" 32 33/* emit 0, 0, 0, 1 */ 34static nir_shader * 35build_nir_fs(struct radv_device *dev) 36{ 37 const struct glsl_type *vec4 = glsl_vec4_type(); 38 nir_variable *f_color; /* vec4, fragment output color */ 39 40 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_resolve_fs"); 41 42 f_color = nir_variable_create(b.shader, nir_var_shader_out, vec4, "f_color"); 43 f_color->data.location = FRAG_RESULT_DATA0; 44 nir_store_var(&b, f_color, nir_imm_vec4(&b, 0.0, 0.0, 0.0, 1.0), 0xf); 45 46 return b.shader; 47} 48 49static VkResult 50create_pipeline(struct radv_device *device, VkShaderModule vs_module_h, VkFormat format, 51 VkPipeline *pipeline) 52{ 53 VkResult result; 54 VkDevice device_h = radv_device_to_handle(device); 55 56 nir_shader *fs_module = build_nir_fs(device); 57 if (!fs_module) { 58 /* XXX: Need more accurate error */ 59 result = VK_ERROR_OUT_OF_HOST_MEMORY; 60 goto cleanup; 61 } 62 63 VkPipelineLayoutCreateInfo pl_create_info = { 64 .sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO, 65 .setLayoutCount = 0, 66 .pSetLayouts = NULL, 67 .pushConstantRangeCount = 0, 68 .pPushConstantRanges = NULL, 69 }; 70 71 if (!device->meta_state.resolve.p_layout) { 72 result = 73 radv_CreatePipelineLayout(radv_device_to_handle(device), &pl_create_info, 74 &device->meta_state.alloc, &device->meta_state.resolve.p_layout); 75 if (result != VK_SUCCESS) 76 goto cleanup; 77 } 78 79 VkFormat color_formats[2] = { format, format }; 80 const VkPipelineRenderingCreateInfo rendering_create_info = { 81 .sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO, 82 .colorAttachmentCount = 2, 83 .pColorAttachmentFormats = color_formats, 84 }; 85 86 result = radv_graphics_pipeline_create( 87 device_h, radv_pipeline_cache_to_handle(&device->meta_state.cache), 88 &(VkGraphicsPipelineCreateInfo){ 89 .sType = VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, 90 .pNext = &rendering_create_info, 91 .stageCount = 2, 92 .pStages = 93 (VkPipelineShaderStageCreateInfo[]){ 94 { 95 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 96 .stage = VK_SHADER_STAGE_VERTEX_BIT, 97 .module = vs_module_h, 98 .pName = "main", 99 }, 100 { 101 .sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, 102 .stage = VK_SHADER_STAGE_FRAGMENT_BIT, 103 .module = vk_shader_module_handle_from_nir(fs_module), 104 .pName = "main", 105 }, 106 }, 107 .pVertexInputState = 108 &(VkPipelineVertexInputStateCreateInfo){ 109 .sType = VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO, 110 .vertexBindingDescriptionCount = 0, 111 .vertexAttributeDescriptionCount = 0, 112 }, 113 .pInputAssemblyState = 114 &(VkPipelineInputAssemblyStateCreateInfo){ 115 .sType = VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO, 116 .topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, 117 .primitiveRestartEnable = false, 118 }, 119 .pViewportState = 120 &(VkPipelineViewportStateCreateInfo){ 121 .sType = VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO, 122 .viewportCount = 1, 123 .scissorCount = 1, 124 }, 125 .pRasterizationState = 126 &(VkPipelineRasterizationStateCreateInfo){ 127 .sType = VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO, 128 .depthClampEnable = false, 129 .rasterizerDiscardEnable = false, 130 .polygonMode = VK_POLYGON_MODE_FILL, 131 .cullMode = VK_CULL_MODE_NONE, 132 .frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE, 133 }, 134 .pMultisampleState = 135 &(VkPipelineMultisampleStateCreateInfo){ 136 .sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO, 137 .rasterizationSamples = 1, 138 .sampleShadingEnable = false, 139 .pSampleMask = NULL, 140 .alphaToCoverageEnable = false, 141 .alphaToOneEnable = false, 142 }, 143 .pColorBlendState = 144 &(VkPipelineColorBlendStateCreateInfo){ 145 .sType = VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO, 146 .logicOpEnable = false, 147 .attachmentCount = 2, 148 .pAttachments = 149 (VkPipelineColorBlendAttachmentState[]){ 150 { 151 .colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | 152 VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT, 153 }, 154 { 155 .colorWriteMask = 0, 156 157 }}, 158 }, 159 .pDynamicState = 160 &(VkPipelineDynamicStateCreateInfo){ 161 .sType = VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO, 162 .dynamicStateCount = 2, 163 .pDynamicStates = 164 (VkDynamicState[]){ 165 VK_DYNAMIC_STATE_VIEWPORT, 166 VK_DYNAMIC_STATE_SCISSOR, 167 }, 168 }, 169 .layout = device->meta_state.resolve.p_layout, 170 .renderPass = VK_NULL_HANDLE, 171 .subpass = 0, 172 }, 173 &(struct radv_graphics_pipeline_create_info){ 174 .use_rectlist = true, 175 .custom_blend_mode = V_028808_CB_RESOLVE, 176 }, 177 &device->meta_state.alloc, pipeline); 178 if (result != VK_SUCCESS) 179 goto cleanup; 180 181 goto cleanup; 182 183cleanup: 184 ralloc_free(fs_module); 185 return result; 186} 187 188void 189radv_device_finish_meta_resolve_state(struct radv_device *device) 190{ 191 struct radv_meta_state *state = &device->meta_state; 192 193 for (uint32_t j = 0; j < NUM_META_FS_KEYS; j++) { 194 radv_DestroyPipeline(radv_device_to_handle(device), state->resolve.pipeline[j], 195 &state->alloc); 196 } 197 radv_DestroyPipelineLayout(radv_device_to_handle(device), state->resolve.p_layout, 198 &state->alloc); 199} 200 201VkResult 202radv_device_init_meta_resolve_state(struct radv_device *device, bool on_demand) 203{ 204 if (on_demand) 205 return VK_SUCCESS; 206 207 VkResult res = VK_SUCCESS; 208 struct radv_meta_state *state = &device->meta_state; 209 nir_shader *vs_module = radv_meta_build_nir_vs_generate_vertices(device); 210 if (!vs_module) { 211 /* XXX: Need more accurate error */ 212 res = VK_ERROR_OUT_OF_HOST_MEMORY; 213 goto cleanup; 214 } 215 216 for (uint32_t i = 0; i < NUM_META_FS_KEYS; ++i) { 217 VkFormat format = radv_fs_key_format_exemplars[i]; 218 unsigned fs_key = radv_format_meta_fs_key(device, format); 219 220 VkShaderModule vs_module_h = vk_shader_module_handle_from_nir(vs_module); 221 res = create_pipeline(device, vs_module_h, format, &state->resolve.pipeline[fs_key]); 222 if (res != VK_SUCCESS) 223 goto cleanup; 224 } 225 226cleanup: 227 ralloc_free(vs_module); 228 229 return res; 230} 231 232static void 233emit_resolve(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image, 234 const struct radv_image *dst_image, VkFormat vk_format, const VkOffset2D *dest_offset, 235 const VkExtent2D *resolve_extent) 236{ 237 struct radv_device *device = cmd_buffer->device; 238 VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer); 239 unsigned fs_key = radv_format_meta_fs_key(device, vk_format); 240 241 cmd_buffer->state.flush_bits |= 242 radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, src_image) | 243 radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, src_image) | 244 radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_image); 245 246 radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS, 247 device->meta_state.resolve.pipeline[fs_key]); 248 249 radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, 250 &(VkViewport){.x = dest_offset->x, 251 .y = dest_offset->y, 252 .width = resolve_extent->width, 253 .height = resolve_extent->height, 254 .minDepth = 0.0f, 255 .maxDepth = 1.0f}); 256 257 radv_CmdSetScissor(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1, 258 &(VkRect2D){ 259 .offset = *dest_offset, 260 .extent = *resolve_extent, 261 }); 262 263 radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0); 264 cmd_buffer->state.flush_bits |= 265 radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_image); 266} 267 268enum radv_resolve_method { 269 RESOLVE_HW, 270 RESOLVE_COMPUTE, 271 RESOLVE_FRAGMENT, 272}; 273 274static bool 275image_hw_resolve_compat(const struct radv_device *device, struct radv_image *src_image, 276 struct radv_image *dst_image) 277{ 278 if (device->physical_device->rad_info.gfx_level >= GFX9) { 279 return dst_image->planes[0].surface.u.gfx9.swizzle_mode == 280 src_image->planes[0].surface.u.gfx9.swizzle_mode; 281 } else { 282 return dst_image->planes[0].surface.micro_tile_mode == 283 src_image->planes[0].surface.micro_tile_mode; 284 } 285} 286 287static void 288radv_pick_resolve_method_images(struct radv_device *device, struct radv_image *src_image, 289 VkFormat src_format, struct radv_image *dest_image, 290 unsigned dest_level, VkImageLayout dest_image_layout, 291 bool dest_render_loop, struct radv_cmd_buffer *cmd_buffer, 292 enum radv_resolve_method *method) 293 294{ 295 uint32_t queue_mask = radv_image_queue_family_mask(dest_image, cmd_buffer->qf, 296 cmd_buffer->qf); 297 298 if (vk_format_is_color(src_format)) { 299 /* Using the fragment resolve path is currently a hint to 300 * avoid decompressing DCC for partial resolves and 301 * re-initialize it after resolving using compute. 302 * TODO: Add support for layered and int to the fragment path. 303 */ 304 if (radv_layout_dcc_compressed(device, dest_image, dest_level, dest_image_layout, 305 dest_render_loop, queue_mask)) { 306 *method = RESOLVE_FRAGMENT; 307 } else if (!image_hw_resolve_compat(device, src_image, dest_image)) { 308 /* The micro tile mode only needs to match for the HW 309 * resolve path which is the default path for non-DCC 310 * resolves. 311 */ 312 *method = RESOLVE_COMPUTE; 313 } 314 315 if (src_format == VK_FORMAT_R16G16_UNORM || src_format == VK_FORMAT_R16G16_SNORM) 316 *method = RESOLVE_COMPUTE; 317 else if (vk_format_is_int(src_format)) 318 *method = RESOLVE_COMPUTE; 319 else if (src_image->info.array_size > 1 || dest_image->info.array_size > 1) 320 *method = RESOLVE_COMPUTE; 321 } else { 322 if (src_image->info.array_size > 1 || dest_image->info.array_size > 1) 323 *method = RESOLVE_COMPUTE; 324 else 325 *method = RESOLVE_FRAGMENT; 326 } 327} 328 329static VkResult 330build_resolve_pipeline(struct radv_device *device, unsigned fs_key) 331{ 332 VkResult result = VK_SUCCESS; 333 334 if (device->meta_state.resolve.pipeline[fs_key]) 335 return result; 336 337 mtx_lock(&device->meta_state.mtx); 338 if (device->meta_state.resolve.pipeline[fs_key]) { 339 mtx_unlock(&device->meta_state.mtx); 340 return result; 341 } 342 343 nir_shader *vs_module = radv_meta_build_nir_vs_generate_vertices(device); 344 345 VkShaderModule vs_module_h = vk_shader_module_handle_from_nir(vs_module); 346 result = create_pipeline(device, vs_module_h, radv_fs_key_format_exemplars[fs_key], 347 &device->meta_state.resolve.pipeline[fs_key]); 348 349 ralloc_free(vs_module); 350 mtx_unlock(&device->meta_state.mtx); 351 return result; 352} 353 354static void 355radv_meta_resolve_hardware_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, 356 VkImageLayout src_image_layout, struct radv_image *dst_image, 357 VkImageLayout dst_image_layout, const VkImageResolve2 *region) 358{ 359 struct radv_device *device = cmd_buffer->device; 360 struct radv_meta_saved_state saved_state; 361 362 radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE); 363 364 assert(src_image->info.samples > 1); 365 assert(dst_image->info.samples == 1); 366 367 unsigned fs_key = radv_format_meta_fs_key(device, dst_image->vk.format); 368 369 /* From the Vulkan 1.0 spec: 370 * 371 * - The aspectMask member of srcSubresource and dstSubresource must 372 * only contain VK_IMAGE_ASPECT_COLOR_BIT 373 * 374 * - The layerCount member of srcSubresource and dstSubresource must 375 * match 376 */ 377 assert(region->srcSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT); 378 assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT); 379 assert(region->srcSubresource.layerCount == region->dstSubresource.layerCount); 380 381 const uint32_t src_base_layer = 382 radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset); 383 384 const uint32_t dst_base_layer = 385 radv_meta_get_iview_layer(dst_image, ®ion->dstSubresource, ®ion->dstOffset); 386 387 /** 388 * From Vulkan 1.0.6 spec: 18.6 Resolving Multisample Images 389 * 390 * extent is the size in texels of the source image to resolve in width, 391 * height and depth. 1D images use only x and width. 2D images use x, y, 392 * width and height. 3D images use x, y, z, width, height and depth. 393 * 394 * srcOffset and dstOffset select the initial x, y, and z offsets in 395 * texels of the sub-regions of the source and destination image data. 396 * extent is the size in texels of the source image to resolve in width, 397 * height and depth. 1D images use only x and width. 2D images use x, y, 398 * width and height. 3D images use x, y, z, width, height and depth. 399 */ 400 const struct VkExtent3D extent = vk_image_sanitize_extent(&src_image->vk, region->extent); 401 const struct VkOffset3D dstOffset = vk_image_sanitize_offset(&dst_image->vk, region->dstOffset); 402 403 uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, 404 cmd_buffer->qf); 405 406 if (radv_layout_dcc_compressed(cmd_buffer->device, dst_image, region->dstSubresource.mipLevel, 407 dst_image_layout, false, queue_mask)) { 408 VkImageSubresourceRange range = { 409 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, 410 .baseMipLevel = region->dstSubresource.mipLevel, 411 .levelCount = 1, 412 .baseArrayLayer = dst_base_layer, 413 .layerCount = region->dstSubresource.layerCount, 414 }; 415 416 cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, dst_image, &range, 0xffffffff); 417 } 418 419 for (uint32_t layer = 0; layer < region->srcSubresource.layerCount; ++layer) { 420 421 VkResult ret = build_resolve_pipeline(device, fs_key); 422 if (ret != VK_SUCCESS) { 423 cmd_buffer->record_result = ret; 424 break; 425 } 426 427 struct radv_image_view src_iview; 428 radv_image_view_init(&src_iview, cmd_buffer->device, 429 &(VkImageViewCreateInfo){ 430 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, 431 .image = radv_image_to_handle(src_image), 432 .viewType = radv_meta_get_view_type(src_image), 433 .format = src_image->vk.format, 434 .subresourceRange = 435 { 436 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, 437 .baseMipLevel = region->srcSubresource.mipLevel, 438 .levelCount = 1, 439 .baseArrayLayer = src_base_layer + layer, 440 .layerCount = 1, 441 }, 442 }, 443 0, NULL); 444 445 struct radv_image_view dst_iview; 446 radv_image_view_init(&dst_iview, cmd_buffer->device, 447 &(VkImageViewCreateInfo){ 448 .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, 449 .image = radv_image_to_handle(dst_image), 450 .viewType = radv_meta_get_view_type(dst_image), 451 .format = dst_image->vk.format, 452 .subresourceRange = 453 { 454 .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, 455 .baseMipLevel = region->dstSubresource.mipLevel, 456 .levelCount = 1, 457 .baseArrayLayer = dst_base_layer + layer, 458 .layerCount = 1, 459 }, 460 }, 461 0, NULL); 462 463 const VkRenderingAttachmentInfo color_atts[2] = { 464 { 465 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, 466 .imageView = radv_image_view_to_handle(&src_iview), 467 .imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, 468 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, 469 .storeOp = VK_ATTACHMENT_STORE_OP_STORE, 470 }, 471 { 472 .sType = VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO, 473 .imageView = radv_image_view_to_handle(&dst_iview), 474 .imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, 475 .loadOp = VK_ATTACHMENT_LOAD_OP_LOAD, 476 .storeOp = VK_ATTACHMENT_STORE_OP_STORE, 477 }, 478 }; 479 480 const VkRenderingInfo rendering_info = { 481 .sType = VK_STRUCTURE_TYPE_RENDERING_INFO, 482 .renderArea = { 483 .offset = { dstOffset.x, dstOffset.y }, 484 .extent = { extent.width, extent.height }, 485 }, 486 .layerCount = 1, 487 .colorAttachmentCount = 2, 488 .pColorAttachments = color_atts, 489 }; 490 491 radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info); 492 493 emit_resolve(cmd_buffer, src_image, dst_image, dst_iview.vk.format, 494 &(VkOffset2D){ 495 .x = dstOffset.x, 496 .y = dstOffset.y, 497 }, 498 &(VkExtent2D){ 499 .width = extent.width, 500 .height = extent.height, 501 }); 502 503 radv_CmdEndRendering(radv_cmd_buffer_to_handle(cmd_buffer)); 504 505 radv_image_view_finish(&src_iview); 506 radv_image_view_finish(&dst_iview); 507 } 508 509 radv_meta_restore(&saved_state, cmd_buffer); 510} 511 512static void 513resolve_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, 514 VkImageLayout src_image_layout, struct radv_image *dst_image, 515 VkImageLayout dst_image_layout, const VkImageResolve2 *region, 516 enum radv_resolve_method resolve_method) 517{ 518 switch (resolve_method) { 519 case RESOLVE_HW: 520 radv_meta_resolve_hardware_image(cmd_buffer, src_image, src_image_layout, dst_image, 521 dst_image_layout, region); 522 break; 523 case RESOLVE_FRAGMENT: 524 radv_meta_resolve_fragment_image(cmd_buffer, src_image, src_image_layout, dst_image, 525 dst_image_layout, region); 526 break; 527 case RESOLVE_COMPUTE: 528 radv_meta_resolve_compute_image(cmd_buffer, src_image, src_image->vk.format, src_image_layout, 529 dst_image, dst_image->vk.format, dst_image_layout, region); 530 break; 531 default: 532 assert(!"Invalid resolve method selected"); 533 } 534} 535 536VKAPI_ATTR void VKAPI_CALL 537radv_CmdResolveImage2(VkCommandBuffer commandBuffer, 538 const VkResolveImageInfo2 *pResolveImageInfo) 539{ 540 RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer); 541 RADV_FROM_HANDLE(radv_image, src_image, pResolveImageInfo->srcImage); 542 RADV_FROM_HANDLE(radv_image, dst_image, pResolveImageInfo->dstImage); 543 VkImageLayout src_image_layout = pResolveImageInfo->srcImageLayout; 544 VkImageLayout dst_image_layout = pResolveImageInfo->dstImageLayout; 545 const struct radv_physical_device *pdevice = cmd_buffer->device->physical_device; 546 enum radv_resolve_method resolve_method = 547 pdevice->rad_info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW; 548 549 /* we can use the hw resolve only for single full resolves */ 550 if (pResolveImageInfo->regionCount == 1) { 551 if (pResolveImageInfo->pRegions[0].srcOffset.x || 552 pResolveImageInfo->pRegions[0].srcOffset.y || pResolveImageInfo->pRegions[0].srcOffset.z) 553 resolve_method = RESOLVE_COMPUTE; 554 if (pResolveImageInfo->pRegions[0].dstOffset.x || 555 pResolveImageInfo->pRegions[0].dstOffset.y || pResolveImageInfo->pRegions[0].dstOffset.z) 556 resolve_method = RESOLVE_COMPUTE; 557 558 if (pResolveImageInfo->pRegions[0].extent.width != src_image->info.width || 559 pResolveImageInfo->pRegions[0].extent.height != src_image->info.height || 560 pResolveImageInfo->pRegions[0].extent.depth != src_image->info.depth) 561 resolve_method = RESOLVE_COMPUTE; 562 } else 563 resolve_method = RESOLVE_COMPUTE; 564 565 for (uint32_t r = 0; r < pResolveImageInfo->regionCount; r++) { 566 const VkImageResolve2 *region = &pResolveImageInfo->pRegions[r]; 567 568 radv_pick_resolve_method_images(cmd_buffer->device, src_image, src_image->vk.format, dst_image, 569 region->dstSubresource.mipLevel, dst_image_layout, false, 570 cmd_buffer, &resolve_method); 571 572 resolve_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region, 573 resolve_method); 574 } 575} 576 577static void 578radv_cmd_buffer_resolve_subpass_hw(struct radv_cmd_buffer *cmd_buffer) 579{ 580 struct vk_framebuffer *fb = cmd_buffer->state.framebuffer; 581 const struct radv_subpass *subpass = cmd_buffer->state.subpass; 582 struct radv_meta_saved_state saved_state; 583 584 radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE); 585 586 for (uint32_t i = 0; i < subpass->color_count; ++i) { 587 struct radv_subpass_attachment src_att = subpass->color_attachments[i]; 588 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i]; 589 590 if (dest_att.attachment == VK_ATTACHMENT_UNUSED) 591 continue; 592 593 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview; 594 struct radv_image *src_img = src_iview->image; 595 596 struct radv_image_view *dest_iview = cmd_buffer->state.attachments[dest_att.attachment].iview; 597 struct radv_image *dst_img = dest_iview->image; 598 VkImageLayout dst_image_layout = cmd_buffer->state.attachments[dest_att.attachment].current_layout; 599 600 uint32_t queue_mask = radv_image_queue_family_mask(dst_img, cmd_buffer->qf, 601 cmd_buffer->qf); 602 603 if (radv_layout_dcc_compressed(cmd_buffer->device, dst_img, dest_iview->vk.base_mip_level, 604 dst_image_layout, false, queue_mask)) { 605 VkImageSubresourceRange range = { 606 .aspectMask = dest_iview->vk.aspects, 607 .baseMipLevel = dest_iview->vk.base_mip_level, 608 .levelCount = dest_iview->vk.level_count, 609 .baseArrayLayer = dest_iview->vk.base_array_layer, 610 .layerCount = dest_iview->vk.layer_count, 611 }; 612 613 cmd_buffer->state.flush_bits |= radv_init_dcc(cmd_buffer, dst_img, &range, 0xffffffff); 614 cmd_buffer->state.attachments[dest_att.attachment].current_layout = 615 VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; 616 } 617 618 struct radv_subpass resolve_subpass = { 619 .color_count = 2, 620 .color_attachments = (struct radv_subpass_attachment[]){src_att, dest_att}, 621 .depth_stencil_attachment = NULL, 622 }; 623 624 radv_cmd_buffer_set_subpass(cmd_buffer, &resolve_subpass); 625 626 VkResult ret = build_resolve_pipeline( 627 cmd_buffer->device, radv_format_meta_fs_key(cmd_buffer->device, dest_iview->vk.format)); 628 if (ret != VK_SUCCESS) { 629 cmd_buffer->record_result = ret; 630 continue; 631 } 632 633 emit_resolve(cmd_buffer, src_img, dst_img, dest_iview->vk.format, &(VkOffset2D){0, 0}, 634 &(VkExtent2D){fb->width, fb->height}); 635 636 radv_cmd_buffer_restore_subpass(cmd_buffer, subpass); 637 } 638 639 radv_meta_restore(&saved_state, cmd_buffer); 640} 641 642/** 643 * Emit any needed resolves for the current subpass. 644 */ 645void 646radv_cmd_buffer_resolve_subpass(struct radv_cmd_buffer *cmd_buffer) 647{ 648 const struct radv_physical_device *pdevice = cmd_buffer->device->physical_device; 649 const struct radv_subpass *subpass = cmd_buffer->state.subpass; 650 enum radv_resolve_method resolve_method = 651 pdevice->rad_info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW; 652 653 if (!subpass->has_color_resolve && !subpass->ds_resolve_attachment) 654 return; 655 656 radv_describe_begin_render_pass_resolve(cmd_buffer); 657 658 if (subpass->ds_resolve_attachment) { 659 struct radv_subpass_attachment src_att = *subpass->depth_stencil_attachment; 660 struct radv_subpass_attachment dst_att = *subpass->ds_resolve_attachment; 661 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview; 662 struct radv_image_view *dst_iview = cmd_buffer->state.attachments[dst_att.attachment].iview; 663 664 /* Make sure to not clear the depth/stencil attachment after resolves. */ 665 cmd_buffer->state.attachments[dst_att.attachment].pending_clear_aspects = 0; 666 667 radv_pick_resolve_method_images(cmd_buffer->device, src_iview->image, src_iview->vk.format, 668 dst_iview->image, dst_iview->vk.base_mip_level, dst_att.layout, 669 dst_att.in_render_loop, cmd_buffer, &resolve_method); 670 671 if ((src_iview->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT) && 672 subpass->depth_resolve_mode != VK_RESOLVE_MODE_NONE) { 673 if (resolve_method == RESOLVE_FRAGMENT) { 674 radv_depth_stencil_resolve_subpass_fs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT, 675 subpass->depth_resolve_mode); 676 } else { 677 assert(resolve_method == RESOLVE_COMPUTE); 678 radv_depth_stencil_resolve_subpass_cs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT, 679 subpass->depth_resolve_mode); 680 } 681 } 682 683 if ((src_iview->vk.aspects & VK_IMAGE_ASPECT_STENCIL_BIT) && 684 subpass->stencil_resolve_mode != VK_RESOLVE_MODE_NONE) { 685 if (resolve_method == RESOLVE_FRAGMENT) { 686 radv_depth_stencil_resolve_subpass_fs(cmd_buffer, VK_IMAGE_ASPECT_STENCIL_BIT, 687 subpass->stencil_resolve_mode); 688 } else { 689 assert(resolve_method == RESOLVE_COMPUTE); 690 radv_depth_stencil_resolve_subpass_cs(cmd_buffer, VK_IMAGE_ASPECT_STENCIL_BIT, 691 subpass->stencil_resolve_mode); 692 } 693 } 694 695 /* From the Vulkan spec 1.2.165: 696 * 697 * "VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT specifies 698 * write access to a color, resolve, or depth/stencil 699 * resolve attachment during a render pass or via 700 * certain subpass load and store operations." 701 * 702 * Yes, it's counterintuitive but it makes sense because ds 703 * resolve operations happen late at the end of the subpass. 704 * 705 * That said, RADV is wrong because it executes the subpass 706 * end barrier *before* any subpass resolves instead of after. 707 * 708 * TODO: Fix this properly by executing subpass end barriers 709 * after subpass resolves. 710 */ 711 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB; 712 if (radv_image_has_htile(dst_iview->image)) 713 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_FLUSH_AND_INV_DB_META; 714 } 715 716 if (subpass->has_color_resolve) { 717 for (uint32_t i = 0; i < subpass->color_count; ++i) { 718 struct radv_subpass_attachment src_att = subpass->color_attachments[i]; 719 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i]; 720 721 if (dest_att.attachment == VK_ATTACHMENT_UNUSED) 722 continue; 723 724 /* Make sure to not clear color attachments after resolves. */ 725 cmd_buffer->state.attachments[dest_att.attachment].pending_clear_aspects = 0; 726 727 struct radv_image_view *dst_iview = 728 cmd_buffer->state.attachments[dest_att.attachment].iview; 729 struct radv_image *dst_img = dst_iview->image; 730 struct radv_image_view *src_iview = 731 cmd_buffer->state.attachments[src_att.attachment].iview; 732 struct radv_image *src_img = src_iview->image; 733 734 radv_pick_resolve_method_images(cmd_buffer->device, src_img, src_iview->vk.format, dst_img, 735 dst_iview->vk.base_mip_level, dest_att.layout, 736 dest_att.in_render_loop, cmd_buffer, &resolve_method); 737 738 if (resolve_method == RESOLVE_FRAGMENT) { 739 break; 740 } 741 } 742 743 switch (resolve_method) { 744 case RESOLVE_HW: 745 radv_cmd_buffer_resolve_subpass_hw(cmd_buffer); 746 break; 747 case RESOLVE_COMPUTE: 748 radv_cmd_buffer_resolve_subpass_cs(cmd_buffer); 749 break; 750 case RESOLVE_FRAGMENT: 751 radv_cmd_buffer_resolve_subpass_fs(cmd_buffer); 752 break; 753 default: 754 unreachable("Invalid resolve method"); 755 } 756 } 757 758 radv_describe_end_render_pass_resolve(cmd_buffer); 759} 760 761/** 762 * Decompress CMask/FMask before resolving a multisampled source image inside a 763 * subpass. 764 */ 765void 766radv_decompress_resolve_subpass_src(struct radv_cmd_buffer *cmd_buffer) 767{ 768 const struct radv_subpass *subpass = cmd_buffer->state.subpass; 769 struct vk_framebuffer *fb = cmd_buffer->state.framebuffer; 770 uint32_t layer_count = fb->layers; 771 772 if (subpass->view_mask) 773 layer_count = util_last_bit(subpass->view_mask); 774 775 for (uint32_t i = 0; i < subpass->color_count; ++i) { 776 struct radv_subpass_attachment src_att = subpass->color_attachments[i]; 777 struct radv_subpass_attachment dest_att = subpass->resolve_attachments[i]; 778 779 if (dest_att.attachment == VK_ATTACHMENT_UNUSED) 780 continue; 781 782 struct radv_image_view *src_iview = cmd_buffer->state.attachments[src_att.attachment].iview; 783 struct radv_image *src_image = src_iview->image; 784 785 VkImageResolve2 region = {0}; 786 region.sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2; 787 region.srcSubresource.aspectMask = src_iview->vk.aspects; 788 region.srcSubresource.mipLevel = 0; 789 region.srcSubresource.baseArrayLayer = src_iview->vk.base_array_layer; 790 region.srcSubresource.layerCount = layer_count; 791 792 radv_decompress_resolve_src(cmd_buffer, src_image, src_att.layout, ®ion); 793 } 794} 795 796static struct radv_sample_locations_state * 797radv_get_resolve_sample_locations(struct radv_cmd_buffer *cmd_buffer) 798{ 799 struct radv_cmd_state *state = &cmd_buffer->state; 800 uint32_t subpass_id = radv_get_subpass_id(cmd_buffer); 801 802 for (uint32_t i = 0; i < state->num_subpass_sample_locs; i++) { 803 if (state->subpass_sample_locs[i].subpass_idx == subpass_id) 804 return &state->subpass_sample_locs[i].sample_location; 805 } 806 807 return NULL; 808} 809 810/** 811 * Decompress CMask/FMask before resolving a multisampled source image. 812 */ 813void 814radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, 815 VkImageLayout src_image_layout, const VkImageResolve2 *region) 816{ 817 const uint32_t src_base_layer = 818 radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset); 819 820 VkImageMemoryBarrier2 barrier = { 821 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, 822 .srcStageMask = VK_PIPELINE_STAGE_2_TOP_OF_PIPE_BIT, 823 .srcAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT, 824 .dstStageMask = VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT, 825 .dstAccessMask = VK_ACCESS_2_TRANSFER_READ_BIT, 826 .oldLayout = src_image_layout, 827 .newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 828 .image = radv_image_to_handle(src_image), 829 .subresourceRange = (VkImageSubresourceRange){ 830 .aspectMask = region->srcSubresource.aspectMask, 831 .baseMipLevel = region->srcSubresource.mipLevel, 832 .levelCount = 1, 833 .baseArrayLayer = src_base_layer, 834 .layerCount = region->srcSubresource.layerCount, 835 } 836 }; 837 838 if (src_image->vk.create_flags & VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT) { 839 /* If the depth/stencil image uses different sample 840 * locations, we need them during HTILE decompressions. 841 */ 842 struct radv_sample_locations_state *sample_locs = 843 radv_get_resolve_sample_locations(cmd_buffer); 844 845 barrier.pNext = &(VkSampleLocationsInfoEXT){ 846 .sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT, 847 .sampleLocationsPerPixel = sample_locs->per_pixel, 848 .sampleLocationGridSize = sample_locs->grid_size, 849 .sampleLocationsCount = sample_locs->count, 850 .pSampleLocations = sample_locs->locations, 851 }; 852 } 853 854 VkDependencyInfo dep_info = { 855 .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO, 856 .imageMemoryBarrierCount = 1, 857 .pImageMemoryBarriers = &barrier, 858 }; 859 860 radv_CmdPipelineBarrier2(radv_cmd_buffer_to_handle(cmd_buffer), &dep_info); 861} 862