1/* 2 * Copyright © 2016 Red Hat. 3 * Copyright © 2016 Bas Nieuwenhuizen 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 * IN THE SOFTWARE. 23 */ 24#include <assert.h> 25#include <fcntl.h> 26#include <stdbool.h> 27#include <string.h> 28 29#include "util/mesa-sha1.h" 30#include "radv_private.h" 31#include "sid.h" 32#include "vk_descriptors.h" 33#include "vk_format.h" 34#include "vk_util.h" 35 36static unsigned 37radv_descriptor_type_buffer_count(VkDescriptorType type) 38{ 39 switch (type) { 40 case VK_DESCRIPTOR_TYPE_SAMPLER: 41 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK: 42 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: 43 return 0; 44 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: 45 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: 46 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: 47 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: 48 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE: 49 return 3; 50 default: 51 return 1; 52 } 53} 54 55static bool 56has_equal_immutable_samplers(const VkSampler *samplers, uint32_t count) 57{ 58 if (!samplers) 59 return false; 60 for (uint32_t i = 1; i < count; ++i) { 61 if (memcmp(radv_sampler_from_handle(samplers[0])->state, 62 radv_sampler_from_handle(samplers[i])->state, 16)) { 63 return false; 64 } 65 } 66 return true; 67} 68 69static bool 70radv_mutable_descriptor_type_size_alignment(const VkMutableDescriptorTypeListVALVE *list, 71 uint64_t *out_size, uint64_t *out_align) 72{ 73 uint32_t max_size = 0; 74 uint32_t max_align = 0; 75 76 for (uint32_t i = 0; i < list->descriptorTypeCount; i++) { 77 uint32_t size = 0; 78 uint32_t align = 0; 79 80 switch (list->pDescriptorTypes[i]) { 81 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: 82 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: 83 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: 84 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: 85 case VK_DESCRIPTOR_TYPE_SAMPLER: 86 size = 16; 87 align = 16; 88 break; 89 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: 90 size = 32; 91 align = 32; 92 break; 93 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: 94 size = 64; 95 align = 32; 96 break; 97 default: 98 return false; 99 } 100 101 max_size = MAX2(max_size, size); 102 max_align = MAX2(max_align, align); 103 } 104 105 *out_size = max_size; 106 *out_align = max_align; 107 return true; 108} 109 110VKAPI_ATTR VkResult VKAPI_CALL 111radv_CreateDescriptorSetLayout(VkDevice _device, const VkDescriptorSetLayoutCreateInfo *pCreateInfo, 112 const VkAllocationCallbacks *pAllocator, 113 VkDescriptorSetLayout *pSetLayout) 114{ 115 RADV_FROM_HANDLE(radv_device, device, _device); 116 struct radv_descriptor_set_layout *set_layout; 117 118 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO); 119 const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags = 120 vk_find_struct_const(pCreateInfo->pNext, DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO); 121 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info = 122 vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE); 123 124 uint32_t num_bindings = 0; 125 uint32_t immutable_sampler_count = 0; 126 uint32_t ycbcr_sampler_count = 0; 127 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) { 128 num_bindings = MAX2(num_bindings, pCreateInfo->pBindings[j].binding + 1); 129 if ((pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER || 130 pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) && 131 pCreateInfo->pBindings[j].pImmutableSamplers) { 132 immutable_sampler_count += pCreateInfo->pBindings[j].descriptorCount; 133 134 bool has_ycbcr_sampler = false; 135 for (unsigned i = 0; i < pCreateInfo->pBindings[j].descriptorCount; ++i) { 136 if (radv_sampler_from_handle(pCreateInfo->pBindings[j].pImmutableSamplers[i]) 137 ->ycbcr_sampler) 138 has_ycbcr_sampler = true; 139 } 140 141 if (has_ycbcr_sampler) 142 ycbcr_sampler_count += pCreateInfo->pBindings[j].descriptorCount; 143 } 144 } 145 146 uint32_t samplers_offset = offsetof(struct radv_descriptor_set_layout, binding[num_bindings]); 147 size_t size = samplers_offset + immutable_sampler_count * 4 * sizeof(uint32_t); 148 if (ycbcr_sampler_count > 0) { 149 /* Store block of offsets first, followed by the conversion descriptors (padded to the struct 150 * alignment) */ 151 size += num_bindings * sizeof(uint32_t); 152 size = ALIGN(size, alignof(struct radv_sampler_ycbcr_conversion_state)); 153 size += ycbcr_sampler_count * sizeof(struct radv_sampler_ycbcr_conversion_state); 154 } 155 156 /* We need to allocate decriptor set layouts off the device allocator with DEVICE scope because 157 * they are reference counted and may not be destroyed when vkDestroyDescriptorSetLayout is 158 * called. 159 */ 160 set_layout = vk_descriptor_set_layout_zalloc(&device->vk, size); 161 if (!set_layout) 162 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); 163 164 set_layout->flags = pCreateInfo->flags; 165 set_layout->layout_size = size; 166 167 /* We just allocate all the samplers at the end of the struct */ 168 uint32_t *samplers = (uint32_t *)&set_layout->binding[num_bindings]; 169 struct radv_sampler_ycbcr_conversion_state *ycbcr_samplers = NULL; 170 uint32_t *ycbcr_sampler_offsets = NULL; 171 172 if (ycbcr_sampler_count > 0) { 173 ycbcr_sampler_offsets = samplers + 4 * immutable_sampler_count; 174 set_layout->ycbcr_sampler_offsets_offset = (char *)ycbcr_sampler_offsets - (char *)set_layout; 175 176 uintptr_t first_ycbcr_sampler_offset = 177 (uintptr_t)ycbcr_sampler_offsets + sizeof(uint32_t) * num_bindings; 178 first_ycbcr_sampler_offset = 179 ALIGN(first_ycbcr_sampler_offset, alignof(struct radv_sampler_ycbcr_conversion_state)); 180 ycbcr_samplers = (struct radv_sampler_ycbcr_conversion_state *)first_ycbcr_sampler_offset; 181 } else 182 set_layout->ycbcr_sampler_offsets_offset = 0; 183 184 VkDescriptorSetLayoutBinding *bindings = NULL; 185 VkResult result = 186 vk_create_sorted_bindings(pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings); 187 if (result != VK_SUCCESS) { 188 vk_descriptor_set_layout_unref(&device->vk, &set_layout->vk); 189 return vk_error(device, result); 190 } 191 192 set_layout->binding_count = num_bindings; 193 set_layout->shader_stages = 0; 194 set_layout->dynamic_shader_stages = 0; 195 set_layout->has_immutable_samplers = false; 196 set_layout->size = 0; 197 198 uint32_t buffer_count = 0; 199 uint32_t dynamic_offset_count = 0; 200 201 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) { 202 const VkDescriptorSetLayoutBinding *binding = bindings + j; 203 uint32_t b = binding->binding; 204 uint32_t alignment = 0; 205 unsigned binding_buffer_count = 206 radv_descriptor_type_buffer_count(binding->descriptorType); 207 uint32_t descriptor_count = binding->descriptorCount; 208 bool has_ycbcr_sampler = false; 209 210 /* main image + fmask */ 211 uint32_t max_sampled_image_descriptors = 2; 212 213 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER && 214 binding->pImmutableSamplers) { 215 for (unsigned i = 0; i < binding->descriptorCount; ++i) { 216 struct radv_sampler_ycbcr_conversion *conversion = 217 radv_sampler_from_handle(binding->pImmutableSamplers[i])->ycbcr_sampler; 218 219 if (conversion) { 220 has_ycbcr_sampler = true; 221 max_sampled_image_descriptors = MAX2(max_sampled_image_descriptors, 222 vk_format_get_plane_count(conversion->state.format)); 223 } 224 } 225 } 226 227 switch (binding->descriptorType) { 228 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: 229 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: 230 assert(!(pCreateInfo->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR)); 231 set_layout->binding[b].dynamic_offset_count = 1; 232 set_layout->dynamic_shader_stages |= binding->stageFlags; 233 if (binding->stageFlags & RADV_RT_STAGE_BITS) 234 set_layout->dynamic_shader_stages |= VK_SHADER_STAGE_COMPUTE_BIT; 235 set_layout->binding[b].size = 0; 236 alignment = 1; 237 break; 238 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: 239 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: 240 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: 241 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: 242 set_layout->binding[b].size = 16; 243 alignment = 16; 244 break; 245 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: 246 set_layout->binding[b].size = 32; 247 alignment = 32; 248 break; 249 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: 250 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: 251 /* main descriptor + fmask descriptor */ 252 set_layout->binding[b].size = 64; 253 alignment = 32; 254 break; 255 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: 256 /* main descriptor + fmask descriptor + sampler */ 257 set_layout->binding[b].size = 96; 258 alignment = 32; 259 break; 260 case VK_DESCRIPTOR_TYPE_SAMPLER: 261 set_layout->binding[b].size = 16; 262 alignment = 16; 263 break; 264 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE: { 265 uint64_t mutable_size = 0, mutable_align = 0; 266 radv_mutable_descriptor_type_size_alignment(&mutable_info->pMutableDescriptorTypeLists[j], 267 &mutable_size, &mutable_align); 268 assert(mutable_size && mutable_align); 269 set_layout->binding[b].size = mutable_size; 270 alignment = mutable_align; 271 break; 272 } 273 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK: 274 alignment = 16; 275 set_layout->binding[b].size = descriptor_count; 276 descriptor_count = 1; 277 break; 278 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: 279 set_layout->binding[b].size = 16; 280 alignment = 16; 281 break; 282 default: 283 break; 284 } 285 286 set_layout->size = align(set_layout->size, alignment); 287 set_layout->binding[b].type = binding->descriptorType; 288 set_layout->binding[b].array_size = descriptor_count; 289 set_layout->binding[b].offset = set_layout->size; 290 set_layout->binding[b].buffer_offset = buffer_count; 291 set_layout->binding[b].dynamic_offset_offset = dynamic_offset_count; 292 293 if (variable_flags && binding->binding < variable_flags->bindingCount && 294 (variable_flags->pBindingFlags[binding->binding] & 295 VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) { 296 assert( 297 !binding->pImmutableSamplers); /* Terribly ill defined how many samplers are valid */ 298 assert(binding->binding == num_bindings - 1); 299 300 set_layout->has_variable_descriptors = true; 301 } 302 303 if ((binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER || 304 binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) && 305 binding->pImmutableSamplers) { 306 set_layout->binding[b].immutable_samplers_offset = samplers_offset; 307 set_layout->binding[b].immutable_samplers_equal = 308 has_equal_immutable_samplers(binding->pImmutableSamplers, binding->descriptorCount); 309 set_layout->has_immutable_samplers = true; 310 311 for (uint32_t i = 0; i < binding->descriptorCount; i++) 312 memcpy(samplers + 4 * i, 313 &radv_sampler_from_handle(binding->pImmutableSamplers[i])->state, 16); 314 315 /* Don't reserve space for the samplers if they're not accessed. */ 316 if (set_layout->binding[b].immutable_samplers_equal) { 317 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER && 318 max_sampled_image_descriptors <= 2) 319 set_layout->binding[b].size -= 32; 320 else if (binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) 321 set_layout->binding[b].size -= 16; 322 } 323 samplers += 4 * binding->descriptorCount; 324 samplers_offset += 4 * sizeof(uint32_t) * binding->descriptorCount; 325 326 if (has_ycbcr_sampler) { 327 ycbcr_sampler_offsets[b] = (const char *)ycbcr_samplers - (const char *)set_layout; 328 for (uint32_t i = 0; i < binding->descriptorCount; i++) { 329 if (radv_sampler_from_handle(binding->pImmutableSamplers[i])->ycbcr_sampler) 330 ycbcr_samplers[i] = 331 radv_sampler_from_handle(binding->pImmutableSamplers[i])->ycbcr_sampler->state; 332 else 333 ycbcr_samplers[i].format = VK_FORMAT_UNDEFINED; 334 } 335 ycbcr_samplers += binding->descriptorCount; 336 } 337 } 338 339 set_layout->size += descriptor_count * set_layout->binding[b].size; 340 buffer_count += descriptor_count * binding_buffer_count; 341 dynamic_offset_count += descriptor_count * set_layout->binding[b].dynamic_offset_count; 342 set_layout->shader_stages |= binding->stageFlags; 343 } 344 345 free(bindings); 346 347 set_layout->buffer_count = buffer_count; 348 set_layout->dynamic_offset_count = dynamic_offset_count; 349 350 *pSetLayout = radv_descriptor_set_layout_to_handle(set_layout); 351 352 return VK_SUCCESS; 353} 354 355VKAPI_ATTR void VKAPI_CALL 356radv_GetDescriptorSetLayoutSupport(VkDevice device, 357 const VkDescriptorSetLayoutCreateInfo *pCreateInfo, 358 VkDescriptorSetLayoutSupport *pSupport) 359{ 360 VkDescriptorSetLayoutBinding *bindings = NULL; 361 VkResult result = 362 vk_create_sorted_bindings(pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings); 363 if (result != VK_SUCCESS) { 364 pSupport->supported = false; 365 return; 366 } 367 368 const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags = 369 vk_find_struct_const(pCreateInfo->pNext, DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO); 370 VkDescriptorSetVariableDescriptorCountLayoutSupport *variable_count = vk_find_struct( 371 (void *)pCreateInfo->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_LAYOUT_SUPPORT); 372 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info = 373 vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE); 374 if (variable_count) { 375 variable_count->maxVariableDescriptorCount = 0; 376 } 377 378 bool supported = true; 379 uint64_t size = 0; 380 for (uint32_t i = 0; i < pCreateInfo->bindingCount; i++) { 381 const VkDescriptorSetLayoutBinding *binding = bindings + i; 382 383 uint64_t descriptor_size = 0; 384 uint64_t descriptor_alignment = 1; 385 uint32_t descriptor_count = binding->descriptorCount; 386 switch (binding->descriptorType) { 387 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: 388 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: 389 break; 390 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: 391 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: 392 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: 393 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: 394 descriptor_size = 16; 395 descriptor_alignment = 16; 396 break; 397 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: 398 descriptor_size = 32; 399 descriptor_alignment = 32; 400 break; 401 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: 402 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: 403 descriptor_size = 64; 404 descriptor_alignment = 32; 405 break; 406 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: 407 if (!has_equal_immutable_samplers(binding->pImmutableSamplers, descriptor_count)) { 408 descriptor_size = 64; 409 } else { 410 descriptor_size = 96; 411 } 412 descriptor_alignment = 32; 413 break; 414 case VK_DESCRIPTOR_TYPE_SAMPLER: 415 if (!has_equal_immutable_samplers(binding->pImmutableSamplers, descriptor_count)) { 416 descriptor_size = 16; 417 descriptor_alignment = 16; 418 } 419 break; 420 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK: 421 descriptor_alignment = 16; 422 descriptor_size = descriptor_count; 423 descriptor_count = 1; 424 break; 425 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE: 426 if (!radv_mutable_descriptor_type_size_alignment( 427 &mutable_info->pMutableDescriptorTypeLists[i], &descriptor_size, 428 &descriptor_alignment)) { 429 supported = false; 430 } 431 break; 432 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: 433 descriptor_size = 16; 434 descriptor_alignment = 16; 435 break; 436 default: 437 break; 438 } 439 440 if (size && !align_u64(size, descriptor_alignment)) { 441 supported = false; 442 } 443 size = align_u64(size, descriptor_alignment); 444 445 uint64_t max_count = INT32_MAX; 446 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) 447 max_count = INT32_MAX - size; 448 else if (descriptor_size) 449 max_count = (INT32_MAX - size) / descriptor_size; 450 451 if (max_count < descriptor_count) { 452 supported = false; 453 } 454 if (variable_flags && binding->binding < variable_flags->bindingCount && variable_count && 455 (variable_flags->pBindingFlags[binding->binding] & 456 VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) { 457 variable_count->maxVariableDescriptorCount = MIN2(UINT32_MAX, max_count); 458 } 459 size += descriptor_count * descriptor_size; 460 } 461 462 free(bindings); 463 464 pSupport->supported = supported; 465} 466 467/* 468 * Pipeline layouts. These have nothing to do with the pipeline. They are 469 * just multiple descriptor set layouts pasted together. 470 */ 471void 472radv_pipeline_layout_init(struct radv_device *device, struct radv_pipeline_layout *layout, 473 bool independent_sets) 474{ 475 memset(layout, 0, sizeof(*layout)); 476 477 vk_object_base_init(&device->vk, &layout->base, VK_OBJECT_TYPE_PIPELINE_LAYOUT); 478 479 layout->independent_sets = independent_sets; 480} 481 482void 483radv_pipeline_layout_add_set(struct radv_pipeline_layout *layout, uint32_t set_idx, 484 struct radv_descriptor_set_layout *set_layout) 485{ 486 unsigned dynamic_offset_count = 0; 487 488 if (layout->set[set_idx].layout) 489 return; 490 491 layout->num_sets = MAX2(set_idx + 1, layout->num_sets); 492 493 layout->set[set_idx].layout = set_layout; 494 vk_descriptor_set_layout_ref(&set_layout->vk); 495 496 for (uint32_t b = 0; b < set_layout->binding_count; b++) { 497 dynamic_offset_count += set_layout->binding[b].array_size * set_layout->binding[b].dynamic_offset_count; 498 } 499 500 layout->set[set_idx].dynamic_offset_start = layout->dynamic_offset_count; 501 502 layout->dynamic_offset_count += dynamic_offset_count; 503 layout->dynamic_shader_stages |= set_layout->dynamic_shader_stages; 504} 505 506void 507radv_pipeline_layout_hash(struct radv_pipeline_layout *layout) 508{ 509 struct mesa_sha1 ctx; 510 511 _mesa_sha1_init(&ctx); 512 for (uint32_t i = 0; i < layout->num_sets; i++) { 513 struct radv_descriptor_set_layout *set_layout = layout->set[i].layout; 514 515 if (!set_layout) 516 continue; 517 518 /* Hash the entire set layout except for the vk_object_base and the reference counter. The 519 * rest of the set layout is carefully constructed to not have pointers so a full hash instead 520 * of a per-field hash should be ok. 521 */ 522 uint32_t hash_offset = sizeof(struct vk_object_base) + sizeof(uint32_t); 523 _mesa_sha1_update(&ctx, (const char *)set_layout + hash_offset, 524 set_layout->layout_size - hash_offset); 525 } 526 _mesa_sha1_update(&ctx, &layout->push_constant_size, sizeof(layout->push_constant_size)); 527 _mesa_sha1_final(&ctx, layout->sha1); 528} 529 530void 531radv_pipeline_layout_finish(struct radv_device *device, struct radv_pipeline_layout *layout) 532{ 533 for (uint32_t i = 0; i < layout->num_sets; i++) { 534 if (!layout->set[i].layout) 535 continue; 536 537 vk_descriptor_set_layout_unref(&device->vk, &layout->set[i].layout->vk); 538 } 539 540 vk_object_base_finish(&layout->base); 541} 542 543VKAPI_ATTR VkResult VKAPI_CALL 544radv_CreatePipelineLayout(VkDevice _device, const VkPipelineLayoutCreateInfo *pCreateInfo, 545 const VkAllocationCallbacks *pAllocator, 546 VkPipelineLayout *pPipelineLayout) 547{ 548 RADV_FROM_HANDLE(radv_device, device, _device); 549 struct radv_pipeline_layout *layout; 550 551 assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO); 552 553 layout = vk_alloc2(&device->vk.alloc, pAllocator, sizeof(*layout), 8, 554 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); 555 if (layout == NULL) 556 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); 557 558 radv_pipeline_layout_init(device, layout, 559 pCreateInfo->flags & VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT); 560 561 layout->num_sets = pCreateInfo->setLayoutCount; 562 563 for (uint32_t set = 0; set < pCreateInfo->setLayoutCount; set++) { 564 RADV_FROM_HANDLE(radv_descriptor_set_layout, set_layout, pCreateInfo->pSetLayouts[set]); 565 566 /* From the Vulkan spec 1.3.211: 567 * 568 * "VUID-VkPipelineLayoutCreateInfo-flags-06562 569 * If flags: does not include VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT, elements of 570 * pSetLayouts must be valid VkDescriptorSetLayout objects" 571 */ 572 if (set_layout == NULL) { 573 assert(layout->independent_sets); 574 continue; 575 } 576 577 radv_pipeline_layout_add_set(layout, set, set_layout); 578 } 579 580 layout->push_constant_size = 0; 581 582 for (unsigned i = 0; i < pCreateInfo->pushConstantRangeCount; ++i) { 583 const VkPushConstantRange *range = pCreateInfo->pPushConstantRanges + i; 584 layout->push_constant_size = MAX2(layout->push_constant_size, range->offset + range->size); 585 } 586 587 layout->push_constant_size = align(layout->push_constant_size, 16); 588 589 radv_pipeline_layout_hash(layout); 590 591 *pPipelineLayout = radv_pipeline_layout_to_handle(layout); 592 593 return VK_SUCCESS; 594} 595 596VKAPI_ATTR void VKAPI_CALL 597radv_DestroyPipelineLayout(VkDevice _device, VkPipelineLayout _pipelineLayout, 598 const VkAllocationCallbacks *pAllocator) 599{ 600 RADV_FROM_HANDLE(radv_device, device, _device); 601 RADV_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, _pipelineLayout); 602 603 if (!pipeline_layout) 604 return; 605 606 radv_pipeline_layout_finish(device, pipeline_layout); 607 608 vk_free2(&device->vk.alloc, pAllocator, pipeline_layout); 609} 610 611static VkResult 612radv_descriptor_set_create(struct radv_device *device, struct radv_descriptor_pool *pool, 613 struct radv_descriptor_set_layout *layout, const uint32_t *variable_count, 614 struct radv_descriptor_set **out_set) 615{ 616 if (pool->entry_count == pool->max_entry_count) 617 return VK_ERROR_OUT_OF_POOL_MEMORY; 618 619 struct radv_descriptor_set *set; 620 uint32_t buffer_count = layout->buffer_count; 621 if (variable_count) { 622 unsigned stride = 623 radv_descriptor_type_buffer_count(layout->binding[layout->binding_count - 1].type); 624 buffer_count = 625 layout->binding[layout->binding_count - 1].buffer_offset + *variable_count * stride; 626 } 627 unsigned range_offset = 628 sizeof(struct radv_descriptor_set_header) + sizeof(struct radeon_winsys_bo *) * buffer_count; 629 const unsigned dynamic_offset_count = layout->dynamic_offset_count; 630 unsigned mem_size = 631 range_offset + sizeof(struct radv_descriptor_range) * dynamic_offset_count; 632 633 if (pool->host_memory_base) { 634 if (pool->host_memory_end - pool->host_memory_ptr < mem_size) 635 return VK_ERROR_OUT_OF_POOL_MEMORY; 636 637 set = (struct radv_descriptor_set *)pool->host_memory_ptr; 638 pool->host_memory_ptr += mem_size; 639 } else { 640 set = vk_alloc2(&device->vk.alloc, NULL, mem_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); 641 642 if (!set) 643 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); 644 } 645 646 memset(set, 0, mem_size); 647 648 vk_object_base_init(&device->vk, &set->header.base, VK_OBJECT_TYPE_DESCRIPTOR_SET); 649 650 if (dynamic_offset_count) { 651 set->header.dynamic_descriptors = 652 (struct radv_descriptor_range *)((uint8_t *)set + range_offset); 653 } 654 655 set->header.layout = layout; 656 set->header.buffer_count = buffer_count; 657 uint32_t layout_size = layout->size; 658 if (variable_count) { 659 uint32_t stride = layout->binding[layout->binding_count - 1].size; 660 if (layout->binding[layout->binding_count - 1].type == 661 VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) 662 stride = 1; 663 664 layout_size = layout->binding[layout->binding_count - 1].offset + *variable_count * stride; 665 } 666 layout_size = align_u32(layout_size, 32); 667 set->header.size = layout_size; 668 669 /* try to allocate linearly first, so that we don't spend 670 * time looking for gaps if the app only allocates & 671 * resets via the pool. */ 672 if (pool->current_offset + layout_size <= pool->size) { 673 set->header.bo = pool->bo; 674 set->header.mapped_ptr = (uint32_t *)(pool->mapped_ptr + pool->current_offset); 675 set->header.va = pool->bo ? (radv_buffer_get_va(set->header.bo) + pool->current_offset) : 0; 676 if (!pool->host_memory_base) { 677 pool->entries[pool->entry_count].offset = pool->current_offset; 678 pool->entries[pool->entry_count].size = layout_size; 679 } 680 pool->entries[pool->entry_count].set = set; 681 pool->current_offset += layout_size; 682 } else if (!pool->host_memory_base) { 683 uint64_t offset = 0; 684 int index; 685 686 for (index = 0; index < pool->entry_count; ++index) { 687 if (pool->entries[index].offset - offset >= layout_size) 688 break; 689 offset = pool->entries[index].offset + pool->entries[index].size; 690 } 691 692 if (pool->size - offset < layout_size) { 693 vk_free2(&device->vk.alloc, NULL, set); 694 return VK_ERROR_OUT_OF_POOL_MEMORY; 695 } 696 set->header.bo = pool->bo; 697 set->header.mapped_ptr = (uint32_t *)(pool->mapped_ptr + offset); 698 set->header.va = pool->bo ? (radv_buffer_get_va(set->header.bo) + offset) : 0; 699 memmove(&pool->entries[index + 1], &pool->entries[index], 700 sizeof(pool->entries[0]) * (pool->entry_count - index)); 701 pool->entries[index].offset = offset; 702 pool->entries[index].size = layout_size; 703 pool->entries[index].set = set; 704 } else 705 return VK_ERROR_OUT_OF_POOL_MEMORY; 706 707 if (layout->has_immutable_samplers) { 708 for (unsigned i = 0; i < layout->binding_count; ++i) { 709 if (!layout->binding[i].immutable_samplers_offset || 710 layout->binding[i].immutable_samplers_equal) 711 continue; 712 713 unsigned offset = layout->binding[i].offset / 4; 714 if (layout->binding[i].type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) 715 offset += radv_combined_image_descriptor_sampler_offset(layout->binding + i) / 4; 716 717 const uint32_t *samplers = 718 (const uint32_t *)((const char *)layout + layout->binding[i].immutable_samplers_offset); 719 for (unsigned j = 0; j < layout->binding[i].array_size; ++j) { 720 memcpy(set->header.mapped_ptr + offset, samplers + 4 * j, 16); 721 offset += layout->binding[i].size / 4; 722 } 723 } 724 } 725 726 pool->entry_count++; 727 vk_descriptor_set_layout_ref(&layout->vk); 728 *out_set = set; 729 return VK_SUCCESS; 730} 731 732static void 733radv_descriptor_set_destroy(struct radv_device *device, struct radv_descriptor_pool *pool, 734 struct radv_descriptor_set *set, bool free_bo) 735{ 736 vk_descriptor_set_layout_unref(&device->vk, &set->header.layout->vk); 737 738 if (pool->host_memory_base) 739 return; 740 741 if (free_bo && !pool->host_memory_base) { 742 for (int i = 0; i < pool->entry_count; ++i) { 743 if (pool->entries[i].set == set) { 744 memmove(&pool->entries[i], &pool->entries[i + 1], 745 sizeof(pool->entries[i]) * (pool->entry_count - i - 1)); 746 --pool->entry_count; 747 break; 748 } 749 } 750 } 751 vk_object_base_finish(&set->header.base); 752 vk_free2(&device->vk.alloc, NULL, set); 753} 754 755static void 756radv_destroy_descriptor_pool(struct radv_device *device, const VkAllocationCallbacks *pAllocator, 757 struct radv_descriptor_pool *pool) 758{ 759 for (int i = 0; i < pool->entry_count; ++i) { 760 radv_descriptor_set_destroy(device, pool, pool->entries[i].set, false); 761 } 762 763 if (pool->bo) 764 device->ws->buffer_destroy(device->ws, pool->bo); 765 if (pool->host_bo) 766 vk_free2(&device->vk.alloc, pAllocator, pool->host_bo); 767 768 vk_object_base_finish(&pool->base); 769 vk_free2(&device->vk.alloc, pAllocator, pool); 770} 771 772VKAPI_ATTR VkResult VKAPI_CALL 773radv_CreateDescriptorPool(VkDevice _device, const VkDescriptorPoolCreateInfo *pCreateInfo, 774 const VkAllocationCallbacks *pAllocator, 775 VkDescriptorPool *pDescriptorPool) 776{ 777 RADV_FROM_HANDLE(radv_device, device, _device); 778 struct radv_descriptor_pool *pool; 779 uint64_t size = sizeof(struct radv_descriptor_pool); 780 uint64_t bo_size = 0, bo_count = 0, range_count = 0; 781 782 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info = 783 vk_find_struct_const(pCreateInfo->pNext, MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE); 784 785 vk_foreach_struct_const(ext, pCreateInfo->pNext) 786 { 787 switch (ext->sType) { 788 case VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_INLINE_UNIFORM_BLOCK_CREATE_INFO: { 789 const VkDescriptorPoolInlineUniformBlockCreateInfo *info = 790 (const VkDescriptorPoolInlineUniformBlockCreateInfo *)ext; 791 /* the sizes are 4 aligned, and we need to align to at 792 * most 32, which needs at most 28 bytes extra per 793 * binding. */ 794 bo_size += 28llu * info->maxInlineUniformBlockBindings; 795 break; 796 } 797 default: 798 break; 799 } 800 } 801 802 for (unsigned i = 0; i < pCreateInfo->poolSizeCount; ++i) { 803 bo_count += radv_descriptor_type_buffer_count(pCreateInfo->pPoolSizes[i].type) * 804 pCreateInfo->pPoolSizes[i].descriptorCount; 805 806 switch (pCreateInfo->pPoolSizes[i].type) { 807 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: 808 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: 809 range_count += pCreateInfo->pPoolSizes[i].descriptorCount; 810 break; 811 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: 812 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: 813 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: 814 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: 815 case VK_DESCRIPTOR_TYPE_SAMPLER: 816 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: 817 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: 818 /* 32 as we may need to align for images */ 819 bo_size += 32 * pCreateInfo->pPoolSizes[i].descriptorCount; 820 break; 821 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: 822 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: 823 bo_size += 64 * pCreateInfo->pPoolSizes[i].descriptorCount; 824 break; 825 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE: 826 /* Per spec, if a mutable descriptor type list is provided for the pool entry, we 827 * allocate enough memory to hold any subset of that list. 828 * If there is no mutable descriptor type list available, 829 * we must allocate enough for any supported mutable descriptor type, i.e. 64 bytes. */ 830 if (mutable_info && i < mutable_info->mutableDescriptorTypeListCount) { 831 uint64_t mutable_size, mutable_alignment; 832 if (radv_mutable_descriptor_type_size_alignment( 833 &mutable_info->pMutableDescriptorTypeLists[i], &mutable_size, 834 &mutable_alignment)) { 835 /* 32 as we may need to align for images */ 836 mutable_size = align(mutable_size, 32); 837 bo_size += mutable_size * pCreateInfo->pPoolSizes[i].descriptorCount; 838 } 839 } else { 840 bo_size += 64 * pCreateInfo->pPoolSizes[i].descriptorCount; 841 } 842 break; 843 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: 844 bo_size += 96 * pCreateInfo->pPoolSizes[i].descriptorCount; 845 break; 846 case VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK: 847 bo_size += pCreateInfo->pPoolSizes[i].descriptorCount; 848 break; 849 default: 850 break; 851 } 852 } 853 854 uint64_t entries_size = sizeof(struct radv_descriptor_pool_entry) * pCreateInfo->maxSets; 855 size += entries_size; 856 857 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) { 858 uint64_t host_size = pCreateInfo->maxSets * sizeof(struct radv_descriptor_set); 859 host_size += sizeof(struct radeon_winsys_bo *) * bo_count; 860 host_size += sizeof(struct radv_descriptor_range) * range_count; 861 size += host_size; 862 } 863 864 pool = vk_alloc2(&device->vk.alloc, pAllocator, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); 865 if (!pool) 866 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); 867 868 memset(pool, 0, sizeof(*pool)); 869 870 vk_object_base_init(&device->vk, &pool->base, VK_OBJECT_TYPE_DESCRIPTOR_POOL); 871 872 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) { 873 pool->host_memory_base = (uint8_t *)pool + sizeof(struct radv_descriptor_pool) + entries_size; 874 pool->host_memory_ptr = pool->host_memory_base; 875 pool->host_memory_end = (uint8_t *)pool + size; 876 } 877 878 if (bo_size) { 879 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_HOST_ONLY_BIT_VALVE)) { 880 enum radeon_bo_flag flags = RADEON_FLAG_NO_INTERPROCESS_SHARING | RADEON_FLAG_READ_ONLY | 881 RADEON_FLAG_32BIT; 882 883 if (device->instance->zero_vram) 884 flags |= RADEON_FLAG_ZERO_VRAM; 885 886 VkResult result = device->ws->buffer_create( 887 device->ws, bo_size, 32, RADEON_DOMAIN_VRAM, flags, RADV_BO_PRIORITY_DESCRIPTOR, 0, 888 &pool->bo); 889 if (result != VK_SUCCESS) { 890 radv_destroy_descriptor_pool(device, pAllocator, pool); 891 return vk_error(device, result); 892 } 893 pool->mapped_ptr = (uint8_t *)device->ws->buffer_map(pool->bo); 894 if (!pool->mapped_ptr) { 895 radv_destroy_descriptor_pool(device, pAllocator, pool); 896 return vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY); 897 } 898 } else { 899 pool->host_bo = 900 vk_alloc2(&device->vk.alloc, pAllocator, bo_size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); 901 if (!pool->host_bo) { 902 radv_destroy_descriptor_pool(device, pAllocator, pool); 903 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); 904 } 905 pool->mapped_ptr = pool->host_bo; 906 } 907 } 908 pool->size = bo_size; 909 pool->max_entry_count = pCreateInfo->maxSets; 910 911 *pDescriptorPool = radv_descriptor_pool_to_handle(pool); 912 return VK_SUCCESS; 913} 914 915VKAPI_ATTR void VKAPI_CALL 916radv_DestroyDescriptorPool(VkDevice _device, VkDescriptorPool _pool, 917 const VkAllocationCallbacks *pAllocator) 918{ 919 RADV_FROM_HANDLE(radv_device, device, _device); 920 RADV_FROM_HANDLE(radv_descriptor_pool, pool, _pool); 921 922 if (!pool) 923 return; 924 925 radv_destroy_descriptor_pool(device, pAllocator, pool); 926} 927 928VKAPI_ATTR VkResult VKAPI_CALL 929radv_ResetDescriptorPool(VkDevice _device, VkDescriptorPool descriptorPool, 930 VkDescriptorPoolResetFlags flags) 931{ 932 RADV_FROM_HANDLE(radv_device, device, _device); 933 RADV_FROM_HANDLE(radv_descriptor_pool, pool, descriptorPool); 934 935 for (int i = 0; i < pool->entry_count; ++i) { 936 radv_descriptor_set_destroy(device, pool, pool->entries[i].set, false); 937 } 938 pool->entry_count = 0; 939 940 pool->current_offset = 0; 941 pool->host_memory_ptr = pool->host_memory_base; 942 943 return VK_SUCCESS; 944} 945 946VKAPI_ATTR VkResult VKAPI_CALL 947radv_AllocateDescriptorSets(VkDevice _device, const VkDescriptorSetAllocateInfo *pAllocateInfo, 948 VkDescriptorSet *pDescriptorSets) 949{ 950 RADV_FROM_HANDLE(radv_device, device, _device); 951 RADV_FROM_HANDLE(radv_descriptor_pool, pool, pAllocateInfo->descriptorPool); 952 953 VkResult result = VK_SUCCESS; 954 uint32_t i; 955 struct radv_descriptor_set *set = NULL; 956 957 const VkDescriptorSetVariableDescriptorCountAllocateInfo *variable_counts = vk_find_struct_const( 958 pAllocateInfo->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_ALLOCATE_INFO); 959 const uint32_t zero = 0; 960 961 /* allocate a set of buffers for each shader to contain descriptors */ 962 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) { 963 RADV_FROM_HANDLE(radv_descriptor_set_layout, layout, pAllocateInfo->pSetLayouts[i]); 964 965 const uint32_t *variable_count = NULL; 966 if (layout->has_variable_descriptors && variable_counts) { 967 if (i < variable_counts->descriptorSetCount) 968 variable_count = variable_counts->pDescriptorCounts + i; 969 else 970 variable_count = &zero; 971 } 972 973 assert(!(layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR)); 974 975 result = radv_descriptor_set_create(device, pool, layout, variable_count, &set); 976 if (result != VK_SUCCESS) 977 break; 978 979 pDescriptorSets[i] = radv_descriptor_set_to_handle(set); 980 } 981 982 if (result != VK_SUCCESS) { 983 radv_FreeDescriptorSets(_device, pAllocateInfo->descriptorPool, i, pDescriptorSets); 984 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) { 985 pDescriptorSets[i] = VK_NULL_HANDLE; 986 } 987 } 988 return result; 989} 990 991VKAPI_ATTR VkResult VKAPI_CALL 992radv_FreeDescriptorSets(VkDevice _device, VkDescriptorPool descriptorPool, uint32_t count, 993 const VkDescriptorSet *pDescriptorSets) 994{ 995 RADV_FROM_HANDLE(radv_device, device, _device); 996 RADV_FROM_HANDLE(radv_descriptor_pool, pool, descriptorPool); 997 998 for (uint32_t i = 0; i < count; i++) { 999 RADV_FROM_HANDLE(radv_descriptor_set, set, pDescriptorSets[i]); 1000 1001 if (set && !pool->host_memory_base) 1002 radv_descriptor_set_destroy(device, pool, set, true); 1003 } 1004 return VK_SUCCESS; 1005} 1006 1007static ALWAYS_INLINE void 1008write_texel_buffer_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, 1009 unsigned *dst, struct radeon_winsys_bo **buffer_list, 1010 const VkBufferView _buffer_view) 1011{ 1012 RADV_FROM_HANDLE(radv_buffer_view, buffer_view, _buffer_view); 1013 1014 if (!buffer_view) { 1015 memset(dst, 0, 4 * 4); 1016 if (!cmd_buffer) 1017 *buffer_list = NULL; 1018 return; 1019 } 1020 1021 memcpy(dst, buffer_view->state, 4 * 4); 1022 1023 if (cmd_buffer) 1024 radv_cs_add_buffer(device->ws, cmd_buffer->cs, buffer_view->bo); 1025 else 1026 *buffer_list = buffer_view->bo; 1027} 1028 1029static ALWAYS_INLINE void 1030write_buffer_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, 1031 unsigned *dst, struct radeon_winsys_bo **buffer_list, 1032 const VkDescriptorBufferInfo *buffer_info) 1033{ 1034 RADV_FROM_HANDLE(radv_buffer, buffer, buffer_info->buffer); 1035 1036 if (!buffer) { 1037 memset(dst, 0, 4 * 4); 1038 if (!cmd_buffer) 1039 *buffer_list = NULL; 1040 return; 1041 } 1042 1043 uint64_t va = radv_buffer_get_va(buffer->bo); 1044 1045 uint32_t range = vk_buffer_range(&buffer->vk, buffer_info->offset, buffer_info->range); 1046 assert(buffer->vk.size > 0 && range > 0); 1047 1048 /* robustBufferAccess is relaxed enough to allow this (in combination 1049 * with the alignment/size we return from vkGetBufferMemoryRequirements) 1050 * and this allows the shader compiler to create more efficient 8/16-bit 1051 * buffer accesses. */ 1052 range = align(range, 4); 1053 1054 va += buffer_info->offset + buffer->offset; 1055 1056 uint32_t rsrc_word3 = 1057 S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) | S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) | 1058 S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) | S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W); 1059 1060 if (device->physical_device->rad_info.gfx_level >= GFX11) { 1061 rsrc_word3 |= S_008F0C_FORMAT(V_008F0C_GFX11_FORMAT_32_FLOAT) | 1062 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW); 1063 } else if (device->physical_device->rad_info.gfx_level >= GFX10) { 1064 rsrc_word3 |= S_008F0C_FORMAT(V_008F0C_GFX10_FORMAT_32_FLOAT) | 1065 S_008F0C_OOB_SELECT(V_008F0C_OOB_SELECT_RAW) | S_008F0C_RESOURCE_LEVEL(1); 1066 } else { 1067 rsrc_word3 |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) | 1068 S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32); 1069 } 1070 1071 dst[0] = va; 1072 dst[1] = S_008F04_BASE_ADDRESS_HI(va >> 32); 1073 dst[2] = range; 1074 dst[3] = rsrc_word3; 1075 1076 if (cmd_buffer) 1077 radv_cs_add_buffer(device->ws, cmd_buffer->cs, buffer->bo); 1078 else 1079 *buffer_list = buffer->bo; 1080} 1081 1082static ALWAYS_INLINE void 1083write_block_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, void *dst, 1084 const VkWriteDescriptorSet *writeset) 1085{ 1086 const VkWriteDescriptorSetInlineUniformBlock *inline_ub = 1087 vk_find_struct_const(writeset->pNext, WRITE_DESCRIPTOR_SET_INLINE_UNIFORM_BLOCK); 1088 1089 memcpy(dst, inline_ub->pData, inline_ub->dataSize); 1090} 1091 1092static ALWAYS_INLINE void 1093write_dynamic_buffer_descriptor(struct radv_device *device, struct radv_descriptor_range *range, 1094 struct radeon_winsys_bo **buffer_list, 1095 const VkDescriptorBufferInfo *buffer_info) 1096{ 1097 RADV_FROM_HANDLE(radv_buffer, buffer, buffer_info->buffer); 1098 uint64_t va; 1099 unsigned size; 1100 1101 if (!buffer) { 1102 range->va = 0; 1103 *buffer_list = NULL; 1104 return; 1105 } 1106 1107 va = radv_buffer_get_va(buffer->bo); 1108 1109 size = vk_buffer_range(&buffer->vk, buffer_info->offset, buffer_info->range); 1110 assert(buffer->vk.size > 0 && size > 0); 1111 1112 /* robustBufferAccess is relaxed enough to allow this (in combination 1113 * with the alignment/size we return from vkGetBufferMemoryRequirements) 1114 * and this allows the shader compiler to create more efficient 8/16-bit 1115 * buffer accesses. */ 1116 size = align(size, 4); 1117 1118 va += buffer_info->offset + buffer->offset; 1119 range->va = va; 1120 range->size = size; 1121 1122 *buffer_list = buffer->bo; 1123} 1124 1125static ALWAYS_INLINE void 1126write_image_descriptor(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, 1127 unsigned size, unsigned *dst, struct radeon_winsys_bo **buffer_list, 1128 VkDescriptorType descriptor_type, const VkDescriptorImageInfo *image_info) 1129{ 1130 RADV_FROM_HANDLE(radv_image_view, iview, image_info->imageView); 1131 union radv_descriptor *descriptor; 1132 1133 if (!iview) { 1134 memset(dst, 0, size); 1135 if (!cmd_buffer) 1136 *buffer_list = NULL; 1137 return; 1138 } 1139 1140 if (descriptor_type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) { 1141 descriptor = &iview->storage_descriptor; 1142 } else { 1143 descriptor = &iview->descriptor; 1144 } 1145 assert(size > 0); 1146 1147 memcpy(dst, descriptor, size); 1148 1149 const uint32_t max_bindings = sizeof(iview->image->bindings) / 1150 sizeof(iview->image->bindings[0]); 1151 for (uint32_t b = 0; b < max_bindings; b++) { 1152 if (cmd_buffer) { 1153 if (iview->image->bindings[b].bo) 1154 radv_cs_add_buffer(device->ws, cmd_buffer->cs, iview->image->bindings[b].bo); 1155 } else { 1156 *buffer_list = iview->image->bindings[b].bo; 1157 buffer_list++; 1158 } 1159 } 1160} 1161 1162static ALWAYS_INLINE void 1163write_combined_image_sampler_descriptor(struct radv_device *device, 1164 struct radv_cmd_buffer *cmd_buffer, unsigned sampler_offset, 1165 unsigned *dst, struct radeon_winsys_bo **buffer_list, 1166 VkDescriptorType descriptor_type, 1167 const VkDescriptorImageInfo *image_info, bool has_sampler) 1168{ 1169 write_image_descriptor(device, cmd_buffer, sampler_offset, dst, buffer_list, descriptor_type, 1170 image_info); 1171 /* copy over sampler state */ 1172 if (has_sampler) { 1173 RADV_FROM_HANDLE(radv_sampler, sampler, image_info->sampler); 1174 memcpy(dst + sampler_offset / sizeof(*dst), sampler->state, 16); 1175 } 1176} 1177 1178static ALWAYS_INLINE void 1179write_sampler_descriptor(struct radv_device *device, unsigned *dst, 1180 const VkDescriptorImageInfo *image_info) 1181{ 1182 RADV_FROM_HANDLE(radv_sampler, sampler, image_info->sampler); 1183 1184 memcpy(dst, sampler->state, 16); 1185} 1186 1187static ALWAYS_INLINE void 1188write_accel_struct(void *ptr, VkAccelerationStructureKHR _accel_struct) 1189{ 1190 RADV_FROM_HANDLE(radv_acceleration_structure, accel_struct, _accel_struct); 1191 uint64_t va = accel_struct ? radv_accel_struct_get_va(accel_struct) : 0; 1192 memcpy(ptr, &va, sizeof(va)); 1193} 1194 1195static ALWAYS_INLINE void 1196radv_update_descriptor_sets_impl(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, 1197 VkDescriptorSet dstSetOverride, uint32_t descriptorWriteCount, 1198 const VkWriteDescriptorSet *pDescriptorWrites, 1199 uint32_t descriptorCopyCount, 1200 const VkCopyDescriptorSet *pDescriptorCopies) 1201{ 1202 uint32_t i, j; 1203 for (i = 0; i < descriptorWriteCount; i++) { 1204 const VkWriteDescriptorSet *writeset = &pDescriptorWrites[i]; 1205 RADV_FROM_HANDLE(radv_descriptor_set, set, 1206 dstSetOverride ? dstSetOverride : writeset->dstSet); 1207 const struct radv_descriptor_set_binding_layout *binding_layout = 1208 set->header.layout->binding + writeset->dstBinding; 1209 uint32_t *ptr = set->header.mapped_ptr; 1210 struct radeon_winsys_bo **buffer_list = set->descriptors; 1211 /* Immutable samplers are not copied into push descriptors when they are 1212 * allocated, so if we are writing push descriptors we have to copy the 1213 * immutable samplers into them now. 1214 */ 1215 const bool copy_immutable_samplers = cmd_buffer && 1216 binding_layout->immutable_samplers_offset && 1217 !binding_layout->immutable_samplers_equal; 1218 const uint32_t *samplers = radv_immutable_samplers(set->header.layout, binding_layout); 1219 const VkWriteDescriptorSetAccelerationStructureKHR *accel_structs = NULL; 1220 1221 ptr += binding_layout->offset / 4; 1222 1223 if (writeset->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) { 1224 write_block_descriptor(device, cmd_buffer, (uint8_t *)ptr + writeset->dstArrayElement, 1225 writeset); 1226 continue; 1227 } else if (writeset->descriptorType == VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR) { 1228 accel_structs = 1229 vk_find_struct_const(writeset->pNext, WRITE_DESCRIPTOR_SET_ACCELERATION_STRUCTURE_KHR); 1230 } 1231 1232 ptr += binding_layout->size * writeset->dstArrayElement / 4; 1233 buffer_list += binding_layout->buffer_offset; 1234 buffer_list += writeset->dstArrayElement; 1235 for (j = 0; j < writeset->descriptorCount; ++j) { 1236 switch (writeset->descriptorType) { 1237 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: 1238 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: { 1239 unsigned idx = writeset->dstArrayElement + j; 1240 idx += binding_layout->dynamic_offset_offset; 1241 assert(!(set->header.layout->flags & 1242 VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR)); 1243 write_dynamic_buffer_descriptor(device, set->header.dynamic_descriptors + idx, 1244 buffer_list, writeset->pBufferInfo + j); 1245 break; 1246 } 1247 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: 1248 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: 1249 write_buffer_descriptor(device, cmd_buffer, ptr, buffer_list, 1250 writeset->pBufferInfo + j); 1251 break; 1252 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: 1253 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: 1254 write_texel_buffer_descriptor(device, cmd_buffer, ptr, buffer_list, 1255 writeset->pTexelBufferView[j]); 1256 break; 1257 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: 1258 write_image_descriptor(device, cmd_buffer, 32, ptr, buffer_list, 1259 writeset->descriptorType, writeset->pImageInfo + j); 1260 break; 1261 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: 1262 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: 1263 write_image_descriptor(device, cmd_buffer, 64, ptr, buffer_list, 1264 writeset->descriptorType, writeset->pImageInfo + j); 1265 break; 1266 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: { 1267 unsigned sampler_offset = radv_combined_image_descriptor_sampler_offset(binding_layout); 1268 write_combined_image_sampler_descriptor( 1269 device, cmd_buffer, sampler_offset, ptr, buffer_list, writeset->descriptorType, 1270 writeset->pImageInfo + j, !binding_layout->immutable_samplers_offset); 1271 if (copy_immutable_samplers) { 1272 const unsigned idx = writeset->dstArrayElement + j; 1273 memcpy((char *)ptr + sampler_offset, samplers + 4 * idx, 16); 1274 } 1275 break; 1276 } 1277 case VK_DESCRIPTOR_TYPE_SAMPLER: 1278 if (!binding_layout->immutable_samplers_offset) { 1279 write_sampler_descriptor(device, ptr, writeset->pImageInfo + j); 1280 } else if (copy_immutable_samplers) { 1281 unsigned idx = writeset->dstArrayElement + j; 1282 memcpy(ptr, samplers + 4 * idx, 16); 1283 } 1284 break; 1285 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: 1286 write_accel_struct(ptr, accel_structs->pAccelerationStructures[j]); 1287 break; 1288 default: 1289 break; 1290 } 1291 ptr += binding_layout->size / 4; 1292 ++buffer_list; 1293 } 1294 } 1295 1296 for (i = 0; i < descriptorCopyCount; i++) { 1297 const VkCopyDescriptorSet *copyset = &pDescriptorCopies[i]; 1298 RADV_FROM_HANDLE(radv_descriptor_set, src_set, copyset->srcSet); 1299 RADV_FROM_HANDLE(radv_descriptor_set, dst_set, copyset->dstSet); 1300 const struct radv_descriptor_set_binding_layout *src_binding_layout = 1301 src_set->header.layout->binding + copyset->srcBinding; 1302 const struct radv_descriptor_set_binding_layout *dst_binding_layout = 1303 dst_set->header.layout->binding + copyset->dstBinding; 1304 uint32_t *src_ptr = src_set->header.mapped_ptr; 1305 uint32_t *dst_ptr = dst_set->header.mapped_ptr; 1306 struct radeon_winsys_bo **src_buffer_list = src_set->descriptors; 1307 struct radeon_winsys_bo **dst_buffer_list = dst_set->descriptors; 1308 1309 src_ptr += src_binding_layout->offset / 4; 1310 dst_ptr += dst_binding_layout->offset / 4; 1311 1312 if (src_binding_layout->type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) { 1313 src_ptr += copyset->srcArrayElement / 4; 1314 dst_ptr += copyset->dstArrayElement / 4; 1315 1316 memcpy(dst_ptr, src_ptr, copyset->descriptorCount); 1317 continue; 1318 } 1319 1320 src_ptr += src_binding_layout->size * copyset->srcArrayElement / 4; 1321 dst_ptr += dst_binding_layout->size * copyset->dstArrayElement / 4; 1322 1323 src_buffer_list += src_binding_layout->buffer_offset; 1324 src_buffer_list += copyset->srcArrayElement; 1325 1326 dst_buffer_list += dst_binding_layout->buffer_offset; 1327 dst_buffer_list += copyset->dstArrayElement; 1328 1329 /* In case of copies between mutable descriptor types 1330 * and non-mutable descriptor types. */ 1331 size_t copy_size = MIN2(src_binding_layout->size, dst_binding_layout->size); 1332 1333 for (j = 0; j < copyset->descriptorCount; ++j) { 1334 switch (src_binding_layout->type) { 1335 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: 1336 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: { 1337 unsigned src_idx = copyset->srcArrayElement + j; 1338 unsigned dst_idx = copyset->dstArrayElement + j; 1339 struct radv_descriptor_range *src_range, *dst_range; 1340 src_idx += src_binding_layout->dynamic_offset_offset; 1341 dst_idx += dst_binding_layout->dynamic_offset_offset; 1342 1343 src_range = src_set->header.dynamic_descriptors + src_idx; 1344 dst_range = dst_set->header.dynamic_descriptors + dst_idx; 1345 *dst_range = *src_range; 1346 break; 1347 } 1348 default: 1349 memcpy(dst_ptr, src_ptr, copy_size); 1350 } 1351 src_ptr += src_binding_layout->size / 4; 1352 dst_ptr += dst_binding_layout->size / 4; 1353 1354 unsigned src_buffer_count = radv_descriptor_type_buffer_count(src_binding_layout->type); 1355 unsigned dst_buffer_count = radv_descriptor_type_buffer_count(dst_binding_layout->type); 1356 for (unsigned k = 0; k < dst_buffer_count; k++) { 1357 if (k < src_buffer_count) 1358 dst_buffer_list[k] = src_buffer_list[k]; 1359 else 1360 dst_buffer_list[k] = NULL; 1361 } 1362 1363 dst_buffer_list += dst_buffer_count; 1364 src_buffer_list += src_buffer_count; 1365 } 1366 } 1367} 1368 1369VKAPI_ATTR void VKAPI_CALL 1370radv_UpdateDescriptorSets(VkDevice _device, uint32_t descriptorWriteCount, 1371 const VkWriteDescriptorSet *pDescriptorWrites, 1372 uint32_t descriptorCopyCount, 1373 const VkCopyDescriptorSet *pDescriptorCopies) 1374{ 1375 RADV_FROM_HANDLE(radv_device, device, _device); 1376 1377 radv_update_descriptor_sets_impl(device, NULL, VK_NULL_HANDLE, descriptorWriteCount, 1378 pDescriptorWrites, descriptorCopyCount, pDescriptorCopies); 1379} 1380 1381void 1382radv_cmd_update_descriptor_sets(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer, 1383 VkDescriptorSet dstSetOverride, uint32_t descriptorWriteCount, 1384 const VkWriteDescriptorSet *pDescriptorWrites, 1385 uint32_t descriptorCopyCount, 1386 const VkCopyDescriptorSet *pDescriptorCopies) 1387{ 1388 /* Assume cmd_buffer != NULL to optimize out cmd_buffer checks in generic code above. */ 1389 assume(cmd_buffer != NULL); 1390 radv_update_descriptor_sets_impl(device, cmd_buffer, dstSetOverride, descriptorWriteCount, 1391 pDescriptorWrites, descriptorCopyCount, pDescriptorCopies); 1392} 1393 1394VKAPI_ATTR VkResult VKAPI_CALL 1395radv_CreateDescriptorUpdateTemplate(VkDevice _device, 1396 const VkDescriptorUpdateTemplateCreateInfo *pCreateInfo, 1397 const VkAllocationCallbacks *pAllocator, 1398 VkDescriptorUpdateTemplate *pDescriptorUpdateTemplate) 1399{ 1400 RADV_FROM_HANDLE(radv_device, device, _device); 1401 const uint32_t entry_count = pCreateInfo->descriptorUpdateEntryCount; 1402 const size_t size = sizeof(struct radv_descriptor_update_template) + 1403 sizeof(struct radv_descriptor_update_template_entry) * entry_count; 1404 struct radv_descriptor_set_layout *set_layout = NULL; 1405 struct radv_descriptor_update_template *templ; 1406 uint32_t i; 1407 1408 templ = vk_alloc2(&device->vk.alloc, pAllocator, size, 8, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); 1409 if (!templ) 1410 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); 1411 1412 vk_object_base_init(&device->vk, &templ->base, VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE); 1413 1414 templ->entry_count = entry_count; 1415 1416 if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR) { 1417 RADV_FROM_HANDLE(radv_pipeline_layout, pipeline_layout, pCreateInfo->pipelineLayout); 1418 1419 /* descriptorSetLayout should be ignored for push descriptors 1420 * and instead it refers to pipelineLayout and set. 1421 */ 1422 assert(pCreateInfo->set < MAX_SETS); 1423 set_layout = pipeline_layout->set[pCreateInfo->set].layout; 1424 1425 templ->bind_point = pCreateInfo->pipelineBindPoint; 1426 } else { 1427 assert(pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET); 1428 set_layout = radv_descriptor_set_layout_from_handle(pCreateInfo->descriptorSetLayout); 1429 } 1430 1431 for (i = 0; i < entry_count; i++) { 1432 const VkDescriptorUpdateTemplateEntry *entry = &pCreateInfo->pDescriptorUpdateEntries[i]; 1433 const struct radv_descriptor_set_binding_layout *binding_layout = 1434 set_layout->binding + entry->dstBinding; 1435 const uint32_t buffer_offset = binding_layout->buffer_offset + entry->dstArrayElement; 1436 const uint32_t *immutable_samplers = NULL; 1437 uint32_t dst_offset; 1438 uint32_t dst_stride; 1439 1440 /* dst_offset is an offset into dynamic_descriptors when the descriptor 1441 is dynamic, and an offset into mapped_ptr otherwise */ 1442 switch (entry->descriptorType) { 1443 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: 1444 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: 1445 assert(pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_DESCRIPTOR_SET); 1446 dst_offset = binding_layout->dynamic_offset_offset + entry->dstArrayElement; 1447 dst_stride = 0; /* Not used */ 1448 break; 1449 default: 1450 switch (entry->descriptorType) { 1451 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: 1452 case VK_DESCRIPTOR_TYPE_SAMPLER: 1453 /* Immutable samplers are copied into push descriptors when they are pushed */ 1454 if (pCreateInfo->templateType == 1455 VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR && 1456 binding_layout->immutable_samplers_offset && 1457 !binding_layout->immutable_samplers_equal) { 1458 immutable_samplers = 1459 radv_immutable_samplers(set_layout, binding_layout) + entry->dstArrayElement * 4; 1460 } 1461 break; 1462 default: 1463 break; 1464 } 1465 dst_offset = binding_layout->offset / 4; 1466 if (entry->descriptorType == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) 1467 dst_offset += entry->dstArrayElement / 4; 1468 else 1469 dst_offset += binding_layout->size * entry->dstArrayElement / 4; 1470 1471 dst_stride = binding_layout->size / 4; 1472 break; 1473 } 1474 1475 templ->entry[i] = (struct radv_descriptor_update_template_entry){ 1476 .descriptor_type = entry->descriptorType, 1477 .descriptor_count = entry->descriptorCount, 1478 .src_offset = entry->offset, 1479 .src_stride = entry->stride, 1480 .dst_offset = dst_offset, 1481 .dst_stride = dst_stride, 1482 .buffer_offset = buffer_offset, 1483 .has_sampler = !binding_layout->immutable_samplers_offset, 1484 .sampler_offset = radv_combined_image_descriptor_sampler_offset(binding_layout), 1485 .immutable_samplers = immutable_samplers}; 1486 } 1487 1488 *pDescriptorUpdateTemplate = radv_descriptor_update_template_to_handle(templ); 1489 return VK_SUCCESS; 1490} 1491 1492VKAPI_ATTR void VKAPI_CALL 1493radv_DestroyDescriptorUpdateTemplate(VkDevice _device, 1494 VkDescriptorUpdateTemplate descriptorUpdateTemplate, 1495 const VkAllocationCallbacks *pAllocator) 1496{ 1497 RADV_FROM_HANDLE(radv_device, device, _device); 1498 RADV_FROM_HANDLE(radv_descriptor_update_template, templ, descriptorUpdateTemplate); 1499 1500 if (!templ) 1501 return; 1502 1503 vk_object_base_finish(&templ->base); 1504 vk_free2(&device->vk.alloc, pAllocator, templ); 1505} 1506 1507static ALWAYS_INLINE void 1508radv_update_descriptor_set_with_template_impl(struct radv_device *device, 1509 struct radv_cmd_buffer *cmd_buffer, 1510 struct radv_descriptor_set *set, 1511 VkDescriptorUpdateTemplate descriptorUpdateTemplate, 1512 const void *pData) 1513{ 1514 RADV_FROM_HANDLE(radv_descriptor_update_template, templ, descriptorUpdateTemplate); 1515 uint32_t i; 1516 1517 for (i = 0; i < templ->entry_count; ++i) { 1518 struct radeon_winsys_bo **buffer_list = set->descriptors + templ->entry[i].buffer_offset; 1519 uint32_t *pDst = set->header.mapped_ptr + templ->entry[i].dst_offset; 1520 const uint8_t *pSrc = ((const uint8_t *)pData) + templ->entry[i].src_offset; 1521 uint32_t j; 1522 1523 if (templ->entry[i].descriptor_type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK) { 1524 memcpy((uint8_t *)pDst, pSrc, templ->entry[i].descriptor_count); 1525 continue; 1526 } 1527 1528 for (j = 0; j < templ->entry[i].descriptor_count; ++j) { 1529 switch (templ->entry[i].descriptor_type) { 1530 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: 1531 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: { 1532 const unsigned idx = templ->entry[i].dst_offset + j; 1533 assert(!(set->header.layout->flags & 1534 VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR)); 1535 write_dynamic_buffer_descriptor(device, set->header.dynamic_descriptors + idx, 1536 buffer_list, (struct VkDescriptorBufferInfo *)pSrc); 1537 break; 1538 } 1539 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER: 1540 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: 1541 write_buffer_descriptor(device, cmd_buffer, pDst, buffer_list, 1542 (struct VkDescriptorBufferInfo *)pSrc); 1543 break; 1544 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER: 1545 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER: 1546 write_texel_buffer_descriptor(device, cmd_buffer, pDst, buffer_list, 1547 *(VkBufferView *)pSrc); 1548 break; 1549 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: 1550 write_image_descriptor(device, cmd_buffer, 32, pDst, buffer_list, 1551 templ->entry[i].descriptor_type, 1552 (struct VkDescriptorImageInfo *)pSrc); 1553 break; 1554 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: 1555 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT: 1556 write_image_descriptor(device, cmd_buffer, 64, pDst, buffer_list, 1557 templ->entry[i].descriptor_type, 1558 (struct VkDescriptorImageInfo *)pSrc); 1559 break; 1560 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER: 1561 write_combined_image_sampler_descriptor( 1562 device, cmd_buffer, templ->entry[i].sampler_offset, pDst, buffer_list, 1563 templ->entry[i].descriptor_type, (struct VkDescriptorImageInfo *)pSrc, 1564 templ->entry[i].has_sampler); 1565 if (cmd_buffer && templ->entry[i].immutable_samplers) { 1566 memcpy((char *)pDst + templ->entry[i].sampler_offset, 1567 templ->entry[i].immutable_samplers + 4 * j, 16); 1568 } 1569 break; 1570 case VK_DESCRIPTOR_TYPE_SAMPLER: 1571 if (templ->entry[i].has_sampler) 1572 write_sampler_descriptor(device, pDst, (struct VkDescriptorImageInfo *)pSrc); 1573 else if (cmd_buffer && templ->entry[i].immutable_samplers) 1574 memcpy(pDst, templ->entry[i].immutable_samplers + 4 * j, 16); 1575 break; 1576 case VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR: 1577 write_accel_struct(pDst, *(const VkAccelerationStructureKHR *)pSrc); 1578 break; 1579 default: 1580 break; 1581 } 1582 pSrc += templ->entry[i].src_stride; 1583 pDst += templ->entry[i].dst_stride; 1584 ++buffer_list; 1585 } 1586 } 1587} 1588 1589void 1590radv_cmd_update_descriptor_set_with_template(struct radv_device *device, 1591 struct radv_cmd_buffer *cmd_buffer, 1592 struct radv_descriptor_set *set, 1593 VkDescriptorUpdateTemplate descriptorUpdateTemplate, 1594 const void *pData) 1595{ 1596 /* Assume cmd_buffer != NULL to optimize out cmd_buffer checks in generic code above. */ 1597 assume(cmd_buffer != NULL); 1598 radv_update_descriptor_set_with_template_impl(device, cmd_buffer, set, descriptorUpdateTemplate, pData); 1599} 1600 1601VKAPI_ATTR void VKAPI_CALL 1602radv_UpdateDescriptorSetWithTemplate(VkDevice _device, VkDescriptorSet descriptorSet, 1603 VkDescriptorUpdateTemplate descriptorUpdateTemplate, 1604 const void *pData) 1605{ 1606 RADV_FROM_HANDLE(radv_device, device, _device); 1607 RADV_FROM_HANDLE(radv_descriptor_set, set, descriptorSet); 1608 1609 radv_update_descriptor_set_with_template_impl(device, NULL, set, descriptorUpdateTemplate, pData); 1610} 1611 1612VKAPI_ATTR void VKAPI_CALL 1613radv_GetDescriptorSetLayoutHostMappingInfoVALVE( 1614 VkDevice _device, const VkDescriptorSetBindingReferenceVALVE *pBindingReference, 1615 VkDescriptorSetLayoutHostMappingInfoVALVE *pHostMapping) 1616{ 1617 struct radv_descriptor_set_layout *set_layout = 1618 radv_descriptor_set_layout_from_handle(pBindingReference->descriptorSetLayout); 1619 1620 const struct radv_descriptor_set_binding_layout *binding_layout = 1621 set_layout->binding + pBindingReference->binding; 1622 1623 pHostMapping->descriptorOffset = binding_layout->offset; 1624 pHostMapping->descriptorSize = binding_layout->size; 1625} 1626 1627VKAPI_ATTR void VKAPI_CALL 1628radv_GetDescriptorSetHostMappingVALVE(VkDevice _device, VkDescriptorSet descriptorSet, 1629 void **ppData) 1630{ 1631 RADV_FROM_HANDLE(radv_descriptor_set, set, descriptorSet); 1632 *ppData = set->header.mapped_ptr; 1633} 1634 1635VKAPI_ATTR VkResult VKAPI_CALL 1636radv_CreateSamplerYcbcrConversion(VkDevice _device, 1637 const VkSamplerYcbcrConversionCreateInfo *pCreateInfo, 1638 const VkAllocationCallbacks *pAllocator, 1639 VkSamplerYcbcrConversion *pYcbcrConversion) 1640{ 1641 RADV_FROM_HANDLE(radv_device, device, _device); 1642 struct radv_sampler_ycbcr_conversion *conversion = NULL; 1643 1644 conversion = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*conversion), 8, 1645 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); 1646 1647 if (conversion == NULL) 1648 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY); 1649 1650 vk_object_base_init(&device->vk, &conversion->base, VK_OBJECT_TYPE_SAMPLER_YCBCR_CONVERSION); 1651 1652 conversion->state.format = pCreateInfo->format; 1653 conversion->state.ycbcr_model = pCreateInfo->ycbcrModel; 1654 conversion->state.ycbcr_range = pCreateInfo->ycbcrRange; 1655 conversion->state.components = pCreateInfo->components; 1656 conversion->state.chroma_offsets[0] = pCreateInfo->xChromaOffset; 1657 conversion->state.chroma_offsets[1] = pCreateInfo->yChromaOffset; 1658 conversion->state.chroma_filter = pCreateInfo->chromaFilter; 1659 1660 *pYcbcrConversion = radv_sampler_ycbcr_conversion_to_handle(conversion); 1661 return VK_SUCCESS; 1662} 1663 1664VKAPI_ATTR void VKAPI_CALL 1665radv_DestroySamplerYcbcrConversion(VkDevice _device, VkSamplerYcbcrConversion ycbcrConversion, 1666 const VkAllocationCallbacks *pAllocator) 1667{ 1668 RADV_FROM_HANDLE(radv_device, device, _device); 1669 RADV_FROM_HANDLE(radv_sampler_ycbcr_conversion, ycbcr_conversion, ycbcrConversion); 1670 1671 if (!ycbcr_conversion) 1672 return; 1673 1674 vk_object_base_finish(&ycbcr_conversion->base); 1675 vk_free2(&device->vk.alloc, pAllocator, ycbcr_conversion); 1676} 1677