1/* 2 * Copyright © 2015 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Jason Ekstrand (jason@jlekstrand.net) 25 * 26 */ 27 28#include "vtn_private.h" 29#include "spirv_info.h" 30#include "nir_deref.h" 31#include <vulkan/vulkan_core.h> 32 33static struct vtn_pointer* 34vtn_align_pointer(struct vtn_builder *b, struct vtn_pointer *ptr, 35 unsigned alignment) 36{ 37 if (alignment == 0) 38 return ptr; 39 40 if (!util_is_power_of_two_nonzero(alignment)) { 41 vtn_warn("Provided alignment is not a power of two"); 42 alignment = 1 << (ffs(alignment) - 1); 43 } 44 45 /* If this pointer doesn't have a deref, bail. This either means we're 46 * using the old offset+alignment pointers which don't support carrying 47 * alignment information or we're a pointer that is below the block 48 * boundary in our access chain in which case alignment is meaningless. 49 */ 50 if (ptr->deref == NULL) 51 return ptr; 52 53 /* Ignore alignment information on logical pointers. This way, we don't 54 * trip up drivers with unnecessary casts. 55 */ 56 nir_address_format addr_format = vtn_mode_to_address_format(b, ptr->mode); 57 if (addr_format == nir_address_format_logical) 58 return ptr; 59 60 struct vtn_pointer *copy = ralloc(b, struct vtn_pointer); 61 *copy = *ptr; 62 copy->deref = nir_alignment_deref_cast(&b->nb, ptr->deref, alignment, 0); 63 64 return copy; 65} 66 67static void 68ptr_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 69 const struct vtn_decoration *dec, void *void_ptr) 70{ 71 struct vtn_pointer *ptr = void_ptr; 72 73 switch (dec->decoration) { 74 case SpvDecorationNonUniformEXT: 75 ptr->access |= ACCESS_NON_UNIFORM; 76 break; 77 78 default: 79 break; 80 } 81} 82 83struct access_align { 84 enum gl_access_qualifier access; 85 uint32_t alignment; 86}; 87 88static void 89access_align_cb(struct vtn_builder *b, struct vtn_value *val, int member, 90 const struct vtn_decoration *dec, void *void_ptr) 91{ 92 struct access_align *aa = void_ptr; 93 94 switch (dec->decoration) { 95 case SpvDecorationAlignment: 96 aa->alignment = dec->operands[0]; 97 break; 98 99 case SpvDecorationNonUniformEXT: 100 aa->access |= ACCESS_NON_UNIFORM; 101 break; 102 103 default: 104 break; 105 } 106} 107 108static struct vtn_pointer* 109vtn_decorate_pointer(struct vtn_builder *b, struct vtn_value *val, 110 struct vtn_pointer *ptr) 111{ 112 struct access_align aa = { 0, }; 113 vtn_foreach_decoration(b, val, access_align_cb, &aa); 114 115 ptr = vtn_align_pointer(b, ptr, aa.alignment); 116 117 /* If we're adding access flags, make a copy of the pointer. We could 118 * probably just OR them in without doing so but this prevents us from 119 * leaking them any further than actually specified in the SPIR-V. 120 */ 121 if (aa.access & ~ptr->access) { 122 struct vtn_pointer *copy = ralloc(b, struct vtn_pointer); 123 *copy = *ptr; 124 copy->access |= aa.access; 125 return copy; 126 } 127 128 return ptr; 129} 130 131struct vtn_value * 132vtn_push_pointer(struct vtn_builder *b, uint32_t value_id, 133 struct vtn_pointer *ptr) 134{ 135 struct vtn_value *val = vtn_push_value(b, value_id, vtn_value_type_pointer); 136 val->pointer = vtn_decorate_pointer(b, val, ptr); 137 return val; 138} 139 140void 141vtn_copy_value(struct vtn_builder *b, uint32_t src_value_id, 142 uint32_t dst_value_id) 143{ 144 struct vtn_value *src = vtn_untyped_value(b, src_value_id); 145 struct vtn_value *dst = vtn_untyped_value(b, dst_value_id); 146 struct vtn_value src_copy = *src; 147 148 vtn_fail_if(dst->value_type != vtn_value_type_invalid, 149 "SPIR-V id %u has already been written by another instruction", 150 dst_value_id); 151 152 vtn_fail_if(dst->type->id != src->type->id, 153 "Result Type must equal Operand type"); 154 155 src_copy.name = dst->name; 156 src_copy.decoration = dst->decoration; 157 src_copy.type = dst->type; 158 *dst = src_copy; 159 160 if (dst->value_type == vtn_value_type_pointer) 161 dst->pointer = vtn_decorate_pointer(b, dst, dst->pointer); 162} 163 164static struct vtn_access_chain * 165vtn_access_chain_create(struct vtn_builder *b, unsigned length) 166{ 167 struct vtn_access_chain *chain; 168 169 /* Subtract 1 from the length since there's already one built in */ 170 size_t size = sizeof(*chain) + 171 (MAX2(length, 1) - 1) * sizeof(chain->link[0]); 172 chain = rzalloc_size(b, size); 173 chain->length = length; 174 175 return chain; 176} 177 178static bool 179vtn_mode_is_cross_invocation(struct vtn_builder *b, 180 enum vtn_variable_mode mode) 181{ 182 return mode == vtn_variable_mode_ssbo || 183 mode == vtn_variable_mode_ubo || 184 mode == vtn_variable_mode_phys_ssbo || 185 mode == vtn_variable_mode_push_constant || 186 mode == vtn_variable_mode_workgroup || 187 mode == vtn_variable_mode_cross_workgroup; 188} 189 190static bool 191vtn_pointer_is_external_block(struct vtn_builder *b, 192 struct vtn_pointer *ptr) 193{ 194 return ptr->mode == vtn_variable_mode_ssbo || 195 ptr->mode == vtn_variable_mode_ubo || 196 ptr->mode == vtn_variable_mode_phys_ssbo; 197} 198 199static nir_ssa_def * 200vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link, 201 unsigned stride, unsigned bit_size) 202{ 203 vtn_assert(stride > 0); 204 if (link.mode == vtn_access_mode_literal) { 205 return nir_imm_intN_t(&b->nb, link.id * stride, bit_size); 206 } else { 207 nir_ssa_def *ssa = vtn_ssa_value(b, link.id)->def; 208 if (ssa->bit_size != bit_size) 209 ssa = nir_i2i(&b->nb, ssa, bit_size); 210 return nir_imul_imm(&b->nb, ssa, stride); 211 } 212} 213 214static VkDescriptorType 215vk_desc_type_for_mode(struct vtn_builder *b, enum vtn_variable_mode mode) 216{ 217 switch (mode) { 218 case vtn_variable_mode_ubo: 219 return VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER; 220 case vtn_variable_mode_ssbo: 221 return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; 222 case vtn_variable_mode_accel_struct: 223 return VK_DESCRIPTOR_TYPE_ACCELERATION_STRUCTURE_KHR; 224 default: 225 vtn_fail("Invalid mode for vulkan_resource_index"); 226 } 227} 228 229static nir_ssa_def * 230vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var, 231 nir_ssa_def *desc_array_index) 232{ 233 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN); 234 235 if (!desc_array_index) 236 desc_array_index = nir_imm_int(&b->nb, 0); 237 238 if (b->vars_used_indirectly) { 239 vtn_assert(var->var); 240 _mesa_set_add(b->vars_used_indirectly, var->var); 241 } 242 243 nir_intrinsic_instr *instr = 244 nir_intrinsic_instr_create(b->nb.shader, 245 nir_intrinsic_vulkan_resource_index); 246 instr->src[0] = nir_src_for_ssa(desc_array_index); 247 nir_intrinsic_set_desc_set(instr, var->descriptor_set); 248 nir_intrinsic_set_binding(instr, var->binding); 249 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, var->mode)); 250 251 nir_address_format addr_format = vtn_mode_to_address_format(b, var->mode); 252 nir_ssa_dest_init(&instr->instr, &instr->dest, 253 nir_address_format_num_components(addr_format), 254 nir_address_format_bit_size(addr_format), NULL); 255 instr->num_components = instr->dest.ssa.num_components; 256 nir_builder_instr_insert(&b->nb, &instr->instr); 257 258 return &instr->dest.ssa; 259} 260 261static nir_ssa_def * 262vtn_resource_reindex(struct vtn_builder *b, enum vtn_variable_mode mode, 263 nir_ssa_def *base_index, nir_ssa_def *offset_index) 264{ 265 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN); 266 267 nir_intrinsic_instr *instr = 268 nir_intrinsic_instr_create(b->nb.shader, 269 nir_intrinsic_vulkan_resource_reindex); 270 instr->src[0] = nir_src_for_ssa(base_index); 271 instr->src[1] = nir_src_for_ssa(offset_index); 272 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, mode)); 273 274 nir_address_format addr_format = vtn_mode_to_address_format(b, mode); 275 nir_ssa_dest_init(&instr->instr, &instr->dest, 276 nir_address_format_num_components(addr_format), 277 nir_address_format_bit_size(addr_format), NULL); 278 instr->num_components = instr->dest.ssa.num_components; 279 nir_builder_instr_insert(&b->nb, &instr->instr); 280 281 return &instr->dest.ssa; 282} 283 284static nir_ssa_def * 285vtn_descriptor_load(struct vtn_builder *b, enum vtn_variable_mode mode, 286 nir_ssa_def *desc_index) 287{ 288 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN); 289 290 nir_intrinsic_instr *desc_load = 291 nir_intrinsic_instr_create(b->nb.shader, 292 nir_intrinsic_load_vulkan_descriptor); 293 desc_load->src[0] = nir_src_for_ssa(desc_index); 294 nir_intrinsic_set_desc_type(desc_load, vk_desc_type_for_mode(b, mode)); 295 296 nir_address_format addr_format = vtn_mode_to_address_format(b, mode); 297 nir_ssa_dest_init(&desc_load->instr, &desc_load->dest, 298 nir_address_format_num_components(addr_format), 299 nir_address_format_bit_size(addr_format), NULL); 300 desc_load->num_components = desc_load->dest.ssa.num_components; 301 nir_builder_instr_insert(&b->nb, &desc_load->instr); 302 303 return &desc_load->dest.ssa; 304} 305 306static struct vtn_pointer * 307vtn_pointer_dereference(struct vtn_builder *b, 308 struct vtn_pointer *base, 309 struct vtn_access_chain *deref_chain) 310{ 311 struct vtn_type *type = base->type; 312 enum gl_access_qualifier access = base->access | deref_chain->access; 313 unsigned idx = 0; 314 315 nir_deref_instr *tail; 316 if (base->deref) { 317 tail = base->deref; 318 } else if (b->options->environment == NIR_SPIRV_VULKAN && 319 (vtn_pointer_is_external_block(b, base) || 320 base->mode == vtn_variable_mode_accel_struct)) { 321 nir_ssa_def *block_index = base->block_index; 322 323 /* We dereferencing an external block pointer. Correctness of this 324 * operation relies on one particular line in the SPIR-V spec, section 325 * entitled "Validation Rules for Shader Capabilities": 326 * 327 * "Block and BufferBlock decorations cannot decorate a structure 328 * type that is nested at any level inside another structure type 329 * decorated with Block or BufferBlock." 330 * 331 * This means that we can detect the point where we cross over from 332 * descriptor indexing to buffer indexing by looking for the block 333 * decorated struct type. Anything before the block decorated struct 334 * type is a descriptor indexing operation and anything after the block 335 * decorated struct is a buffer offset operation. 336 */ 337 338 /* Figure out the descriptor array index if any 339 * 340 * Some of the Vulkan CTS tests with hand-rolled SPIR-V have been known 341 * to forget the Block or BufferBlock decoration from time to time. 342 * It's more robust if we check for both !block_index and for the type 343 * to contain a block. This way there's a decent chance that arrays of 344 * UBOs/SSBOs will work correctly even if variable pointers are 345 * completley toast. 346 */ 347 nir_ssa_def *desc_arr_idx = NULL; 348 if (!block_index || vtn_type_contains_block(b, type) || 349 base->mode == vtn_variable_mode_accel_struct) { 350 /* If our type contains a block, then we're still outside the block 351 * and we need to process enough levels of dereferences to get inside 352 * of it. Same applies to acceleration structures. 353 */ 354 if (deref_chain->ptr_as_array) { 355 unsigned aoa_size = glsl_get_aoa_size(type->type); 356 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[idx], 357 MAX2(aoa_size, 1), 32); 358 idx++; 359 } 360 361 for (; idx < deref_chain->length; idx++) { 362 if (type->base_type != vtn_base_type_array) { 363 vtn_assert(type->base_type == vtn_base_type_struct); 364 break; 365 } 366 367 unsigned aoa_size = glsl_get_aoa_size(type->array_element->type); 368 nir_ssa_def *arr_offset = 369 vtn_access_link_as_ssa(b, deref_chain->link[idx], 370 MAX2(aoa_size, 1), 32); 371 if (desc_arr_idx) 372 desc_arr_idx = nir_iadd(&b->nb, desc_arr_idx, arr_offset); 373 else 374 desc_arr_idx = arr_offset; 375 376 type = type->array_element; 377 access |= type->access; 378 } 379 } 380 381 if (!block_index) { 382 vtn_assert(base->var && base->type); 383 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx); 384 } else if (desc_arr_idx) { 385 block_index = vtn_resource_reindex(b, base->mode, 386 block_index, desc_arr_idx); 387 } 388 389 if (idx == deref_chain->length) { 390 /* The entire deref was consumed in finding the block index. Return 391 * a pointer which just has a block index and a later access chain 392 * will dereference deeper. 393 */ 394 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 395 ptr->mode = base->mode; 396 ptr->type = type; 397 ptr->block_index = block_index; 398 ptr->access = access; 399 return ptr; 400 } 401 402 /* If we got here, there's more access chain to handle and we have the 403 * final block index. Insert a descriptor load and cast to a deref to 404 * start the deref chain. 405 */ 406 nir_ssa_def *desc = vtn_descriptor_load(b, base->mode, block_index); 407 408 assert(base->mode == vtn_variable_mode_ssbo || 409 base->mode == vtn_variable_mode_ubo); 410 nir_variable_mode nir_mode = 411 base->mode == vtn_variable_mode_ssbo ? nir_var_mem_ssbo : nir_var_mem_ubo; 412 413 tail = nir_build_deref_cast(&b->nb, desc, nir_mode, 414 vtn_type_get_nir_type(b, type, base->mode), 415 base->ptr_type->stride); 416 } else if (base->mode == vtn_variable_mode_shader_record) { 417 /* For ShaderRecordBufferKHR variables, we don't have a nir_variable. 418 * It's just a fancy handle around a pointer to the shader record for 419 * the current shader. 420 */ 421 tail = nir_build_deref_cast(&b->nb, nir_load_shader_record_ptr(&b->nb), 422 nir_var_mem_constant, 423 vtn_type_get_nir_type(b, base->type, 424 base->mode), 425 0 /* ptr_as_array stride */); 426 } else { 427 assert(base->var && base->var->var); 428 tail = nir_build_deref_var(&b->nb, base->var->var); 429 if (base->ptr_type && base->ptr_type->type) { 430 tail->dest.ssa.num_components = 431 glsl_get_vector_elements(base->ptr_type->type); 432 tail->dest.ssa.bit_size = glsl_get_bit_size(base->ptr_type->type); 433 } 434 } 435 436 if (idx == 0 && deref_chain->ptr_as_array) { 437 /* We start with a deref cast to get the stride. Hopefully, we'll be 438 * able to delete that cast eventually. 439 */ 440 tail = nir_build_deref_cast(&b->nb, &tail->dest.ssa, tail->modes, 441 tail->type, base->ptr_type->stride); 442 443 nir_ssa_def *index = vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 444 tail->dest.ssa.bit_size); 445 tail = nir_build_deref_ptr_as_array(&b->nb, tail, index); 446 idx++; 447 } 448 449 for (; idx < deref_chain->length; idx++) { 450 if (glsl_type_is_struct_or_ifc(type->type)) { 451 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal); 452 unsigned field = deref_chain->link[idx].id; 453 tail = nir_build_deref_struct(&b->nb, tail, field); 454 type = type->members[field]; 455 } else { 456 nir_ssa_def *arr_index = 457 vtn_access_link_as_ssa(b, deref_chain->link[idx], 1, 458 tail->dest.ssa.bit_size); 459 tail = nir_build_deref_array(&b->nb, tail, arr_index); 460 type = type->array_element; 461 } 462 tail->arr.in_bounds = deref_chain->in_bounds; 463 464 access |= type->access; 465 } 466 467 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 468 ptr->mode = base->mode; 469 ptr->type = type; 470 ptr->var = base->var; 471 ptr->deref = tail; 472 ptr->access = access; 473 474 return ptr; 475} 476 477nir_deref_instr * 478vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr) 479{ 480 if (!ptr->deref) { 481 struct vtn_access_chain chain = { 482 .length = 0, 483 }; 484 ptr = vtn_pointer_dereference(b, ptr, &chain); 485 } 486 487 return ptr->deref; 488} 489 490static void 491_vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_instr *deref, 492 struct vtn_ssa_value *inout, 493 enum gl_access_qualifier access) 494{ 495 if (glsl_type_is_vector_or_scalar(deref->type)) { 496 if (load) { 497 inout->def = nir_load_deref_with_access(&b->nb, deref, access); 498 } else { 499 nir_store_deref_with_access(&b->nb, deref, inout->def, ~0, access); 500 } 501 } else if (glsl_type_is_array(deref->type) || 502 glsl_type_is_matrix(deref->type)) { 503 unsigned elems = glsl_get_length(deref->type); 504 for (unsigned i = 0; i < elems; i++) { 505 nir_deref_instr *child = 506 nir_build_deref_array_imm(&b->nb, deref, i); 507 _vtn_local_load_store(b, load, child, inout->elems[i], access); 508 } 509 } else { 510 vtn_assert(glsl_type_is_struct_or_ifc(deref->type)); 511 unsigned elems = glsl_get_length(deref->type); 512 for (unsigned i = 0; i < elems; i++) { 513 nir_deref_instr *child = nir_build_deref_struct(&b->nb, deref, i); 514 _vtn_local_load_store(b, load, child, inout->elems[i], access); 515 } 516 } 517} 518 519nir_deref_instr * 520vtn_nir_deref(struct vtn_builder *b, uint32_t id) 521{ 522 struct vtn_pointer *ptr = vtn_pointer(b, id); 523 return vtn_pointer_to_deref(b, ptr); 524} 525 526/* 527 * Gets the NIR-level deref tail, which may have as a child an array deref 528 * selecting which component due to OpAccessChain supporting per-component 529 * indexing in SPIR-V. 530 */ 531static nir_deref_instr * 532get_deref_tail(nir_deref_instr *deref) 533{ 534 if (deref->deref_type != nir_deref_type_array) 535 return deref; 536 537 nir_deref_instr *parent = 538 nir_instr_as_deref(deref->parent.ssa->parent_instr); 539 540 if (glsl_type_is_vector(parent->type)) 541 return parent; 542 else 543 return deref; 544} 545 546struct vtn_ssa_value * 547vtn_local_load(struct vtn_builder *b, nir_deref_instr *src, 548 enum gl_access_qualifier access) 549{ 550 nir_deref_instr *src_tail = get_deref_tail(src); 551 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type); 552 _vtn_local_load_store(b, true, src_tail, val, access); 553 554 if (src_tail != src) { 555 val->type = src->type; 556 val->def = nir_vector_extract(&b->nb, val->def, src->arr.index.ssa); 557 } 558 559 return val; 560} 561 562void 563vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src, 564 nir_deref_instr *dest, enum gl_access_qualifier access) 565{ 566 nir_deref_instr *dest_tail = get_deref_tail(dest); 567 568 if (dest_tail != dest) { 569 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type); 570 _vtn_local_load_store(b, true, dest_tail, val, access); 571 572 val->def = nir_vector_insert(&b->nb, val->def, src->def, 573 dest->arr.index.ssa); 574 _vtn_local_load_store(b, false, dest_tail, val, access); 575 } else { 576 _vtn_local_load_store(b, false, dest_tail, src, access); 577 } 578} 579 580static nir_ssa_def * 581vtn_pointer_to_descriptor(struct vtn_builder *b, struct vtn_pointer *ptr) 582{ 583 assert(ptr->mode == vtn_variable_mode_accel_struct); 584 if (!ptr->block_index) { 585 struct vtn_access_chain chain = { 586 .length = 0, 587 }; 588 ptr = vtn_pointer_dereference(b, ptr, &chain); 589 } 590 591 vtn_assert(ptr->deref == NULL && ptr->block_index != NULL); 592 return vtn_descriptor_load(b, ptr->mode, ptr->block_index); 593} 594 595static void 596_vtn_variable_load_store(struct vtn_builder *b, bool load, 597 struct vtn_pointer *ptr, 598 enum gl_access_qualifier access, 599 struct vtn_ssa_value **inout) 600{ 601 if (ptr->mode == vtn_variable_mode_uniform || 602 ptr->mode == vtn_variable_mode_image) { 603 if (ptr->type->base_type == vtn_base_type_image || 604 ptr->type->base_type == vtn_base_type_sampler) { 605 /* See also our handling of OpTypeSampler and OpTypeImage */ 606 vtn_assert(load); 607 (*inout)->def = vtn_pointer_to_ssa(b, ptr); 608 return; 609 } else if (ptr->type->base_type == vtn_base_type_sampled_image) { 610 /* See also our handling of OpTypeSampledImage */ 611 vtn_assert(load); 612 struct vtn_sampled_image si = { 613 .image = vtn_pointer_to_deref(b, ptr), 614 .sampler = vtn_pointer_to_deref(b, ptr), 615 }; 616 (*inout)->def = vtn_sampled_image_to_nir_ssa(b, si); 617 return; 618 } 619 } else if (ptr->mode == vtn_variable_mode_accel_struct) { 620 vtn_assert(load); 621 (*inout)->def = vtn_pointer_to_descriptor(b, ptr); 622 return; 623 } 624 625 enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type); 626 switch (base_type) { 627 case GLSL_TYPE_UINT: 628 case GLSL_TYPE_INT: 629 case GLSL_TYPE_UINT16: 630 case GLSL_TYPE_INT16: 631 case GLSL_TYPE_UINT8: 632 case GLSL_TYPE_INT8: 633 case GLSL_TYPE_UINT64: 634 case GLSL_TYPE_INT64: 635 case GLSL_TYPE_FLOAT: 636 case GLSL_TYPE_FLOAT16: 637 case GLSL_TYPE_BOOL: 638 case GLSL_TYPE_DOUBLE: 639 if (glsl_type_is_vector_or_scalar(ptr->type->type)) { 640 /* We hit a vector or scalar; go ahead and emit the load[s] */ 641 nir_deref_instr *deref = vtn_pointer_to_deref(b, ptr); 642 if (vtn_mode_is_cross_invocation(b, ptr->mode)) { 643 /* If it's cross-invocation, we call nir_load/store_deref 644 * directly. The vtn_local_load/store helpers are too clever and 645 * do magic to avoid array derefs of vectors. That magic is both 646 * less efficient than the direct load/store and, in the case of 647 * stores, is broken because it creates a race condition if two 648 * threads are writing to different components of the same vector 649 * due to the load+insert+store it uses to emulate the array 650 * deref. 651 */ 652 if (load) { 653 (*inout)->def = nir_load_deref_with_access(&b->nb, deref, 654 ptr->type->access | access); 655 } else { 656 nir_store_deref_with_access(&b->nb, deref, (*inout)->def, ~0, 657 ptr->type->access | access); 658 } 659 } else { 660 if (load) { 661 *inout = vtn_local_load(b, deref, ptr->type->access | access); 662 } else { 663 vtn_local_store(b, *inout, deref, ptr->type->access | access); 664 } 665 } 666 return; 667 } 668 FALLTHROUGH; 669 670 case GLSL_TYPE_INTERFACE: 671 case GLSL_TYPE_ARRAY: 672 case GLSL_TYPE_STRUCT: { 673 unsigned elems = glsl_get_length(ptr->type->type); 674 struct vtn_access_chain chain = { 675 .length = 1, 676 .link = { 677 { .mode = vtn_access_mode_literal, }, 678 } 679 }; 680 for (unsigned i = 0; i < elems; i++) { 681 chain.link[0].id = i; 682 struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain); 683 _vtn_variable_load_store(b, load, elem, ptr->type->access | access, 684 &(*inout)->elems[i]); 685 } 686 return; 687 } 688 689 default: 690 vtn_fail("Invalid access chain type"); 691 } 692} 693 694struct vtn_ssa_value * 695vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src, 696 enum gl_access_qualifier access) 697{ 698 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src->type->type); 699 _vtn_variable_load_store(b, true, src, src->access | access, &val); 700 return val; 701} 702 703void 704vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src, 705 struct vtn_pointer *dest, enum gl_access_qualifier access) 706{ 707 _vtn_variable_load_store(b, false, dest, dest->access | access, &src); 708} 709 710static void 711_vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 712 struct vtn_pointer *src, enum gl_access_qualifier dest_access, 713 enum gl_access_qualifier src_access) 714{ 715 vtn_assert(glsl_get_bare_type(src->type->type) == 716 glsl_get_bare_type(dest->type->type)); 717 enum glsl_base_type base_type = glsl_get_base_type(src->type->type); 718 switch (base_type) { 719 case GLSL_TYPE_UINT: 720 case GLSL_TYPE_INT: 721 case GLSL_TYPE_UINT16: 722 case GLSL_TYPE_INT16: 723 case GLSL_TYPE_UINT8: 724 case GLSL_TYPE_INT8: 725 case GLSL_TYPE_UINT64: 726 case GLSL_TYPE_INT64: 727 case GLSL_TYPE_FLOAT: 728 case GLSL_TYPE_FLOAT16: 729 case GLSL_TYPE_DOUBLE: 730 case GLSL_TYPE_BOOL: 731 /* At this point, we have a scalar, vector, or matrix so we know that 732 * there cannot be any structure splitting still in the way. By 733 * stopping at the matrix level rather than the vector level, we 734 * ensure that matrices get loaded in the optimal way even if they 735 * are storred row-major in a UBO. 736 */ 737 vtn_variable_store(b, vtn_variable_load(b, src, src_access), dest, dest_access); 738 return; 739 740 case GLSL_TYPE_INTERFACE: 741 case GLSL_TYPE_ARRAY: 742 case GLSL_TYPE_STRUCT: { 743 struct vtn_access_chain chain = { 744 .length = 1, 745 .link = { 746 { .mode = vtn_access_mode_literal, }, 747 } 748 }; 749 unsigned elems = glsl_get_length(src->type->type); 750 for (unsigned i = 0; i < elems; i++) { 751 chain.link[0].id = i; 752 struct vtn_pointer *src_elem = 753 vtn_pointer_dereference(b, src, &chain); 754 struct vtn_pointer *dest_elem = 755 vtn_pointer_dereference(b, dest, &chain); 756 757 _vtn_variable_copy(b, dest_elem, src_elem, dest_access, src_access); 758 } 759 return; 760 } 761 762 default: 763 vtn_fail("Invalid access chain type"); 764 } 765} 766 767static void 768vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest, 769 struct vtn_pointer *src, enum gl_access_qualifier dest_access, 770 enum gl_access_qualifier src_access) 771{ 772 /* TODO: At some point, we should add a special-case for when we can 773 * just emit a copy_var intrinsic. 774 */ 775 _vtn_variable_copy(b, dest, src, dest_access, src_access); 776} 777 778static void 779set_mode_system_value(struct vtn_builder *b, nir_variable_mode *mode) 780{ 781 vtn_assert(*mode == nir_var_system_value || *mode == nir_var_shader_in || 782 /* Hack for NV_mesh_shader due to lack of dedicated storage class. */ 783 *mode == nir_var_mem_task_payload); 784 *mode = nir_var_system_value; 785} 786 787static void 788vtn_get_builtin_location(struct vtn_builder *b, 789 SpvBuiltIn builtin, int *location, 790 nir_variable_mode *mode) 791{ 792 switch (builtin) { 793 case SpvBuiltInPosition: 794 case SpvBuiltInPositionPerViewNV: 795 *location = VARYING_SLOT_POS; 796 break; 797 case SpvBuiltInPointSize: 798 *location = VARYING_SLOT_PSIZ; 799 break; 800 case SpvBuiltInClipDistance: 801 case SpvBuiltInClipDistancePerViewNV: 802 *location = VARYING_SLOT_CLIP_DIST0; 803 break; 804 case SpvBuiltInCullDistance: 805 case SpvBuiltInCullDistancePerViewNV: 806 *location = VARYING_SLOT_CULL_DIST0; 807 break; 808 case SpvBuiltInVertexId: 809 case SpvBuiltInVertexIndex: 810 /* The Vulkan spec defines VertexIndex to be non-zero-based and doesn't 811 * allow VertexId. The ARB_gl_spirv spec defines VertexId to be the 812 * same as gl_VertexID, which is non-zero-based, and removes 813 * VertexIndex. Since they're both defined to be non-zero-based, we use 814 * SYSTEM_VALUE_VERTEX_ID for both. 815 */ 816 *location = SYSTEM_VALUE_VERTEX_ID; 817 set_mode_system_value(b, mode); 818 break; 819 case SpvBuiltInInstanceIndex: 820 *location = SYSTEM_VALUE_INSTANCE_INDEX; 821 set_mode_system_value(b, mode); 822 break; 823 case SpvBuiltInInstanceId: 824 *location = SYSTEM_VALUE_INSTANCE_ID; 825 set_mode_system_value(b, mode); 826 break; 827 case SpvBuiltInPrimitiveId: 828 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) { 829 vtn_assert(*mode == nir_var_shader_in); 830 *location = VARYING_SLOT_PRIMITIVE_ID; 831 } else if (*mode == nir_var_shader_out) { 832 *location = VARYING_SLOT_PRIMITIVE_ID; 833 } else { 834 *location = SYSTEM_VALUE_PRIMITIVE_ID; 835 set_mode_system_value(b, mode); 836 } 837 break; 838 case SpvBuiltInInvocationId: 839 *location = SYSTEM_VALUE_INVOCATION_ID; 840 set_mode_system_value(b, mode); 841 break; 842 case SpvBuiltInLayer: 843 case SpvBuiltInLayerPerViewNV: 844 *location = VARYING_SLOT_LAYER; 845 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 846 *mode = nir_var_shader_in; 847 else if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 848 *mode = nir_var_shader_out; 849 else if (b->options && b->options->caps.shader_viewport_index_layer && 850 (b->shader->info.stage == MESA_SHADER_VERTEX || 851 b->shader->info.stage == MESA_SHADER_TESS_EVAL || 852 b->shader->info.stage == MESA_SHADER_MESH)) 853 *mode = nir_var_shader_out; 854 else 855 vtn_fail("invalid stage for SpvBuiltInLayer"); 856 break; 857 case SpvBuiltInViewportIndex: 858 *location = VARYING_SLOT_VIEWPORT; 859 if (b->shader->info.stage == MESA_SHADER_GEOMETRY) 860 *mode = nir_var_shader_out; 861 else if (b->options && b->options->caps.shader_viewport_index_layer && 862 (b->shader->info.stage == MESA_SHADER_VERTEX || 863 b->shader->info.stage == MESA_SHADER_TESS_EVAL || 864 b->shader->info.stage == MESA_SHADER_MESH)) 865 *mode = nir_var_shader_out; 866 else if (b->shader->info.stage == MESA_SHADER_FRAGMENT) 867 *mode = nir_var_shader_in; 868 else 869 vtn_fail("invalid stage for SpvBuiltInViewportIndex"); 870 break; 871 case SpvBuiltInViewportMaskNV: 872 case SpvBuiltInViewportMaskPerViewNV: 873 *location = VARYING_SLOT_VIEWPORT_MASK; 874 *mode = nir_var_shader_out; 875 break; 876 case SpvBuiltInTessLevelOuter: 877 *location = VARYING_SLOT_TESS_LEVEL_OUTER; 878 break; 879 case SpvBuiltInTessLevelInner: 880 *location = VARYING_SLOT_TESS_LEVEL_INNER; 881 break; 882 case SpvBuiltInTessCoord: 883 *location = SYSTEM_VALUE_TESS_COORD; 884 set_mode_system_value(b, mode); 885 break; 886 case SpvBuiltInPatchVertices: 887 *location = SYSTEM_VALUE_VERTICES_IN; 888 set_mode_system_value(b, mode); 889 break; 890 case SpvBuiltInFragCoord: 891 vtn_assert(*mode == nir_var_shader_in); 892 *mode = nir_var_system_value; 893 *location = SYSTEM_VALUE_FRAG_COORD; 894 break; 895 case SpvBuiltInPointCoord: 896 vtn_assert(*mode == nir_var_shader_in); 897 set_mode_system_value(b, mode); 898 *location = SYSTEM_VALUE_POINT_COORD; 899 break; 900 case SpvBuiltInFrontFacing: 901 *location = SYSTEM_VALUE_FRONT_FACE; 902 set_mode_system_value(b, mode); 903 break; 904 case SpvBuiltInSampleId: 905 *location = SYSTEM_VALUE_SAMPLE_ID; 906 set_mode_system_value(b, mode); 907 break; 908 case SpvBuiltInSamplePosition: 909 *location = SYSTEM_VALUE_SAMPLE_POS; 910 set_mode_system_value(b, mode); 911 break; 912 case SpvBuiltInSampleMask: 913 if (*mode == nir_var_shader_out) { 914 *location = FRAG_RESULT_SAMPLE_MASK; 915 } else { 916 *location = SYSTEM_VALUE_SAMPLE_MASK_IN; 917 set_mode_system_value(b, mode); 918 } 919 break; 920 case SpvBuiltInFragDepth: 921 *location = FRAG_RESULT_DEPTH; 922 vtn_assert(*mode == nir_var_shader_out); 923 break; 924 case SpvBuiltInHelperInvocation: 925 *location = SYSTEM_VALUE_HELPER_INVOCATION; 926 set_mode_system_value(b, mode); 927 break; 928 case SpvBuiltInNumWorkgroups: 929 *location = SYSTEM_VALUE_NUM_WORKGROUPS; 930 set_mode_system_value(b, mode); 931 break; 932 case SpvBuiltInWorkgroupSize: 933 case SpvBuiltInEnqueuedWorkgroupSize: 934 *location = SYSTEM_VALUE_WORKGROUP_SIZE; 935 set_mode_system_value(b, mode); 936 break; 937 case SpvBuiltInWorkgroupId: 938 *location = SYSTEM_VALUE_WORKGROUP_ID; 939 set_mode_system_value(b, mode); 940 break; 941 case SpvBuiltInLocalInvocationId: 942 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID; 943 set_mode_system_value(b, mode); 944 break; 945 case SpvBuiltInLocalInvocationIndex: 946 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX; 947 set_mode_system_value(b, mode); 948 break; 949 case SpvBuiltInGlobalInvocationId: 950 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID; 951 set_mode_system_value(b, mode); 952 break; 953 case SpvBuiltInGlobalLinearId: 954 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX; 955 set_mode_system_value(b, mode); 956 break; 957 case SpvBuiltInGlobalOffset: 958 *location = SYSTEM_VALUE_BASE_GLOBAL_INVOCATION_ID; 959 set_mode_system_value(b, mode); 960 break; 961 case SpvBuiltInBaseVertex: 962 /* OpenGL gl_BaseVertex (SYSTEM_VALUE_BASE_VERTEX) is not the same 963 * semantic as Vulkan BaseVertex (SYSTEM_VALUE_FIRST_VERTEX). 964 */ 965 if (b->options->environment == NIR_SPIRV_OPENGL) 966 *location = SYSTEM_VALUE_BASE_VERTEX; 967 else 968 *location = SYSTEM_VALUE_FIRST_VERTEX; 969 set_mode_system_value(b, mode); 970 break; 971 case SpvBuiltInBaseInstance: 972 *location = SYSTEM_VALUE_BASE_INSTANCE; 973 set_mode_system_value(b, mode); 974 break; 975 case SpvBuiltInDrawIndex: 976 *location = SYSTEM_VALUE_DRAW_ID; 977 set_mode_system_value(b, mode); 978 break; 979 case SpvBuiltInSubgroupSize: 980 *location = SYSTEM_VALUE_SUBGROUP_SIZE; 981 set_mode_system_value(b, mode); 982 break; 983 case SpvBuiltInSubgroupId: 984 *location = SYSTEM_VALUE_SUBGROUP_ID; 985 set_mode_system_value(b, mode); 986 break; 987 case SpvBuiltInSubgroupLocalInvocationId: 988 *location = SYSTEM_VALUE_SUBGROUP_INVOCATION; 989 set_mode_system_value(b, mode); 990 break; 991 case SpvBuiltInNumSubgroups: 992 *location = SYSTEM_VALUE_NUM_SUBGROUPS; 993 set_mode_system_value(b, mode); 994 break; 995 case SpvBuiltInDeviceIndex: 996 *location = SYSTEM_VALUE_DEVICE_INDEX; 997 set_mode_system_value(b, mode); 998 break; 999 case SpvBuiltInViewIndex: 1000 if (b->options && b->options->view_index_is_input) { 1001 *location = VARYING_SLOT_VIEW_INDEX; 1002 vtn_assert(*mode == nir_var_shader_in); 1003 } else { 1004 *location = SYSTEM_VALUE_VIEW_INDEX; 1005 set_mode_system_value(b, mode); 1006 } 1007 break; 1008 case SpvBuiltInSubgroupEqMask: 1009 *location = SYSTEM_VALUE_SUBGROUP_EQ_MASK, 1010 set_mode_system_value(b, mode); 1011 break; 1012 case SpvBuiltInSubgroupGeMask: 1013 *location = SYSTEM_VALUE_SUBGROUP_GE_MASK, 1014 set_mode_system_value(b, mode); 1015 break; 1016 case SpvBuiltInSubgroupGtMask: 1017 *location = SYSTEM_VALUE_SUBGROUP_GT_MASK, 1018 set_mode_system_value(b, mode); 1019 break; 1020 case SpvBuiltInSubgroupLeMask: 1021 *location = SYSTEM_VALUE_SUBGROUP_LE_MASK, 1022 set_mode_system_value(b, mode); 1023 break; 1024 case SpvBuiltInSubgroupLtMask: 1025 *location = SYSTEM_VALUE_SUBGROUP_LT_MASK, 1026 set_mode_system_value(b, mode); 1027 break; 1028 case SpvBuiltInFragStencilRefEXT: 1029 *location = FRAG_RESULT_STENCIL; 1030 vtn_assert(*mode == nir_var_shader_out); 1031 break; 1032 case SpvBuiltInWorkDim: 1033 *location = SYSTEM_VALUE_WORK_DIM; 1034 set_mode_system_value(b, mode); 1035 break; 1036 case SpvBuiltInGlobalSize: 1037 *location = SYSTEM_VALUE_GLOBAL_GROUP_SIZE; 1038 set_mode_system_value(b, mode); 1039 break; 1040 case SpvBuiltInBaryCoordNoPerspAMD: 1041 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL; 1042 set_mode_system_value(b, mode); 1043 break; 1044 case SpvBuiltInBaryCoordNoPerspCentroidAMD: 1045 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID; 1046 set_mode_system_value(b, mode); 1047 break; 1048 case SpvBuiltInBaryCoordNoPerspSampleAMD: 1049 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE; 1050 set_mode_system_value(b, mode); 1051 break; 1052 case SpvBuiltInBaryCoordSmoothAMD: 1053 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL; 1054 set_mode_system_value(b, mode); 1055 break; 1056 case SpvBuiltInBaryCoordSmoothCentroidAMD: 1057 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID; 1058 set_mode_system_value(b, mode); 1059 break; 1060 case SpvBuiltInBaryCoordSmoothSampleAMD: 1061 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE; 1062 set_mode_system_value(b, mode); 1063 break; 1064 case SpvBuiltInBaryCoordPullModelAMD: 1065 *location = SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL; 1066 set_mode_system_value(b, mode); 1067 break; 1068 case SpvBuiltInLaunchIdKHR: 1069 *location = SYSTEM_VALUE_RAY_LAUNCH_ID; 1070 set_mode_system_value(b, mode); 1071 break; 1072 case SpvBuiltInLaunchSizeKHR: 1073 *location = SYSTEM_VALUE_RAY_LAUNCH_SIZE; 1074 set_mode_system_value(b, mode); 1075 break; 1076 case SpvBuiltInWorldRayOriginKHR: 1077 *location = SYSTEM_VALUE_RAY_WORLD_ORIGIN; 1078 set_mode_system_value(b, mode); 1079 break; 1080 case SpvBuiltInWorldRayDirectionKHR: 1081 *location = SYSTEM_VALUE_RAY_WORLD_DIRECTION; 1082 set_mode_system_value(b, mode); 1083 break; 1084 case SpvBuiltInObjectRayOriginKHR: 1085 *location = SYSTEM_VALUE_RAY_OBJECT_ORIGIN; 1086 set_mode_system_value(b, mode); 1087 break; 1088 case SpvBuiltInObjectRayDirectionKHR: 1089 *location = SYSTEM_VALUE_RAY_OBJECT_DIRECTION; 1090 set_mode_system_value(b, mode); 1091 break; 1092 case SpvBuiltInObjectToWorldKHR: 1093 *location = SYSTEM_VALUE_RAY_OBJECT_TO_WORLD; 1094 set_mode_system_value(b, mode); 1095 break; 1096 case SpvBuiltInWorldToObjectKHR: 1097 *location = SYSTEM_VALUE_RAY_WORLD_TO_OBJECT; 1098 set_mode_system_value(b, mode); 1099 break; 1100 case SpvBuiltInRayTminKHR: 1101 *location = SYSTEM_VALUE_RAY_T_MIN; 1102 set_mode_system_value(b, mode); 1103 break; 1104 case SpvBuiltInRayTmaxKHR: 1105 case SpvBuiltInHitTNV: 1106 *location = SYSTEM_VALUE_RAY_T_MAX; 1107 set_mode_system_value(b, mode); 1108 break; 1109 case SpvBuiltInInstanceCustomIndexKHR: 1110 *location = SYSTEM_VALUE_RAY_INSTANCE_CUSTOM_INDEX; 1111 set_mode_system_value(b, mode); 1112 break; 1113 case SpvBuiltInHitKindKHR: 1114 *location = SYSTEM_VALUE_RAY_HIT_KIND; 1115 set_mode_system_value(b, mode); 1116 break; 1117 case SpvBuiltInIncomingRayFlagsKHR: 1118 *location = SYSTEM_VALUE_RAY_FLAGS; 1119 set_mode_system_value(b, mode); 1120 break; 1121 case SpvBuiltInRayGeometryIndexKHR: 1122 *location = SYSTEM_VALUE_RAY_GEOMETRY_INDEX; 1123 set_mode_system_value(b, mode); 1124 break; 1125 case SpvBuiltInCullMaskKHR: 1126 *location = SYSTEM_VALUE_CULL_MASK; 1127 set_mode_system_value(b, mode); 1128 break; 1129 case SpvBuiltInShadingRateKHR: 1130 *location = SYSTEM_VALUE_FRAG_SHADING_RATE; 1131 set_mode_system_value(b, mode); 1132 break; 1133 case SpvBuiltInPrimitiveShadingRateKHR: 1134 if (b->shader->info.stage == MESA_SHADER_VERTEX || 1135 b->shader->info.stage == MESA_SHADER_GEOMETRY || 1136 b->shader->info.stage == MESA_SHADER_MESH) { 1137 *location = VARYING_SLOT_PRIMITIVE_SHADING_RATE; 1138 *mode = nir_var_shader_out; 1139 } else { 1140 vtn_fail("invalid stage for SpvBuiltInPrimitiveShadingRateKHR"); 1141 } 1142 break; 1143 case SpvBuiltInPrimitiveCountNV: 1144 *location = VARYING_SLOT_PRIMITIVE_COUNT; 1145 break; 1146 case SpvBuiltInPrimitiveIndicesNV: 1147 *location = VARYING_SLOT_PRIMITIVE_INDICES; 1148 break; 1149 case SpvBuiltInTaskCountNV: 1150 /* NV_mesh_shader only. */ 1151 *location = VARYING_SLOT_TASK_COUNT; 1152 *mode = nir_var_shader_out; 1153 break; 1154 case SpvBuiltInMeshViewCountNV: 1155 *location = SYSTEM_VALUE_MESH_VIEW_COUNT; 1156 set_mode_system_value(b, mode); 1157 break; 1158 case SpvBuiltInMeshViewIndicesNV: 1159 *location = SYSTEM_VALUE_MESH_VIEW_INDICES; 1160 set_mode_system_value(b, mode); 1161 break; 1162 default: 1163 vtn_fail("Unsupported builtin: %s (%u)", 1164 spirv_builtin_to_string(builtin), builtin); 1165 } 1166} 1167 1168static void 1169apply_var_decoration(struct vtn_builder *b, 1170 struct nir_variable_data *var_data, 1171 const struct vtn_decoration *dec) 1172{ 1173 switch (dec->decoration) { 1174 case SpvDecorationRelaxedPrecision: 1175 var_data->precision = GLSL_PRECISION_MEDIUM; 1176 break; 1177 case SpvDecorationNoPerspective: 1178 var_data->interpolation = INTERP_MODE_NOPERSPECTIVE; 1179 break; 1180 case SpvDecorationFlat: 1181 var_data->interpolation = INTERP_MODE_FLAT; 1182 break; 1183 case SpvDecorationExplicitInterpAMD: 1184 var_data->interpolation = INTERP_MODE_EXPLICIT; 1185 break; 1186 case SpvDecorationCentroid: 1187 var_data->centroid = true; 1188 break; 1189 case SpvDecorationSample: 1190 var_data->sample = true; 1191 break; 1192 case SpvDecorationInvariant: 1193 var_data->invariant = true; 1194 break; 1195 case SpvDecorationConstant: 1196 var_data->read_only = true; 1197 break; 1198 case SpvDecorationNonReadable: 1199 var_data->access |= ACCESS_NON_READABLE; 1200 break; 1201 case SpvDecorationNonWritable: 1202 var_data->read_only = true; 1203 var_data->access |= ACCESS_NON_WRITEABLE; 1204 break; 1205 case SpvDecorationRestrict: 1206 var_data->access |= ACCESS_RESTRICT; 1207 break; 1208 case SpvDecorationAliased: 1209 var_data->access &= ~ACCESS_RESTRICT; 1210 break; 1211 case SpvDecorationVolatile: 1212 var_data->access |= ACCESS_VOLATILE; 1213 break; 1214 case SpvDecorationCoherent: 1215 var_data->access |= ACCESS_COHERENT; 1216 break; 1217 case SpvDecorationComponent: 1218 var_data->location_frac = dec->operands[0]; 1219 break; 1220 case SpvDecorationIndex: 1221 var_data->index = dec->operands[0]; 1222 break; 1223 case SpvDecorationBuiltIn: { 1224 SpvBuiltIn builtin = dec->operands[0]; 1225 1226 nir_variable_mode mode = var_data->mode; 1227 vtn_get_builtin_location(b, builtin, &var_data->location, &mode); 1228 var_data->mode = mode; 1229 1230 switch (builtin) { 1231 case SpvBuiltInTessLevelOuter: 1232 case SpvBuiltInTessLevelInner: 1233 case SpvBuiltInClipDistance: 1234 case SpvBuiltInClipDistancePerViewNV: 1235 case SpvBuiltInCullDistance: 1236 case SpvBuiltInCullDistancePerViewNV: 1237 var_data->compact = true; 1238 break; 1239 default: 1240 break; 1241 } 1242 1243 break; 1244 } 1245 1246 case SpvDecorationSpecId: 1247 case SpvDecorationRowMajor: 1248 case SpvDecorationColMajor: 1249 case SpvDecorationMatrixStride: 1250 case SpvDecorationUniform: 1251 case SpvDecorationUniformId: 1252 case SpvDecorationLinkageAttributes: 1253 break; /* Do nothing with these here */ 1254 1255 case SpvDecorationPatch: 1256 var_data->patch = true; 1257 break; 1258 1259 case SpvDecorationLocation: 1260 vtn_fail("Should be handled earlier by var_decoration_cb()"); 1261 1262 case SpvDecorationBlock: 1263 case SpvDecorationBufferBlock: 1264 case SpvDecorationArrayStride: 1265 case SpvDecorationGLSLShared: 1266 case SpvDecorationGLSLPacked: 1267 break; /* These can apply to a type but we don't care about them */ 1268 1269 case SpvDecorationBinding: 1270 case SpvDecorationDescriptorSet: 1271 case SpvDecorationNoContraction: 1272 case SpvDecorationInputAttachmentIndex: 1273 vtn_warn("Decoration not allowed for variable or structure member: %s", 1274 spirv_decoration_to_string(dec->decoration)); 1275 break; 1276 1277 case SpvDecorationXfbBuffer: 1278 var_data->explicit_xfb_buffer = true; 1279 var_data->xfb.buffer = dec->operands[0]; 1280 var_data->always_active_io = true; 1281 break; 1282 case SpvDecorationXfbStride: 1283 var_data->explicit_xfb_stride = true; 1284 var_data->xfb.stride = dec->operands[0]; 1285 break; 1286 case SpvDecorationOffset: 1287 var_data->explicit_offset = true; 1288 var_data->offset = dec->operands[0]; 1289 break; 1290 1291 case SpvDecorationStream: 1292 var_data->stream = dec->operands[0]; 1293 break; 1294 1295 case SpvDecorationCPacked: 1296 case SpvDecorationSaturatedConversion: 1297 case SpvDecorationFuncParamAttr: 1298 case SpvDecorationFPRoundingMode: 1299 case SpvDecorationFPFastMathMode: 1300 case SpvDecorationAlignment: 1301 if (b->shader->info.stage != MESA_SHADER_KERNEL) { 1302 vtn_warn("Decoration only allowed for CL-style kernels: %s", 1303 spirv_decoration_to_string(dec->decoration)); 1304 } 1305 break; 1306 1307 case SpvDecorationUserSemantic: 1308 case SpvDecorationUserTypeGOOGLE: 1309 /* User semantic decorations can safely be ignored by the driver. */ 1310 break; 1311 1312 case SpvDecorationRestrictPointerEXT: 1313 case SpvDecorationAliasedPointerEXT: 1314 /* TODO: We should actually plumb alias information through NIR. */ 1315 break; 1316 1317 case SpvDecorationPerPrimitiveNV: 1318 vtn_fail_if( 1319 !(b->shader->info.stage == MESA_SHADER_MESH && var_data->mode == nir_var_shader_out) && 1320 !(b->shader->info.stage == MESA_SHADER_FRAGMENT && var_data->mode == nir_var_shader_in), 1321 "PerPrimitiveNV decoration only allowed for Mesh shader outputs or Fragment shader inputs"); 1322 var_data->per_primitive = true; 1323 break; 1324 1325 case SpvDecorationPerTaskNV: 1326 vtn_fail_if( 1327 (b->shader->info.stage != MESA_SHADER_MESH && 1328 b->shader->info.stage != MESA_SHADER_TASK) || 1329 var_data->mode != nir_var_mem_task_payload, 1330 "PerTaskNV decoration only allowed on Task/Mesh payload variables."); 1331 break; 1332 1333 case SpvDecorationPerViewNV: 1334 vtn_fail_if(b->shader->info.stage != MESA_SHADER_MESH, 1335 "PerViewNV decoration only allowed in Mesh shaders"); 1336 var_data->per_view = true; 1337 break; 1338 1339 default: 1340 vtn_fail_with_decoration("Unhandled decoration", dec->decoration); 1341 } 1342} 1343 1344static void 1345gather_var_kind_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1346 const struct vtn_decoration *dec, void *void_var) 1347{ 1348 struct vtn_variable *vtn_var = void_var; 1349 switch (dec->decoration) { 1350 case SpvDecorationPatch: 1351 vtn_var->var->data.patch = true; 1352 break; 1353 case SpvDecorationPerPrimitiveNV: 1354 vtn_var->var->data.per_primitive = true; 1355 break; 1356 case SpvDecorationPerViewNV: 1357 vtn_var->var->data.per_view = true; 1358 break; 1359 default: 1360 /* Nothing to do. */ 1361 break; 1362 } 1363} 1364 1365static void 1366var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member, 1367 const struct vtn_decoration *dec, void *void_var) 1368{ 1369 struct vtn_variable *vtn_var = void_var; 1370 1371 /* Handle decorations that apply to a vtn_variable as a whole */ 1372 switch (dec->decoration) { 1373 case SpvDecorationBinding: 1374 vtn_var->binding = dec->operands[0]; 1375 vtn_var->explicit_binding = true; 1376 return; 1377 case SpvDecorationDescriptorSet: 1378 vtn_var->descriptor_set = dec->operands[0]; 1379 return; 1380 case SpvDecorationInputAttachmentIndex: 1381 vtn_var->input_attachment_index = dec->operands[0]; 1382 return; 1383 case SpvDecorationPatch: 1384 vtn_var->var->data.patch = true; 1385 break; 1386 case SpvDecorationOffset: 1387 vtn_var->offset = dec->operands[0]; 1388 break; 1389 case SpvDecorationNonWritable: 1390 vtn_var->access |= ACCESS_NON_WRITEABLE; 1391 break; 1392 case SpvDecorationNonReadable: 1393 vtn_var->access |= ACCESS_NON_READABLE; 1394 break; 1395 case SpvDecorationVolatile: 1396 vtn_var->access |= ACCESS_VOLATILE; 1397 break; 1398 case SpvDecorationCoherent: 1399 vtn_var->access |= ACCESS_COHERENT; 1400 break; 1401 case SpvDecorationCounterBuffer: 1402 /* Counter buffer decorations can safely be ignored by the driver. */ 1403 return; 1404 default: 1405 break; 1406 } 1407 1408 if (val->value_type == vtn_value_type_pointer) { 1409 assert(val->pointer->var == void_var); 1410 assert(member == -1); 1411 } else { 1412 assert(val->value_type == vtn_value_type_type); 1413 } 1414 1415 /* Location is odd. If applied to a split structure, we have to walk the 1416 * whole thing and accumulate the location. It's easier to handle as a 1417 * special case. 1418 */ 1419 if (dec->decoration == SpvDecorationLocation) { 1420 unsigned location = dec->operands[0]; 1421 if (b->shader->info.stage == MESA_SHADER_FRAGMENT && 1422 vtn_var->mode == vtn_variable_mode_output) { 1423 location += FRAG_RESULT_DATA0; 1424 } else if (b->shader->info.stage == MESA_SHADER_VERTEX && 1425 vtn_var->mode == vtn_variable_mode_input) { 1426 location += VERT_ATTRIB_GENERIC0; 1427 } else if (vtn_var->mode == vtn_variable_mode_input || 1428 vtn_var->mode == vtn_variable_mode_output) { 1429 location += vtn_var->var->data.patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0; 1430 } else if (vtn_var->mode == vtn_variable_mode_call_data || 1431 vtn_var->mode == vtn_variable_mode_ray_payload) { 1432 /* This location is fine as-is */ 1433 } else if (vtn_var->mode != vtn_variable_mode_uniform && 1434 vtn_var->mode != vtn_variable_mode_image) { 1435 vtn_warn("Location must be on input, output, uniform, sampler or " 1436 "image variable"); 1437 return; 1438 } 1439 1440 if (vtn_var->var->num_members == 0) { 1441 /* This handles the member and lone variable cases */ 1442 vtn_var->var->data.location = location; 1443 } else { 1444 /* This handles the structure member case */ 1445 assert(vtn_var->var->members); 1446 1447 if (member == -1) 1448 vtn_var->base_location = location; 1449 else 1450 vtn_var->var->members[member].location = location; 1451 } 1452 1453 return; 1454 } else { 1455 if (vtn_var->var) { 1456 if (vtn_var->var->num_members == 0) { 1457 /* We call this function on types as well as variables and not all 1458 * struct types get split so we can end up having stray member 1459 * decorations; just ignore them. 1460 */ 1461 if (member == -1) 1462 apply_var_decoration(b, &vtn_var->var->data, dec); 1463 } else if (member >= 0) { 1464 /* Member decorations must come from a type */ 1465 assert(val->value_type == vtn_value_type_type); 1466 apply_var_decoration(b, &vtn_var->var->members[member], dec); 1467 } else { 1468 unsigned length = 1469 glsl_get_length(glsl_without_array(vtn_var->type->type)); 1470 for (unsigned i = 0; i < length; i++) 1471 apply_var_decoration(b, &vtn_var->var->members[i], dec); 1472 } 1473 } else { 1474 /* A few variables, those with external storage, have no actual 1475 * nir_variables associated with them. Fortunately, all decorations 1476 * we care about for those variables are on the type only. 1477 */ 1478 vtn_assert(vtn_var->mode == vtn_variable_mode_ubo || 1479 vtn_var->mode == vtn_variable_mode_ssbo || 1480 vtn_var->mode == vtn_variable_mode_push_constant); 1481 } 1482 } 1483} 1484 1485enum vtn_variable_mode 1486vtn_storage_class_to_mode(struct vtn_builder *b, 1487 SpvStorageClass class, 1488 struct vtn_type *interface_type, 1489 nir_variable_mode *nir_mode_out) 1490{ 1491 enum vtn_variable_mode mode; 1492 nir_variable_mode nir_mode; 1493 switch (class) { 1494 case SpvStorageClassUniform: 1495 /* Assume it's an UBO if we lack the interface_type. */ 1496 if (!interface_type || interface_type->block) { 1497 mode = vtn_variable_mode_ubo; 1498 nir_mode = nir_var_mem_ubo; 1499 } else if (interface_type->buffer_block) { 1500 mode = vtn_variable_mode_ssbo; 1501 nir_mode = nir_var_mem_ssbo; 1502 } else { 1503 /* Default-block uniforms, coming from gl_spirv */ 1504 mode = vtn_variable_mode_uniform; 1505 nir_mode = nir_var_uniform; 1506 } 1507 break; 1508 case SpvStorageClassStorageBuffer: 1509 mode = vtn_variable_mode_ssbo; 1510 nir_mode = nir_var_mem_ssbo; 1511 break; 1512 case SpvStorageClassPhysicalStorageBuffer: 1513 mode = vtn_variable_mode_phys_ssbo; 1514 nir_mode = nir_var_mem_global; 1515 break; 1516 case SpvStorageClassUniformConstant: 1517 /* interface_type is only NULL when OpTypeForwardPointer is used and 1518 * OpTypeForwardPointer can only be used for struct types, not images or 1519 * acceleration structures. 1520 */ 1521 if (interface_type) 1522 interface_type = vtn_type_without_array(interface_type); 1523 1524 if (interface_type && 1525 interface_type->base_type == vtn_base_type_image && 1526 glsl_type_is_image(interface_type->glsl_image)) { 1527 mode = vtn_variable_mode_image; 1528 nir_mode = nir_var_image; 1529 } else if (b->shader->info.stage == MESA_SHADER_KERNEL) { 1530 mode = vtn_variable_mode_constant; 1531 nir_mode = nir_var_mem_constant; 1532 } else { 1533 /* interface_type is only NULL when OpTypeForwardPointer is used and 1534 * OpTypeForwardPointer cannot be used with the UniformConstant 1535 * storage class. 1536 */ 1537 assert(interface_type != NULL); 1538 if (interface_type->base_type == vtn_base_type_accel_struct) { 1539 mode = vtn_variable_mode_accel_struct; 1540 nir_mode = nir_var_uniform; 1541 } else { 1542 mode = vtn_variable_mode_uniform; 1543 nir_mode = nir_var_uniform; 1544 } 1545 } 1546 break; 1547 case SpvStorageClassPushConstant: 1548 mode = vtn_variable_mode_push_constant; 1549 nir_mode = nir_var_mem_push_const; 1550 break; 1551 case SpvStorageClassInput: 1552 mode = vtn_variable_mode_input; 1553 nir_mode = nir_var_shader_in; 1554 1555 /* NV_mesh_shader: fixup due to lack of dedicated storage class */ 1556 if (b->shader->info.stage == MESA_SHADER_MESH) { 1557 mode = vtn_variable_mode_task_payload; 1558 nir_mode = nir_var_mem_task_payload; 1559 } 1560 break; 1561 case SpvStorageClassOutput: 1562 mode = vtn_variable_mode_output; 1563 nir_mode = nir_var_shader_out; 1564 1565 /* NV_mesh_shader: fixup due to lack of dedicated storage class */ 1566 if (b->shader->info.stage == MESA_SHADER_TASK) { 1567 mode = vtn_variable_mode_task_payload; 1568 nir_mode = nir_var_mem_task_payload; 1569 } 1570 break; 1571 case SpvStorageClassPrivate: 1572 mode = vtn_variable_mode_private; 1573 nir_mode = nir_var_shader_temp; 1574 break; 1575 case SpvStorageClassFunction: 1576 mode = vtn_variable_mode_function; 1577 nir_mode = nir_var_function_temp; 1578 break; 1579 case SpvStorageClassWorkgroup: 1580 mode = vtn_variable_mode_workgroup; 1581 nir_mode = nir_var_mem_shared; 1582 break; 1583 case SpvStorageClassAtomicCounter: 1584 mode = vtn_variable_mode_atomic_counter; 1585 nir_mode = nir_var_uniform; 1586 break; 1587 case SpvStorageClassCrossWorkgroup: 1588 mode = vtn_variable_mode_cross_workgroup; 1589 nir_mode = nir_var_mem_global; 1590 break; 1591 case SpvStorageClassImage: 1592 mode = vtn_variable_mode_image; 1593 nir_mode = nir_var_image; 1594 break; 1595 case SpvStorageClassCallableDataKHR: 1596 mode = vtn_variable_mode_call_data; 1597 nir_mode = nir_var_shader_temp; 1598 break; 1599 case SpvStorageClassIncomingCallableDataKHR: 1600 mode = vtn_variable_mode_call_data_in; 1601 nir_mode = nir_var_shader_call_data; 1602 break; 1603 case SpvStorageClassRayPayloadKHR: 1604 mode = vtn_variable_mode_ray_payload; 1605 nir_mode = nir_var_shader_temp; 1606 break; 1607 case SpvStorageClassIncomingRayPayloadKHR: 1608 mode = vtn_variable_mode_ray_payload_in; 1609 nir_mode = nir_var_shader_call_data; 1610 break; 1611 case SpvStorageClassHitAttributeKHR: 1612 mode = vtn_variable_mode_hit_attrib; 1613 nir_mode = nir_var_ray_hit_attrib; 1614 break; 1615 case SpvStorageClassShaderRecordBufferKHR: 1616 mode = vtn_variable_mode_shader_record; 1617 nir_mode = nir_var_mem_constant; 1618 break; 1619 1620 case SpvStorageClassGeneric: 1621 mode = vtn_variable_mode_generic; 1622 nir_mode = nir_var_mem_generic; 1623 break; 1624 default: 1625 vtn_fail("Unhandled variable storage class: %s (%u)", 1626 spirv_storageclass_to_string(class), class); 1627 } 1628 1629 if (nir_mode_out) 1630 *nir_mode_out = nir_mode; 1631 1632 return mode; 1633} 1634 1635nir_address_format 1636vtn_mode_to_address_format(struct vtn_builder *b, enum vtn_variable_mode mode) 1637{ 1638 switch (mode) { 1639 case vtn_variable_mode_ubo: 1640 return b->options->ubo_addr_format; 1641 1642 case vtn_variable_mode_ssbo: 1643 return b->options->ssbo_addr_format; 1644 1645 case vtn_variable_mode_phys_ssbo: 1646 return b->options->phys_ssbo_addr_format; 1647 1648 case vtn_variable_mode_push_constant: 1649 return b->options->push_const_addr_format; 1650 1651 case vtn_variable_mode_workgroup: 1652 return b->options->shared_addr_format; 1653 1654 case vtn_variable_mode_generic: 1655 case vtn_variable_mode_cross_workgroup: 1656 return b->options->global_addr_format; 1657 1658 case vtn_variable_mode_shader_record: 1659 case vtn_variable_mode_constant: 1660 return b->options->constant_addr_format; 1661 1662 case vtn_variable_mode_accel_struct: 1663 return nir_address_format_64bit_global; 1664 1665 case vtn_variable_mode_task_payload: 1666 return b->options->task_payload_addr_format; 1667 1668 case vtn_variable_mode_function: 1669 if (b->physical_ptrs) 1670 return b->options->temp_addr_format; 1671 FALLTHROUGH; 1672 1673 case vtn_variable_mode_private: 1674 case vtn_variable_mode_uniform: 1675 case vtn_variable_mode_atomic_counter: 1676 case vtn_variable_mode_input: 1677 case vtn_variable_mode_output: 1678 case vtn_variable_mode_image: 1679 case vtn_variable_mode_call_data: 1680 case vtn_variable_mode_call_data_in: 1681 case vtn_variable_mode_ray_payload: 1682 case vtn_variable_mode_ray_payload_in: 1683 case vtn_variable_mode_hit_attrib: 1684 return nir_address_format_logical; 1685 } 1686 1687 unreachable("Invalid variable mode"); 1688} 1689 1690nir_ssa_def * 1691vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr) 1692{ 1693 if ((vtn_pointer_is_external_block(b, ptr) && 1694 vtn_type_contains_block(b, ptr->type) && 1695 ptr->mode != vtn_variable_mode_phys_ssbo) || 1696 ptr->mode == vtn_variable_mode_accel_struct) { 1697 /* In this case, we're looking for a block index and not an actual 1698 * deref. 1699 * 1700 * For PhysicalStorageBuffer pointers, we don't have a block index 1701 * at all because we get the pointer directly from the client. This 1702 * assumes that there will never be a SSBO binding variable using the 1703 * PhysicalStorageBuffer storage class. This assumption appears 1704 * to be correct according to the Vulkan spec because the table, 1705 * "Shader Resource and Storage Class Correspondence," the only the 1706 * Uniform storage class with BufferBlock or the StorageBuffer 1707 * storage class with Block can be used. 1708 */ 1709 if (!ptr->block_index) { 1710 /* If we don't have a block_index then we must be a pointer to the 1711 * variable itself. 1712 */ 1713 vtn_assert(!ptr->deref); 1714 1715 struct vtn_access_chain chain = { 1716 .length = 0, 1717 }; 1718 ptr = vtn_pointer_dereference(b, ptr, &chain); 1719 } 1720 1721 return ptr->block_index; 1722 } else { 1723 return &vtn_pointer_to_deref(b, ptr)->dest.ssa; 1724 } 1725} 1726 1727struct vtn_pointer * 1728vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa, 1729 struct vtn_type *ptr_type) 1730{ 1731 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 1732 1733 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer); 1734 struct vtn_type *without_array = 1735 vtn_type_without_array(ptr_type->deref); 1736 1737 nir_variable_mode nir_mode; 1738 ptr->mode = vtn_storage_class_to_mode(b, ptr_type->storage_class, 1739 without_array, &nir_mode); 1740 ptr->type = ptr_type->deref; 1741 ptr->ptr_type = ptr_type; 1742 1743 const struct glsl_type *deref_type = 1744 vtn_type_get_nir_type(b, ptr_type->deref, ptr->mode); 1745 if (!vtn_pointer_is_external_block(b, ptr) && 1746 ptr->mode != vtn_variable_mode_accel_struct) { 1747 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode, 1748 deref_type, ptr_type->stride); 1749 } else if ((vtn_type_contains_block(b, ptr->type) && 1750 ptr->mode != vtn_variable_mode_phys_ssbo) || 1751 ptr->mode == vtn_variable_mode_accel_struct) { 1752 /* This is a pointer to somewhere in an array of blocks, not a 1753 * pointer to somewhere inside the block. Set the block index 1754 * instead of making a cast. 1755 */ 1756 ptr->block_index = ssa; 1757 } else { 1758 /* This is a pointer to something internal or a pointer inside a 1759 * block. It's just a regular cast. 1760 * 1761 * For PhysicalStorageBuffer pointers, we don't have a block index 1762 * at all because we get the pointer directly from the client. This 1763 * assumes that there will never be a SSBO binding variable using the 1764 * PhysicalStorageBuffer storage class. This assumption appears 1765 * to be correct according to the Vulkan spec because the table, 1766 * "Shader Resource and Storage Class Correspondence," the only the 1767 * Uniform storage class with BufferBlock or the StorageBuffer 1768 * storage class with Block can be used. 1769 */ 1770 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode, 1771 deref_type, ptr_type->stride); 1772 ptr->deref->dest.ssa.num_components = 1773 glsl_get_vector_elements(ptr_type->type); 1774 ptr->deref->dest.ssa.bit_size = glsl_get_bit_size(ptr_type->type); 1775 } 1776 1777 return ptr; 1778} 1779 1780static void 1781assign_missing_member_locations(struct vtn_variable *var) 1782{ 1783 unsigned length = 1784 glsl_get_length(glsl_without_array(var->type->type)); 1785 int location = var->base_location; 1786 1787 for (unsigned i = 0; i < length; i++) { 1788 /* From the Vulkan spec: 1789 * 1790 * “If the structure type is a Block but without a Location, then each 1791 * of its members must have a Location decoration.” 1792 * 1793 */ 1794 if (var->type->block) { 1795 assert(var->base_location != -1 || 1796 var->var->members[i].location != -1); 1797 } 1798 1799 /* From the Vulkan spec: 1800 * 1801 * “Any member with its own Location decoration is assigned that 1802 * location. Each remaining member is assigned the location after the 1803 * immediately preceding member in declaration order.” 1804 */ 1805 if (var->var->members[i].location != -1) 1806 location = var->var->members[i].location; 1807 else 1808 var->var->members[i].location = location; 1809 1810 /* Below we use type instead of interface_type, because interface_type 1811 * is only available when it is a Block. This code also supports 1812 * input/outputs that are just structs 1813 */ 1814 const struct glsl_type *member_type = 1815 glsl_get_struct_field(glsl_without_array(var->type->type), i); 1816 1817 location += 1818 glsl_count_attribute_slots(member_type, 1819 false /* is_gl_vertex_input */); 1820 } 1821} 1822 1823nir_deref_instr * 1824vtn_get_call_payload_for_location(struct vtn_builder *b, uint32_t location_id) 1825{ 1826 uint32_t location = vtn_constant_uint(b, location_id); 1827 nir_foreach_variable_with_modes(var, b->nb.shader, nir_var_shader_temp) { 1828 if (var->data.explicit_location && 1829 var->data.location == location) 1830 return nir_build_deref_var(&b->nb, var); 1831 } 1832 vtn_fail("Couldn't find variable with a storage class of CallableDataKHR " 1833 "or RayPayloadKHR and location %d", location); 1834} 1835 1836static bool 1837vtn_type_is_ray_query(struct vtn_type *type) 1838{ 1839 return vtn_type_without_array(type)->base_type == vtn_base_type_ray_query; 1840} 1841 1842static void 1843vtn_create_variable(struct vtn_builder *b, struct vtn_value *val, 1844 struct vtn_type *ptr_type, SpvStorageClass storage_class, 1845 struct vtn_value *initializer) 1846{ 1847 vtn_assert(ptr_type->base_type == vtn_base_type_pointer); 1848 struct vtn_type *type = ptr_type->deref; 1849 1850 struct vtn_type *without_array = vtn_type_without_array(ptr_type->deref); 1851 1852 enum vtn_variable_mode mode; 1853 nir_variable_mode nir_mode; 1854 mode = vtn_storage_class_to_mode(b, storage_class, without_array, &nir_mode); 1855 1856 switch (mode) { 1857 case vtn_variable_mode_ubo: 1858 /* There's no other way to get vtn_variable_mode_ubo */ 1859 vtn_assert(without_array->block); 1860 break; 1861 case vtn_variable_mode_ssbo: 1862 if (storage_class == SpvStorageClassStorageBuffer && 1863 !without_array->block) { 1864 if (b->variable_pointers) { 1865 vtn_fail("Variables in the StorageBuffer storage class must " 1866 "have a struct type with the Block decoration"); 1867 } else { 1868 /* If variable pointers are not present, it's still malformed 1869 * SPIR-V but we can parse it and do the right thing anyway. 1870 * Since some of the 8-bit storage tests have bugs in this are, 1871 * just make it a warning for now. 1872 */ 1873 vtn_warn("Variables in the StorageBuffer storage class must " 1874 "have a struct type with the Block decoration"); 1875 } 1876 } 1877 break; 1878 1879 case vtn_variable_mode_generic: 1880 vtn_fail("Cannot create a variable with the Generic storage class"); 1881 break; 1882 1883 case vtn_variable_mode_image: 1884 if (storage_class == SpvStorageClassImage) 1885 vtn_fail("Cannot create a variable with the Image storage class"); 1886 else 1887 vtn_assert(storage_class == SpvStorageClassUniformConstant); 1888 break; 1889 1890 case vtn_variable_mode_phys_ssbo: 1891 vtn_fail("Cannot create a variable with the " 1892 "PhysicalStorageBuffer storage class"); 1893 break; 1894 1895 default: 1896 /* No tallying is needed */ 1897 break; 1898 } 1899 1900 struct vtn_variable *var = rzalloc(b, struct vtn_variable); 1901 var->type = type; 1902 var->mode = mode; 1903 var->base_location = -1; 1904 1905 val->pointer = rzalloc(b, struct vtn_pointer); 1906 val->pointer->mode = var->mode; 1907 val->pointer->type = var->type; 1908 val->pointer->ptr_type = ptr_type; 1909 val->pointer->var = var; 1910 val->pointer->access = var->type->access; 1911 1912 switch (var->mode) { 1913 case vtn_variable_mode_function: 1914 case vtn_variable_mode_private: 1915 case vtn_variable_mode_uniform: 1916 case vtn_variable_mode_atomic_counter: 1917 case vtn_variable_mode_constant: 1918 case vtn_variable_mode_call_data: 1919 case vtn_variable_mode_call_data_in: 1920 case vtn_variable_mode_image: 1921 case vtn_variable_mode_ray_payload: 1922 case vtn_variable_mode_ray_payload_in: 1923 case vtn_variable_mode_hit_attrib: 1924 /* For these, we create the variable normally */ 1925 var->var = rzalloc(b->shader, nir_variable); 1926 var->var->name = ralloc_strdup(var->var, val->name); 1927 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); 1928 1929 /* This is a total hack but we need some way to flag variables which are 1930 * going to be call payloads. See get_call_payload_deref. 1931 */ 1932 if (storage_class == SpvStorageClassCallableDataKHR || 1933 storage_class == SpvStorageClassRayPayloadKHR) 1934 var->var->data.explicit_location = true; 1935 1936 var->var->data.mode = nir_mode; 1937 var->var->data.location = -1; 1938 var->var->data.ray_query = vtn_type_is_ray_query(var->type); 1939 var->var->interface_type = NULL; 1940 break; 1941 1942 case vtn_variable_mode_ubo: 1943 case vtn_variable_mode_ssbo: 1944 case vtn_variable_mode_push_constant: 1945 case vtn_variable_mode_accel_struct: 1946 case vtn_variable_mode_shader_record: 1947 var->var = rzalloc(b->shader, nir_variable); 1948 var->var->name = ralloc_strdup(var->var, val->name); 1949 1950 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); 1951 var->var->interface_type = var->var->type; 1952 1953 var->var->data.mode = nir_mode; 1954 var->var->data.location = -1; 1955 var->var->data.driver_location = 0; 1956 var->var->data.access = var->type->access; 1957 break; 1958 1959 case vtn_variable_mode_workgroup: 1960 case vtn_variable_mode_cross_workgroup: 1961 case vtn_variable_mode_task_payload: 1962 /* Create the variable normally */ 1963 var->var = rzalloc(b->shader, nir_variable); 1964 var->var->name = ralloc_strdup(var->var, val->name); 1965 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); 1966 var->var->data.mode = nir_mode; 1967 break; 1968 1969 case vtn_variable_mode_input: 1970 case vtn_variable_mode_output: { 1971 var->var = rzalloc(b->shader, nir_variable); 1972 var->var->name = ralloc_strdup(var->var, val->name); 1973 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode); 1974 var->var->data.mode = nir_mode; 1975 1976 /* In order to know whether or not we're a per-vertex inout, we need 1977 * the patch qualifier. This means walking the variable decorations 1978 * early before we actually create any variables. Not a big deal. 1979 * 1980 * GLSLang really likes to place decorations in the most interior 1981 * thing it possibly can. In particular, if you have a struct, it 1982 * will place the patch decorations on the struct members. This 1983 * should be handled by the variable splitting below just fine. 1984 * 1985 * If you have an array-of-struct, things get even more weird as it 1986 * will place the patch decorations on the struct even though it's 1987 * inside an array and some of the members being patch and others not 1988 * makes no sense whatsoever. Since the only sensible thing is for 1989 * it to be all or nothing, we'll call it patch if any of the members 1990 * are declared patch. 1991 */ 1992 vtn_foreach_decoration(b, val, gather_var_kind_cb, var); 1993 if (glsl_type_is_array(var->type->type) && 1994 glsl_type_is_struct_or_ifc(without_array->type)) { 1995 vtn_foreach_decoration(b, vtn_value(b, without_array->id, 1996 vtn_value_type_type), 1997 gather_var_kind_cb, var); 1998 } 1999 2000 struct vtn_type *per_vertex_type = var->type; 2001 if (nir_is_arrayed_io(var->var, b->shader->info.stage)) 2002 per_vertex_type = var->type->array_element; 2003 2004 /* Figure out the interface block type. */ 2005 struct vtn_type *iface_type = per_vertex_type; 2006 if (var->mode == vtn_variable_mode_output && 2007 (b->shader->info.stage == MESA_SHADER_VERTEX || 2008 b->shader->info.stage == MESA_SHADER_TESS_EVAL || 2009 b->shader->info.stage == MESA_SHADER_GEOMETRY)) { 2010 /* For vertex data outputs, we can end up with arrays of blocks for 2011 * transform feedback where each array element corresponds to a 2012 * different XFB output buffer. 2013 */ 2014 while (iface_type->base_type == vtn_base_type_array) 2015 iface_type = iface_type->array_element; 2016 } 2017 if (iface_type->base_type == vtn_base_type_struct && iface_type->block) 2018 var->var->interface_type = vtn_type_get_nir_type(b, iface_type, 2019 var->mode); 2020 2021 /* If it's a block, set it up as per-member so can be splitted later by 2022 * nir_split_per_member_structs. 2023 * 2024 * This is for a couple of reasons. For one, builtins may all come in a 2025 * block and we really want those split out into separate variables. 2026 * For another, interpolation qualifiers can be applied to members of 2027 * the top-level struct and we need to be able to preserve that 2028 * information. 2029 */ 2030 if (per_vertex_type->base_type == vtn_base_type_struct && 2031 per_vertex_type->block) { 2032 var->var->num_members = glsl_get_length(per_vertex_type->type); 2033 var->var->members = rzalloc_array(var->var, struct nir_variable_data, 2034 var->var->num_members); 2035 2036 for (unsigned i = 0; i < var->var->num_members; i++) { 2037 var->var->members[i].mode = nir_mode; 2038 var->var->members[i].patch = var->var->data.patch; 2039 var->var->members[i].location = -1; 2040 } 2041 } 2042 2043 /* For inputs and outputs, we need to grab locations and builtin 2044 * information from the per-vertex type. 2045 */ 2046 vtn_foreach_decoration(b, vtn_value(b, per_vertex_type->id, 2047 vtn_value_type_type), 2048 var_decoration_cb, var); 2049 2050 break; 2051 } 2052 2053 case vtn_variable_mode_phys_ssbo: 2054 case vtn_variable_mode_generic: 2055 unreachable("Should have been caught before"); 2056 } 2057 2058 /* Ignore incorrectly generated Undef initializers. */ 2059 if (b->wa_llvm_spirv_ignore_workgroup_initializer && 2060 initializer && 2061 storage_class == SpvStorageClassWorkgroup) 2062 initializer = NULL; 2063 2064 /* Only initialize variable when there is an initializer and it's not 2065 * undef. 2066 */ 2067 if (initializer && !initializer->is_undef_constant) { 2068 switch (storage_class) { 2069 case SpvStorageClassWorkgroup: 2070 /* VK_KHR_zero_initialize_workgroup_memory. */ 2071 vtn_fail_if(b->options->environment != NIR_SPIRV_VULKAN, 2072 "Only Vulkan supports variable initializer " 2073 "for Workgroup variable %u", 2074 vtn_id_for_value(b, val)); 2075 vtn_fail_if(initializer->value_type != vtn_value_type_constant || 2076 !initializer->is_null_constant, 2077 "Workgroup variable %u can only have OpConstantNull " 2078 "as initializer, but have %u instead", 2079 vtn_id_for_value(b, val), 2080 vtn_id_for_value(b, initializer)); 2081 b->shader->info.zero_initialize_shared_memory = true; 2082 break; 2083 2084 case SpvStorageClassUniformConstant: 2085 vtn_fail_if(b->options->environment != NIR_SPIRV_OPENGL && 2086 b->options->environment != NIR_SPIRV_OPENCL, 2087 "Only OpenGL and OpenCL support variable initializer " 2088 "for UniformConstant variable %u\n", 2089 vtn_id_for_value(b, val)); 2090 vtn_fail_if(initializer->value_type != vtn_value_type_constant, 2091 "UniformConstant variable %u can only have a constant " 2092 "initializer, but have %u instead", 2093 vtn_id_for_value(b, val), 2094 vtn_id_for_value(b, initializer)); 2095 break; 2096 2097 case SpvStorageClassOutput: 2098 case SpvStorageClassPrivate: 2099 vtn_assert(b->options->environment != NIR_SPIRV_OPENCL); 2100 /* These can have any initializer. */ 2101 break; 2102 2103 case SpvStorageClassFunction: 2104 /* These can have any initializer. */ 2105 break; 2106 2107 case SpvStorageClassCrossWorkgroup: 2108 vtn_assert(b->options->environment == NIR_SPIRV_OPENCL); 2109 vtn_fail("Initializer for CrossWorkgroup variable %u " 2110 "not yet supported in Mesa.", 2111 vtn_id_for_value(b, val)); 2112 break; 2113 2114 default: { 2115 const enum nir_spirv_execution_environment env = 2116 b->options->environment; 2117 const char *env_name = 2118 env == NIR_SPIRV_VULKAN ? "Vulkan" : 2119 env == NIR_SPIRV_OPENCL ? "OpenCL" : 2120 env == NIR_SPIRV_OPENGL ? "OpenGL" : 2121 NULL; 2122 vtn_assert(env_name); 2123 vtn_fail("In %s, any OpVariable with an Initializer operand " 2124 "must have %s%s%s, or Function as " 2125 "its Storage Class operand. Variable %u has an " 2126 "Initializer but its Storage Class is %s.", 2127 env_name, 2128 env == NIR_SPIRV_VULKAN ? "Private, Output, Workgroup" : "", 2129 env == NIR_SPIRV_OPENCL ? "CrossWorkgroup, UniformConstant" : "", 2130 env == NIR_SPIRV_OPENGL ? "Private, Output, UniformConstant" : "", 2131 vtn_id_for_value(b, val), 2132 spirv_storageclass_to_string(storage_class)); 2133 } 2134 } 2135 2136 switch (initializer->value_type) { 2137 case vtn_value_type_constant: 2138 var->var->constant_initializer = 2139 nir_constant_clone(initializer->constant, var->var); 2140 break; 2141 case vtn_value_type_pointer: 2142 var->var->pointer_initializer = initializer->pointer->var->var; 2143 break; 2144 default: 2145 vtn_fail("SPIR-V variable initializer %u must be constant or pointer", 2146 vtn_id_for_value(b, initializer)); 2147 } 2148 } 2149 2150 if (var->mode == vtn_variable_mode_uniform || 2151 var->mode == vtn_variable_mode_image || 2152 var->mode == vtn_variable_mode_ssbo) { 2153 /* SSBOs and images are assumed to not alias in the Simple, GLSL and Vulkan memory models */ 2154 var->var->data.access |= b->mem_model != SpvMemoryModelOpenCL ? ACCESS_RESTRICT : 0; 2155 } 2156 2157 vtn_foreach_decoration(b, val, var_decoration_cb, var); 2158 vtn_foreach_decoration(b, val, ptr_decoration_cb, val->pointer); 2159 2160 /* Propagate access flags from the OpVariable decorations. */ 2161 val->pointer->access |= var->access; 2162 2163 if ((var->mode == vtn_variable_mode_input || 2164 var->mode == vtn_variable_mode_output) && 2165 var->var->members) { 2166 assign_missing_member_locations(var); 2167 } 2168 2169 if (var->mode == vtn_variable_mode_uniform || 2170 var->mode == vtn_variable_mode_image || 2171 var->mode == vtn_variable_mode_ubo || 2172 var->mode == vtn_variable_mode_ssbo || 2173 var->mode == vtn_variable_mode_atomic_counter) { 2174 /* XXX: We still need the binding information in the nir_variable 2175 * for these. We should fix that. 2176 */ 2177 var->var->data.binding = var->binding; 2178 var->var->data.explicit_binding = var->explicit_binding; 2179 var->var->data.descriptor_set = var->descriptor_set; 2180 var->var->data.index = var->input_attachment_index; 2181 var->var->data.offset = var->offset; 2182 2183 if (glsl_type_is_image(glsl_without_array(var->var->type))) 2184 var->var->data.image.format = without_array->image_format; 2185 } 2186 2187 if (var->mode == vtn_variable_mode_function) { 2188 vtn_assert(var->var != NULL && var->var->members == NULL); 2189 nir_function_impl_add_variable(b->nb.impl, var->var); 2190 } else if (var->var) { 2191 nir_shader_add_variable(b->shader, var->var); 2192 } else { 2193 vtn_assert(vtn_pointer_is_external_block(b, val->pointer) || 2194 var->mode == vtn_variable_mode_accel_struct || 2195 var->mode == vtn_variable_mode_shader_record); 2196 } 2197} 2198 2199static void 2200vtn_assert_types_equal(struct vtn_builder *b, SpvOp opcode, 2201 struct vtn_type *dst_type, 2202 struct vtn_type *src_type) 2203{ 2204 if (dst_type->id == src_type->id) 2205 return; 2206 2207 if (vtn_types_compatible(b, dst_type, src_type)) { 2208 /* Early versions of GLSLang would re-emit types unnecessarily and you 2209 * would end up with OpLoad, OpStore, or OpCopyMemory opcodes which have 2210 * mismatched source and destination types. 2211 * 2212 * https://github.com/KhronosGroup/glslang/issues/304 2213 * https://github.com/KhronosGroup/glslang/issues/307 2214 * https://bugs.freedesktop.org/show_bug.cgi?id=104338 2215 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 2216 */ 2217 vtn_warn("Source and destination types of %s do not have the same " 2218 "ID (but are compatible): %u vs %u", 2219 spirv_op_to_string(opcode), dst_type->id, src_type->id); 2220 return; 2221 } 2222 2223 vtn_fail("Source and destination types of %s do not match: %s vs. %s", 2224 spirv_op_to_string(opcode), 2225 glsl_get_type_name(dst_type->type), 2226 glsl_get_type_name(src_type->type)); 2227} 2228 2229static nir_ssa_def * 2230nir_shrink_zero_pad_vec(nir_builder *b, nir_ssa_def *val, 2231 unsigned num_components) 2232{ 2233 if (val->num_components == num_components) 2234 return val; 2235 2236 nir_ssa_def *comps[NIR_MAX_VEC_COMPONENTS]; 2237 for (unsigned i = 0; i < num_components; i++) { 2238 if (i < val->num_components) 2239 comps[i] = nir_channel(b, val, i); 2240 else 2241 comps[i] = nir_imm_intN_t(b, 0, val->bit_size); 2242 } 2243 return nir_vec(b, comps, num_components); 2244} 2245 2246static nir_ssa_def * 2247nir_sloppy_bitcast(nir_builder *b, nir_ssa_def *val, 2248 const struct glsl_type *type) 2249{ 2250 const unsigned num_components = glsl_get_vector_elements(type); 2251 const unsigned bit_size = glsl_get_bit_size(type); 2252 2253 /* First, zero-pad to ensure that the value is big enough that when we 2254 * bit-cast it, we don't loose anything. 2255 */ 2256 if (val->bit_size < bit_size) { 2257 const unsigned src_num_components_needed = 2258 vtn_align_u32(val->num_components, bit_size / val->bit_size); 2259 val = nir_shrink_zero_pad_vec(b, val, src_num_components_needed); 2260 } 2261 2262 val = nir_bitcast_vector(b, val, bit_size); 2263 2264 return nir_shrink_zero_pad_vec(b, val, num_components); 2265} 2266 2267static bool 2268vtn_get_mem_operands(struct vtn_builder *b, const uint32_t *w, unsigned count, 2269 unsigned *idx, SpvMemoryAccessMask *access, unsigned *alignment, 2270 SpvScope *dest_scope, SpvScope *src_scope) 2271{ 2272 *access = 0; 2273 *alignment = 0; 2274 if (*idx >= count) 2275 return false; 2276 2277 *access = w[(*idx)++]; 2278 if (*access & SpvMemoryAccessAlignedMask) { 2279 vtn_assert(*idx < count); 2280 *alignment = w[(*idx)++]; 2281 } 2282 2283 if (*access & SpvMemoryAccessMakePointerAvailableMask) { 2284 vtn_assert(*idx < count); 2285 vtn_assert(dest_scope); 2286 *dest_scope = vtn_constant_uint(b, w[(*idx)++]); 2287 } 2288 2289 if (*access & SpvMemoryAccessMakePointerVisibleMask) { 2290 vtn_assert(*idx < count); 2291 vtn_assert(src_scope); 2292 *src_scope = vtn_constant_uint(b, w[(*idx)++]); 2293 } 2294 2295 return true; 2296} 2297 2298static enum gl_access_qualifier 2299spv_access_to_gl_access(SpvMemoryAccessMask access) 2300{ 2301 unsigned result = 0; 2302 2303 if (access & SpvMemoryAccessVolatileMask) 2304 result |= ACCESS_VOLATILE; 2305 if (access & SpvMemoryAccessNontemporalMask) 2306 result |= ACCESS_STREAM_CACHE_POLICY; 2307 2308 return result; 2309} 2310 2311 2312SpvMemorySemanticsMask 2313vtn_mode_to_memory_semantics(enum vtn_variable_mode mode) 2314{ 2315 switch (mode) { 2316 case vtn_variable_mode_ssbo: 2317 case vtn_variable_mode_phys_ssbo: 2318 return SpvMemorySemanticsUniformMemoryMask; 2319 case vtn_variable_mode_workgroup: 2320 return SpvMemorySemanticsWorkgroupMemoryMask; 2321 case vtn_variable_mode_cross_workgroup: 2322 return SpvMemorySemanticsCrossWorkgroupMemoryMask; 2323 case vtn_variable_mode_atomic_counter: 2324 return SpvMemorySemanticsAtomicCounterMemoryMask; 2325 case vtn_variable_mode_image: 2326 return SpvMemorySemanticsImageMemoryMask; 2327 case vtn_variable_mode_output: 2328 return SpvMemorySemanticsOutputMemoryMask; 2329 default: 2330 return SpvMemorySemanticsMaskNone; 2331 } 2332} 2333 2334static void 2335vtn_emit_make_visible_barrier(struct vtn_builder *b, SpvMemoryAccessMask access, 2336 SpvScope scope, enum vtn_variable_mode mode) 2337{ 2338 if (!(access & SpvMemoryAccessMakePointerVisibleMask)) 2339 return; 2340 2341 vtn_emit_memory_barrier(b, scope, SpvMemorySemanticsMakeVisibleMask | 2342 SpvMemorySemanticsAcquireMask | 2343 vtn_mode_to_memory_semantics(mode)); 2344} 2345 2346static void 2347vtn_emit_make_available_barrier(struct vtn_builder *b, SpvMemoryAccessMask access, 2348 SpvScope scope, enum vtn_variable_mode mode) 2349{ 2350 if (!(access & SpvMemoryAccessMakePointerAvailableMask)) 2351 return; 2352 2353 vtn_emit_memory_barrier(b, scope, SpvMemorySemanticsMakeAvailableMask | 2354 SpvMemorySemanticsReleaseMask | 2355 vtn_mode_to_memory_semantics(mode)); 2356} 2357 2358static void 2359ptr_nonuniform_workaround_cb(struct vtn_builder *b, struct vtn_value *val, 2360 int member, const struct vtn_decoration *dec, void *void_ptr) 2361{ 2362 enum gl_access_qualifier *access = void_ptr; 2363 2364 switch (dec->decoration) { 2365 case SpvDecorationNonUniformEXT: 2366 *access |= ACCESS_NON_UNIFORM; 2367 break; 2368 2369 default: 2370 break; 2371 } 2372} 2373 2374void 2375vtn_handle_variables(struct vtn_builder *b, SpvOp opcode, 2376 const uint32_t *w, unsigned count) 2377{ 2378 switch (opcode) { 2379 case SpvOpUndef: { 2380 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef); 2381 val->type = vtn_get_type(b, w[1]); 2382 val->is_undef_constant = true; 2383 break; 2384 } 2385 2386 case SpvOpVariable: { 2387 struct vtn_type *ptr_type = vtn_get_type(b, w[1]); 2388 2389 SpvStorageClass storage_class = w[3]; 2390 2391 const bool is_global = storage_class != SpvStorageClassFunction; 2392 const bool is_io = storage_class == SpvStorageClassInput || 2393 storage_class == SpvStorageClassOutput; 2394 2395 /* Skip global variables that are not used by the entrypoint. Before 2396 * SPIR-V 1.4 the interface is only used for I/O variables, so extra 2397 * variables will still need to be removed later. 2398 */ 2399 if (!b->options->create_library && 2400 (is_io || (b->version >= 0x10400 && is_global))) { 2401 if (!bsearch(&w[2], b->interface_ids, b->interface_ids_count, 4, cmp_uint32_t)) 2402 break; 2403 } 2404 2405 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer); 2406 struct vtn_value *initializer = count > 4 ? vtn_untyped_value(b, w[4]) : NULL; 2407 2408 vtn_create_variable(b, val, ptr_type, storage_class, initializer); 2409 2410 break; 2411 } 2412 2413 case SpvOpConstantSampler: { 2414 /* Synthesize a pointer-to-sampler type, create a variable of that type, 2415 * and give the variable a constant initializer with the sampler params */ 2416 struct vtn_type *sampler_type = vtn_value(b, w[1], vtn_value_type_type)->type; 2417 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer); 2418 2419 struct vtn_type *ptr_type = rzalloc(b, struct vtn_type); 2420 ptr_type = rzalloc(b, struct vtn_type); 2421 ptr_type->base_type = vtn_base_type_pointer; 2422 ptr_type->deref = sampler_type; 2423 ptr_type->storage_class = SpvStorageClassUniform; 2424 2425 ptr_type->type = nir_address_format_to_glsl_type( 2426 vtn_mode_to_address_format(b, vtn_variable_mode_function)); 2427 2428 vtn_create_variable(b, val, ptr_type, ptr_type->storage_class, NULL); 2429 2430 nir_variable *nir_var = val->pointer->var->var; 2431 nir_var->data.sampler.is_inline_sampler = true; 2432 nir_var->data.sampler.addressing_mode = w[3]; 2433 nir_var->data.sampler.normalized_coordinates = w[4]; 2434 nir_var->data.sampler.filter_mode = w[5]; 2435 2436 break; 2437 } 2438 2439 case SpvOpAccessChain: 2440 case SpvOpPtrAccessChain: 2441 case SpvOpInBoundsAccessChain: 2442 case SpvOpInBoundsPtrAccessChain: { 2443 struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4); 2444 enum gl_access_qualifier access = 0; 2445 chain->ptr_as_array = (opcode == SpvOpPtrAccessChain || opcode == SpvOpInBoundsPtrAccessChain); 2446 2447 unsigned idx = 0; 2448 for (int i = 4; i < count; i++) { 2449 struct vtn_value *link_val = vtn_untyped_value(b, w[i]); 2450 if (link_val->value_type == vtn_value_type_constant) { 2451 chain->link[idx].mode = vtn_access_mode_literal; 2452 chain->link[idx].id = vtn_constant_int(b, w[i]); 2453 } else { 2454 chain->link[idx].mode = vtn_access_mode_id; 2455 chain->link[idx].id = w[i]; 2456 } 2457 2458 /* Workaround for https://gitlab.freedesktop.org/mesa/mesa/-/issues/3406 */ 2459 vtn_foreach_decoration(b, link_val, ptr_nonuniform_workaround_cb, &access); 2460 2461 idx++; 2462 } 2463 2464 struct vtn_type *ptr_type = vtn_get_type(b, w[1]); 2465 2466 struct vtn_pointer *base = vtn_pointer(b, w[3]); 2467 2468 chain->in_bounds = (opcode == SpvOpInBoundsAccessChain || opcode == SpvOpInBoundsPtrAccessChain); 2469 2470 /* Workaround for https://gitlab.freedesktop.org/mesa/mesa/-/issues/3406 */ 2471 access |= base->access & ACCESS_NON_UNIFORM; 2472 2473 struct vtn_pointer *ptr = vtn_pointer_dereference(b, base, chain); 2474 ptr->ptr_type = ptr_type; 2475 ptr->access |= access; 2476 vtn_push_pointer(b, w[2], ptr); 2477 break; 2478 } 2479 2480 case SpvOpCopyMemory: { 2481 struct vtn_value *dest_val = vtn_pointer_value(b, w[1]); 2482 struct vtn_value *src_val = vtn_pointer_value(b, w[2]); 2483 struct vtn_pointer *dest = vtn_value_to_pointer(b, dest_val); 2484 struct vtn_pointer *src = vtn_value_to_pointer(b, src_val); 2485 2486 vtn_assert_types_equal(b, opcode, dest_val->type->deref, 2487 src_val->type->deref); 2488 2489 unsigned idx = 3, dest_alignment, src_alignment; 2490 SpvMemoryAccessMask dest_access, src_access; 2491 SpvScope dest_scope, src_scope; 2492 vtn_get_mem_operands(b, w, count, &idx, &dest_access, &dest_alignment, 2493 &dest_scope, &src_scope); 2494 if (!vtn_get_mem_operands(b, w, count, &idx, &src_access, &src_alignment, 2495 NULL, &src_scope)) { 2496 src_alignment = dest_alignment; 2497 src_access = dest_access; 2498 } 2499 src = vtn_align_pointer(b, src, src_alignment); 2500 dest = vtn_align_pointer(b, dest, dest_alignment); 2501 2502 vtn_emit_make_visible_barrier(b, src_access, src_scope, src->mode); 2503 2504 vtn_variable_copy(b, dest, src, 2505 spv_access_to_gl_access(dest_access), 2506 spv_access_to_gl_access(src_access)); 2507 2508 vtn_emit_make_available_barrier(b, dest_access, dest_scope, dest->mode); 2509 break; 2510 } 2511 2512 case SpvOpCopyMemorySized: { 2513 struct vtn_value *dest_val = vtn_pointer_value(b, w[1]); 2514 struct vtn_value *src_val = vtn_pointer_value(b, w[2]); 2515 nir_ssa_def *size = vtn_get_nir_ssa(b, w[3]); 2516 struct vtn_pointer *dest = vtn_value_to_pointer(b, dest_val); 2517 struct vtn_pointer *src = vtn_value_to_pointer(b, src_val); 2518 2519 unsigned idx = 4, dest_alignment, src_alignment; 2520 SpvMemoryAccessMask dest_access, src_access; 2521 SpvScope dest_scope, src_scope; 2522 vtn_get_mem_operands(b, w, count, &idx, &dest_access, &dest_alignment, 2523 &dest_scope, &src_scope); 2524 if (!vtn_get_mem_operands(b, w, count, &idx, &src_access, &src_alignment, 2525 NULL, &src_scope)) { 2526 src_alignment = dest_alignment; 2527 src_access = dest_access; 2528 } 2529 src = vtn_align_pointer(b, src, src_alignment); 2530 dest = vtn_align_pointer(b, dest, dest_alignment); 2531 2532 vtn_emit_make_visible_barrier(b, src_access, src_scope, src->mode); 2533 2534 nir_memcpy_deref_with_access(&b->nb, 2535 vtn_pointer_to_deref(b, dest), 2536 vtn_pointer_to_deref(b, src), 2537 size, 2538 spv_access_to_gl_access(dest_access), 2539 spv_access_to_gl_access(src_access)); 2540 2541 vtn_emit_make_available_barrier(b, dest_access, dest_scope, dest->mode); 2542 break; 2543 } 2544 2545 case SpvOpLoad: { 2546 struct vtn_type *res_type = vtn_get_type(b, w[1]); 2547 struct vtn_value *src_val = vtn_value(b, w[3], vtn_value_type_pointer); 2548 struct vtn_pointer *src = vtn_value_to_pointer(b, src_val); 2549 2550 vtn_assert_types_equal(b, opcode, res_type, src_val->type->deref); 2551 2552 unsigned idx = 4, alignment; 2553 SpvMemoryAccessMask access; 2554 SpvScope scope; 2555 vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, NULL, &scope); 2556 src = vtn_align_pointer(b, src, alignment); 2557 2558 vtn_emit_make_visible_barrier(b, access, scope, src->mode); 2559 2560 vtn_push_ssa_value(b, w[2], vtn_variable_load(b, src, spv_access_to_gl_access(access))); 2561 break; 2562 } 2563 2564 case SpvOpStore: { 2565 struct vtn_value *dest_val = vtn_pointer_value(b, w[1]); 2566 struct vtn_pointer *dest = vtn_value_to_pointer(b, dest_val); 2567 struct vtn_value *src_val = vtn_untyped_value(b, w[2]); 2568 2569 /* OpStore requires us to actually have a storage type */ 2570 vtn_fail_if(dest->type->type == NULL, 2571 "Invalid destination type for OpStore"); 2572 2573 if (glsl_get_base_type(dest->type->type) == GLSL_TYPE_BOOL && 2574 glsl_get_base_type(src_val->type->type) == GLSL_TYPE_UINT) { 2575 /* Early versions of GLSLang would use uint types for UBOs/SSBOs but 2576 * would then store them to a local variable as bool. Work around 2577 * the issue by doing an implicit conversion. 2578 * 2579 * https://github.com/KhronosGroup/glslang/issues/170 2580 * https://bugs.freedesktop.org/show_bug.cgi?id=104424 2581 */ 2582 vtn_warn("OpStore of value of type OpTypeInt to a pointer to type " 2583 "OpTypeBool. Doing an implicit conversion to work around " 2584 "the problem."); 2585 struct vtn_ssa_value *bool_ssa = 2586 vtn_create_ssa_value(b, dest->type->type); 2587 bool_ssa->def = nir_i2b(&b->nb, vtn_ssa_value(b, w[2])->def); 2588 vtn_variable_store(b, bool_ssa, dest, 0); 2589 break; 2590 } 2591 2592 vtn_assert_types_equal(b, opcode, dest_val->type->deref, src_val->type); 2593 2594 unsigned idx = 3, alignment; 2595 SpvMemoryAccessMask access; 2596 SpvScope scope; 2597 vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, &scope, NULL); 2598 dest = vtn_align_pointer(b, dest, alignment); 2599 2600 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]); 2601 vtn_variable_store(b, src, dest, spv_access_to_gl_access(access)); 2602 2603 vtn_emit_make_available_barrier(b, access, scope, dest->mode); 2604 break; 2605 } 2606 2607 case SpvOpArrayLength: { 2608 struct vtn_pointer *ptr = vtn_pointer(b, w[3]); 2609 const uint32_t field = w[4]; 2610 2611 vtn_fail_if(ptr->type->base_type != vtn_base_type_struct, 2612 "OpArrayLength must take a pointer to a structure type"); 2613 vtn_fail_if(field != ptr->type->length - 1 || 2614 ptr->type->members[field]->base_type != vtn_base_type_array, 2615 "OpArrayLength must reference the last memeber of the " 2616 "structure and that must be an array"); 2617 2618 if (b->options->use_deref_buffer_array_length) { 2619 struct vtn_access_chain chain = { 2620 .length = 1, 2621 .link = { 2622 { .mode = vtn_access_mode_literal, .id = field }, 2623 } 2624 }; 2625 struct vtn_pointer *array = vtn_pointer_dereference(b, ptr, &chain); 2626 2627 nir_ssa_def *array_length = 2628 nir_build_deref_buffer_array_length(&b->nb, 32, 2629 vtn_pointer_to_ssa(b, array), 2630 .access=ptr->access | ptr->type->access); 2631 2632 vtn_push_nir_ssa(b, w[2], array_length); 2633 } else { 2634 const uint32_t offset = ptr->type->offsets[field]; 2635 const uint32_t stride = ptr->type->members[field]->stride; 2636 2637 if (!ptr->block_index) { 2638 struct vtn_access_chain chain = { 2639 .length = 0, 2640 }; 2641 ptr = vtn_pointer_dereference(b, ptr, &chain); 2642 vtn_assert(ptr->block_index); 2643 } 2644 2645 nir_ssa_def *buf_size = nir_get_ssbo_size(&b->nb, ptr->block_index, 2646 .access=ptr->access | ptr->type->access); 2647 2648 /* array_length = max(buffer_size - offset, 0) / stride */ 2649 nir_ssa_def *array_length = 2650 nir_udiv_imm(&b->nb, 2651 nir_usub_sat(&b->nb, 2652 buf_size, 2653 nir_imm_int(&b->nb, offset)), 2654 stride); 2655 2656 vtn_push_nir_ssa(b, w[2], array_length); 2657 } 2658 break; 2659 } 2660 2661 case SpvOpConvertPtrToU: { 2662 struct vtn_type *u_type = vtn_get_type(b, w[1]); 2663 struct vtn_type *ptr_type = vtn_get_value_type(b, w[3]); 2664 2665 vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer || 2666 ptr_type->type == NULL, 2667 "OpConvertPtrToU can only be used on physical pointers"); 2668 2669 vtn_fail_if(u_type->base_type != vtn_base_type_vector && 2670 u_type->base_type != vtn_base_type_scalar, 2671 "OpConvertPtrToU can only be used to cast to a vector or " 2672 "scalar type"); 2673 2674 /* The pointer will be converted to an SSA value automatically */ 2675 nir_ssa_def *ptr = vtn_get_nir_ssa(b, w[3]); 2676 nir_ssa_def *u = nir_sloppy_bitcast(&b->nb, ptr, u_type->type); 2677 vtn_push_nir_ssa(b, w[2], u); 2678 break; 2679 } 2680 2681 case SpvOpConvertUToPtr: { 2682 struct vtn_type *ptr_type = vtn_get_type(b, w[1]); 2683 struct vtn_type *u_type = vtn_get_value_type(b, w[3]); 2684 2685 vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer || 2686 ptr_type->type == NULL, 2687 "OpConvertUToPtr can only be used on physical pointers"); 2688 2689 vtn_fail_if(u_type->base_type != vtn_base_type_vector && 2690 u_type->base_type != vtn_base_type_scalar, 2691 "OpConvertUToPtr can only be used to cast from a vector or " 2692 "scalar type"); 2693 2694 nir_ssa_def *u = vtn_get_nir_ssa(b, w[3]); 2695 nir_ssa_def *ptr = nir_sloppy_bitcast(&b->nb, u, ptr_type->type); 2696 vtn_push_pointer(b, w[2], vtn_pointer_from_ssa(b, ptr, ptr_type)); 2697 break; 2698 } 2699 2700 case SpvOpGenericCastToPtrExplicit: { 2701 struct vtn_type *dst_type = vtn_get_type(b, w[1]); 2702 struct vtn_type *src_type = vtn_get_value_type(b, w[3]); 2703 SpvStorageClass storage_class = w[4]; 2704 2705 vtn_fail_if(dst_type->base_type != vtn_base_type_pointer || 2706 dst_type->storage_class != storage_class, 2707 "Result type of an SpvOpGenericCastToPtrExplicit must be " 2708 "an OpTypePointer. Its Storage Class must match the " 2709 "storage class specified in the instruction"); 2710 2711 vtn_fail_if(src_type->base_type != vtn_base_type_pointer || 2712 src_type->deref->id != dst_type->deref->id, 2713 "Source pointer of an SpvOpGenericCastToPtrExplicit must " 2714 "have a type of OpTypePointer whose Type is the same as " 2715 "the Type of Result Type"); 2716 2717 vtn_fail_if(src_type->storage_class != SpvStorageClassGeneric, 2718 "Source pointer of an SpvOpGenericCastToPtrExplicit must " 2719 "point to the Generic Storage Class."); 2720 2721 vtn_fail_if(storage_class != SpvStorageClassWorkgroup && 2722 storage_class != SpvStorageClassCrossWorkgroup && 2723 storage_class != SpvStorageClassFunction, 2724 "Storage must be one of the following literal values from " 2725 "Storage Class: Workgroup, CrossWorkgroup, or Function."); 2726 2727 nir_deref_instr *src_deref = vtn_nir_deref(b, w[3]); 2728 2729 nir_variable_mode nir_mode; 2730 enum vtn_variable_mode mode = 2731 vtn_storage_class_to_mode(b, storage_class, dst_type->deref, &nir_mode); 2732 nir_address_format addr_format = vtn_mode_to_address_format(b, mode); 2733 2734 nir_ssa_def *null_value = 2735 nir_build_imm(&b->nb, nir_address_format_num_components(addr_format), 2736 nir_address_format_bit_size(addr_format), 2737 nir_address_format_null_value(addr_format)); 2738 2739 nir_ssa_def *valid = nir_build_deref_mode_is(&b->nb, 1, &src_deref->dest.ssa, nir_mode); 2740 vtn_push_nir_ssa(b, w[2], nir_bcsel(&b->nb, valid, 2741 &src_deref->dest.ssa, 2742 null_value)); 2743 break; 2744 } 2745 2746 case SpvOpGenericPtrMemSemantics: { 2747 struct vtn_type *dst_type = vtn_get_type(b, w[1]); 2748 struct vtn_type *src_type = vtn_get_value_type(b, w[3]); 2749 2750 vtn_fail_if(dst_type->base_type != vtn_base_type_scalar || 2751 dst_type->type != glsl_uint_type(), 2752 "Result type of an SpvOpGenericPtrMemSemantics must be " 2753 "an OpTypeInt with 32-bit Width and 0 Signedness."); 2754 2755 vtn_fail_if(src_type->base_type != vtn_base_type_pointer || 2756 src_type->storage_class != SpvStorageClassGeneric, 2757 "Source pointer of an SpvOpGenericPtrMemSemantics must " 2758 "point to the Generic Storage Class"); 2759 2760 nir_deref_instr *src_deref = vtn_nir_deref(b, w[3]); 2761 2762 nir_ssa_def *global_bit = 2763 nir_bcsel(&b->nb, nir_build_deref_mode_is(&b->nb, 1, &src_deref->dest.ssa, 2764 nir_var_mem_global), 2765 nir_imm_int(&b->nb, SpvMemorySemanticsCrossWorkgroupMemoryMask), 2766 nir_imm_int(&b->nb, 0)); 2767 2768 nir_ssa_def *shared_bit = 2769 nir_bcsel(&b->nb, nir_build_deref_mode_is(&b->nb, 1, &src_deref->dest.ssa, 2770 nir_var_mem_shared), 2771 nir_imm_int(&b->nb, SpvMemorySemanticsWorkgroupMemoryMask), 2772 nir_imm_int(&b->nb, 0)); 2773 2774 vtn_push_nir_ssa(b, w[2], nir_iand(&b->nb, global_bit, shared_bit)); 2775 break; 2776 } 2777 2778 case SpvOpSubgroupBlockReadINTEL: { 2779 struct vtn_type *res_type = vtn_get_type(b, w[1]); 2780 nir_deref_instr *src = vtn_nir_deref(b, w[3]); 2781 2782 nir_intrinsic_instr *load = 2783 nir_intrinsic_instr_create(b->nb.shader, 2784 nir_intrinsic_load_deref_block_intel); 2785 load->src[0] = nir_src_for_ssa(&src->dest.ssa); 2786 nir_ssa_dest_init_for_type(&load->instr, &load->dest, 2787 res_type->type, NULL); 2788 load->num_components = load->dest.ssa.num_components; 2789 nir_builder_instr_insert(&b->nb, &load->instr); 2790 2791 vtn_push_nir_ssa(b, w[2], &load->dest.ssa); 2792 break; 2793 } 2794 2795 case SpvOpSubgroupBlockWriteINTEL: { 2796 nir_deref_instr *dest = vtn_nir_deref(b, w[1]); 2797 nir_ssa_def *data = vtn_ssa_value(b, w[2])->def; 2798 2799 nir_intrinsic_instr *store = 2800 nir_intrinsic_instr_create(b->nb.shader, 2801 nir_intrinsic_store_deref_block_intel); 2802 store->src[0] = nir_src_for_ssa(&dest->dest.ssa); 2803 store->src[1] = nir_src_for_ssa(data); 2804 store->num_components = data->num_components; 2805 nir_builder_instr_insert(&b->nb, &store->instr); 2806 break; 2807 } 2808 2809 case SpvOpConvertUToAccelerationStructureKHR: { 2810 struct vtn_type *as_type = vtn_get_type(b, w[1]); 2811 struct vtn_type *u_type = vtn_get_value_type(b, w[3]); 2812 vtn_fail_if(!((u_type->base_type == vtn_base_type_vector && 2813 u_type->type == glsl_vector_type(GLSL_TYPE_UINT, 2)) || 2814 (u_type->base_type == vtn_base_type_scalar && 2815 u_type->type == glsl_uint64_t_type())), 2816 "OpConvertUToAccelerationStructure may only be used to " 2817 "cast from a 64-bit scalar integer or a 2-component vector " 2818 "of 32-bit integers"); 2819 vtn_fail_if(as_type->base_type != vtn_base_type_accel_struct, 2820 "The result type of an OpConvertUToAccelerationStructure " 2821 "must be OpTypeAccelerationStructure"); 2822 2823 nir_ssa_def *u = vtn_get_nir_ssa(b, w[3]); 2824 vtn_push_nir_ssa(b, w[2], nir_sloppy_bitcast(&b->nb, u, as_type->type)); 2825 break; 2826 } 2827 2828 default: 2829 vtn_fail_with_opcode("Unhandled opcode", opcode); 2830 } 2831} 2832