1/*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 * SPDX-License-Identifier: MIT
5 */
6
7/**
8 * @file
9 *
10 * We use the bindless descriptor model, which maps fairly closely to how
11 * Vulkan descriptor sets work. The two exceptions are input attachments and
12 * dynamic descriptors, which have to be patched when recording command
13 * buffers. We reserve an extra descriptor set for these. This descriptor set
14 * contains all the input attachments in the pipeline, in order, and then all
15 * the dynamic descriptors. The dynamic descriptors are stored in the CPU-side
16 * datastructure for each tu_descriptor_set, and then combined into one big
17 * descriptor set at CmdBindDescriptors time/draw time.
18 */
19
20#include "tu_descriptor_set.h"
21
22#include <fcntl.h>
23
24#include "util/mesa-sha1.h"
25#include "vk_descriptors.h"
26#include "vk_util.h"
27
28#include "tu_device.h"
29#include "tu_image.h"
30
31static inline uint8_t *
32pool_base(struct tu_descriptor_pool *pool)
33{
34   return pool->host_bo ?: pool->bo->map;
35}
36
37static uint32_t
38descriptor_size(struct tu_device *dev, VkDescriptorType type)
39{
40   switch (type) {
41   case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
42      if (unlikely(dev->instance->debug_flags & TU_DEBUG_DYNAMIC))
43         return A6XX_TEX_CONST_DWORDS * 4;
44
45      /* Input attachment doesn't use descriptor sets at all */
46      return 0;
47   case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
48      /* We make offsets and sizes all 16 dwords, to match how the hardware
49       * interprets indices passed to sample/load/store instructions in
50       * multiples of 16 dwords.  This means that "normal" descriptors are all
51       * of size 16, with padding for smaller descriptors like uniform storage
52       * descriptors which are less than 16 dwords. However combined images
53       * and samplers are actually two descriptors, so they have size 2.
54       */
55      return A6XX_TEX_CONST_DWORDS * 4 * 2;
56   case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
57   case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
58      /* When we support 16-bit storage, we need an extra descriptor setup as
59       * a 32-bit array for isam to work.
60       */
61      if (dev->physical_device->info->a6xx.storage_16bit) {
62         return A6XX_TEX_CONST_DWORDS * 4 * 2;
63      } else {
64         return A6XX_TEX_CONST_DWORDS * 4;
65      }
66   default:
67      return A6XX_TEX_CONST_DWORDS * 4;
68   }
69}
70
71static bool
72is_dynamic(VkDescriptorType type)
73{
74   return type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC ||
75          type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC;
76}
77
78static uint32_t
79mutable_descriptor_size(struct tu_device *dev, const VkMutableDescriptorTypeListVALVE *list)
80{
81   uint32_t max_size = 0;
82
83   for (uint32_t i = 0; i < list->descriptorTypeCount; i++) {
84      uint32_t size = descriptor_size(dev, list->pDescriptorTypes[i]);
85      max_size = MAX2(max_size, size);
86   }
87
88   return max_size;
89}
90
91VKAPI_ATTR VkResult VKAPI_CALL
92tu_CreateDescriptorSetLayout(
93   VkDevice _device,
94   const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
95   const VkAllocationCallbacks *pAllocator,
96   VkDescriptorSetLayout *pSetLayout)
97{
98   TU_FROM_HANDLE(tu_device, device, _device);
99   struct tu_descriptor_set_layout *set_layout;
100
101   assert(pCreateInfo->sType ==
102          VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO);
103   const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags =
104      vk_find_struct_const(
105         pCreateInfo->pNext,
106         DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO);
107   const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
108      vk_find_struct_const(
109         pCreateInfo->pNext,
110         MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
111
112   uint32_t num_bindings = 0;
113   uint32_t immutable_sampler_count = 0;
114   uint32_t ycbcr_sampler_count = 0;
115   for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
116      num_bindings = MAX2(num_bindings, pCreateInfo->pBindings[j].binding + 1);
117      if ((pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
118           pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
119           pCreateInfo->pBindings[j].pImmutableSamplers) {
120         immutable_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
121
122         bool has_ycbcr_sampler = false;
123         for (unsigned i = 0; i < pCreateInfo->pBindings[j].descriptorCount; ++i) {
124            if (tu_sampler_from_handle(pCreateInfo->pBindings[j].pImmutableSamplers[i])->ycbcr_sampler)
125               has_ycbcr_sampler = true;
126         }
127
128         if (has_ycbcr_sampler)
129            ycbcr_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
130      }
131   }
132
133   uint32_t samplers_offset =
134         offsetof(struct tu_descriptor_set_layout, binding[num_bindings]);
135
136   /* note: only need to store TEX_SAMP_DWORDS for immutable samples,
137    * but using struct tu_sampler makes things simpler */
138   uint32_t size = samplers_offset +
139      immutable_sampler_count * sizeof(struct tu_sampler) +
140      ycbcr_sampler_count * sizeof(struct tu_sampler_ycbcr_conversion);
141
142   set_layout = vk_object_zalloc(&device->vk, NULL, size,
143                                 VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT);
144   if (!set_layout)
145      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
146
147   set_layout->flags = pCreateInfo->flags;
148
149   /* We just allocate all the immutable samplers at the end of the struct */
150   struct tu_sampler *samplers = (void*) &set_layout->binding[num_bindings];
151   struct tu_sampler_ycbcr_conversion *ycbcr_samplers =
152      (void*) &samplers[immutable_sampler_count];
153
154   VkDescriptorSetLayoutBinding *bindings = NULL;
155   VkResult result = vk_create_sorted_bindings(
156      pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
157   if (result != VK_SUCCESS) {
158      vk_object_free(&device->vk, pAllocator, set_layout);
159      return vk_error(device, result);
160   }
161
162   set_layout->ref_cnt = 1;
163   set_layout->binding_count = num_bindings;
164   set_layout->shader_stages = 0;
165   set_layout->has_immutable_samplers = false;
166   set_layout->size = 0;
167
168   uint32_t dynamic_offset_size = 0;
169
170   for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
171      const VkDescriptorSetLayoutBinding *binding = bindings + j;
172      uint32_t b = binding->binding;
173
174      set_layout->binding[b].type = binding->descriptorType;
175      set_layout->binding[b].array_size = binding->descriptorCount;
176      set_layout->binding[b].offset = set_layout->size;
177      set_layout->binding[b].dynamic_offset_offset = dynamic_offset_size;
178      set_layout->binding[b].shader_stages = binding->stageFlags;
179
180      if (binding->descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_VALVE) {
181         /* For mutable descriptor types we must allocate a size that fits the
182          * largest descriptor type that the binding can mutate to.
183          */
184         set_layout->binding[b].size =
185            mutable_descriptor_size(device, &mutable_info->pMutableDescriptorTypeLists[j]);
186      } else {
187         set_layout->binding[b].size = descriptor_size(device, binding->descriptorType);
188      }
189
190      if (variable_flags && binding->binding < variable_flags->bindingCount &&
191          (variable_flags->pBindingFlags[binding->binding] &
192           VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) {
193         assert(!binding->pImmutableSamplers); /* Terribly ill defined  how
194                                                  many samplers are valid */
195         assert(binding->binding == num_bindings - 1);
196
197         set_layout->has_variable_descriptors = true;
198      }
199
200      if ((binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
201           binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
202          binding->pImmutableSamplers) {
203         set_layout->binding[b].immutable_samplers_offset = samplers_offset;
204         set_layout->has_immutable_samplers = true;
205
206         for (uint32_t i = 0; i < binding->descriptorCount; i++)
207            samplers[i] = *tu_sampler_from_handle(binding->pImmutableSamplers[i]);
208
209         samplers += binding->descriptorCount;
210         samplers_offset += sizeof(struct tu_sampler) * binding->descriptorCount;
211
212         bool has_ycbcr_sampler = false;
213         for (unsigned i = 0; i < pCreateInfo->pBindings[j].descriptorCount; ++i) {
214            if (tu_sampler_from_handle(binding->pImmutableSamplers[i])->ycbcr_sampler)
215               has_ycbcr_sampler = true;
216         }
217
218         if (has_ycbcr_sampler) {
219            set_layout->binding[b].ycbcr_samplers_offset =
220               (const char*)ycbcr_samplers - (const char*)set_layout;
221            for (uint32_t i = 0; i < binding->descriptorCount; i++) {
222               struct tu_sampler *sampler = tu_sampler_from_handle(binding->pImmutableSamplers[i]);
223               if (sampler->ycbcr_sampler)
224                  ycbcr_samplers[i] = *sampler->ycbcr_sampler;
225               else
226                  ycbcr_samplers[i].ycbcr_model = VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY;
227            }
228            ycbcr_samplers += binding->descriptorCount;
229         } else {
230            set_layout->binding[b].ycbcr_samplers_offset = 0;
231         }
232      }
233
234      if (is_dynamic(binding->descriptorType)) {
235         dynamic_offset_size +=
236            binding->descriptorCount * set_layout->binding[b].size;
237      } else {
238         set_layout->size +=
239            binding->descriptorCount * set_layout->binding[b].size;
240      }
241
242      set_layout->shader_stages |= binding->stageFlags;
243   }
244
245   free(bindings);
246
247   set_layout->dynamic_offset_size = dynamic_offset_size;
248
249   *pSetLayout = tu_descriptor_set_layout_to_handle(set_layout);
250
251   return VK_SUCCESS;
252}
253
254VKAPI_ATTR void VKAPI_CALL
255tu_DestroyDescriptorSetLayout(VkDevice _device,
256                              VkDescriptorSetLayout _set_layout,
257                              const VkAllocationCallbacks *pAllocator)
258{
259   TU_FROM_HANDLE(tu_device, device, _device);
260   TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout, _set_layout);
261
262   if (!set_layout)
263      return;
264
265   tu_descriptor_set_layout_unref(device, set_layout);
266}
267
268void
269tu_descriptor_set_layout_destroy(struct tu_device *device,
270                                 struct tu_descriptor_set_layout *layout)
271{
272   assert(layout->ref_cnt == 0);
273   vk_object_free(&device->vk, NULL, layout);
274}
275
276VKAPI_ATTR void VKAPI_CALL
277tu_GetDescriptorSetLayoutSupport(
278   VkDevice _device,
279   const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
280   VkDescriptorSetLayoutSupport *pSupport)
281{
282   TU_FROM_HANDLE(tu_device, device, _device);
283
284   VkDescriptorSetLayoutBinding *bindings = NULL;
285   VkResult result = vk_create_sorted_bindings(
286      pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
287   if (result != VK_SUCCESS) {
288      pSupport->supported = false;
289      return;
290   }
291
292   const VkDescriptorSetLayoutBindingFlagsCreateInfo *variable_flags =
293      vk_find_struct_const(
294         pCreateInfo->pNext,
295         DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO);
296   VkDescriptorSetVariableDescriptorCountLayoutSupport *variable_count =
297      vk_find_struct(
298         (void *) pCreateInfo->pNext,
299         DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_LAYOUT_SUPPORT);
300   const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
301      vk_find_struct_const(
302         pCreateInfo->pNext,
303         MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
304
305   if (variable_count) {
306      variable_count->maxVariableDescriptorCount = 0;
307   }
308
309   bool supported = true;
310   uint64_t size = 0;
311   for (uint32_t i = 0; i < pCreateInfo->bindingCount; i++) {
312      const VkDescriptorSetLayoutBinding *binding = bindings + i;
313
314      uint64_t descriptor_sz;
315
316      if (is_dynamic(binding->descriptorType)) {
317         descriptor_sz = 0;
318      } else if (binding->descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_VALVE) {
319         const VkMutableDescriptorTypeListVALVE *list =
320            &mutable_info->pMutableDescriptorTypeLists[i];
321
322         for (uint32_t j = 0; j < list->descriptorTypeCount; j++) {
323            /* Don't support the input attachement and combined image sampler type
324             * for mutable descriptors */
325            if (list->pDescriptorTypes[j] == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT ||
326                list->pDescriptorTypes[j] == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) {
327               supported = false;
328               goto out;
329            }
330         }
331
332         descriptor_sz =
333            mutable_descriptor_size(device, &mutable_info->pMutableDescriptorTypeLists[i]);
334      } else {
335         descriptor_sz = descriptor_size(device, binding->descriptorType);
336      }
337      uint64_t descriptor_alignment = 8;
338
339      if (size && !ALIGN_POT(size, descriptor_alignment)) {
340         supported = false;
341      }
342      size = ALIGN_POT(size, descriptor_alignment);
343
344      uint64_t max_count = UINT64_MAX;
345      if (descriptor_sz)
346         max_count = (UINT64_MAX - size) / descriptor_sz;
347
348      if (max_count < binding->descriptorCount) {
349         supported = false;
350      }
351
352      if (variable_flags && binding->binding < variable_flags->bindingCount &&
353          variable_count &&
354          (variable_flags->pBindingFlags[binding->binding] &
355           VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT)) {
356         variable_count->maxVariableDescriptorCount =
357            MIN2(UINT32_MAX, max_count);
358      }
359      size += binding->descriptorCount * descriptor_sz;
360   }
361
362out:
363   free(bindings);
364
365   pSupport->supported = supported;
366}
367
368/* Note: we must hash any values used in tu_lower_io(). */
369
370#define SHA1_UPDATE_VALUE(ctx, x) _mesa_sha1_update(ctx, &(x), sizeof(x));
371
372static void
373sha1_update_ycbcr_sampler(struct mesa_sha1 *ctx,
374                          const struct tu_sampler_ycbcr_conversion *sampler)
375{
376   SHA1_UPDATE_VALUE(ctx, sampler->ycbcr_model);
377   SHA1_UPDATE_VALUE(ctx, sampler->ycbcr_range);
378   SHA1_UPDATE_VALUE(ctx, sampler->format);
379}
380
381static void
382sha1_update_descriptor_set_binding_layout(struct mesa_sha1 *ctx,
383   const struct tu_descriptor_set_binding_layout *layout,
384   const struct tu_descriptor_set_layout *set_layout)
385{
386   SHA1_UPDATE_VALUE(ctx, layout->type);
387   SHA1_UPDATE_VALUE(ctx, layout->offset);
388   SHA1_UPDATE_VALUE(ctx, layout->size);
389   SHA1_UPDATE_VALUE(ctx, layout->array_size);
390   SHA1_UPDATE_VALUE(ctx, layout->dynamic_offset_offset);
391   SHA1_UPDATE_VALUE(ctx, layout->immutable_samplers_offset);
392
393   const struct tu_sampler_ycbcr_conversion *ycbcr_samplers =
394      tu_immutable_ycbcr_samplers(set_layout, layout);
395
396   if (ycbcr_samplers) {
397      for (unsigned i = 0; i < layout->array_size; i++)
398         sha1_update_ycbcr_sampler(ctx, ycbcr_samplers + i);
399   }
400}
401
402
403static void
404sha1_update_descriptor_set_layout(struct mesa_sha1 *ctx,
405                                  const struct tu_descriptor_set_layout *layout)
406{
407   for (uint16_t i = 0; i < layout->binding_count; i++)
408      sha1_update_descriptor_set_binding_layout(ctx, &layout->binding[i],
409                                                layout);
410}
411
412/*
413 * Pipeline layouts.  These have nothing to do with the pipeline.  They are
414 * just multiple descriptor set layouts pasted together.
415 */
416
417VKAPI_ATTR VkResult VKAPI_CALL
418tu_CreatePipelineLayout(VkDevice _device,
419                        const VkPipelineLayoutCreateInfo *pCreateInfo,
420                        const VkAllocationCallbacks *pAllocator,
421                        VkPipelineLayout *pPipelineLayout)
422{
423   TU_FROM_HANDLE(tu_device, device, _device);
424   struct tu_pipeline_layout *layout;
425
426   assert(pCreateInfo->sType ==
427          VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO);
428
429   layout = vk_object_alloc(&device->vk, pAllocator, sizeof(*layout),
430                            VK_OBJECT_TYPE_PIPELINE_LAYOUT);
431   if (layout == NULL)
432      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
433
434   layout->num_sets = pCreateInfo->setLayoutCount;
435   layout->dynamic_offset_size = 0;
436
437   unsigned dynamic_offset_size = 0;
438
439   for (uint32_t set = 0; set < pCreateInfo->setLayoutCount; set++) {
440      TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout,
441                     pCreateInfo->pSetLayouts[set]);
442
443      assert(set < MAX_SETS);
444      layout->set[set].layout = set_layout;
445      layout->set[set].dynamic_offset_start = dynamic_offset_size;
446      tu_descriptor_set_layout_ref(set_layout);
447
448      dynamic_offset_size += set_layout->dynamic_offset_size;
449   }
450
451   layout->dynamic_offset_size = dynamic_offset_size;
452   layout->push_constant_size = 0;
453
454   for (unsigned i = 0; i < pCreateInfo->pushConstantRangeCount; ++i) {
455      const VkPushConstantRange *range = pCreateInfo->pPushConstantRanges + i;
456      layout->push_constant_size =
457         MAX2(layout->push_constant_size, range->offset + range->size);
458   }
459
460   layout->push_constant_size = align(layout->push_constant_size, 16);
461
462   struct mesa_sha1 ctx;
463   _mesa_sha1_init(&ctx);
464   for (unsigned s = 0; s < layout->num_sets; s++) {
465      sha1_update_descriptor_set_layout(&ctx, layout->set[s].layout);
466      _mesa_sha1_update(&ctx, &layout->set[s].dynamic_offset_start,
467                        sizeof(layout->set[s].dynamic_offset_start));
468   }
469   _mesa_sha1_update(&ctx, &layout->num_sets, sizeof(layout->num_sets));
470   _mesa_sha1_update(&ctx, &layout->push_constant_size,
471                     sizeof(layout->push_constant_size));
472   _mesa_sha1_final(&ctx, layout->sha1);
473
474   *pPipelineLayout = tu_pipeline_layout_to_handle(layout);
475
476   return VK_SUCCESS;
477}
478
479VKAPI_ATTR void VKAPI_CALL
480tu_DestroyPipelineLayout(VkDevice _device,
481                         VkPipelineLayout _pipelineLayout,
482                         const VkAllocationCallbacks *pAllocator)
483{
484   TU_FROM_HANDLE(tu_device, device, _device);
485   TU_FROM_HANDLE(tu_pipeline_layout, pipeline_layout, _pipelineLayout);
486
487   if (!pipeline_layout)
488      return;
489
490   for (uint32_t i = 0; i < pipeline_layout->num_sets; i++)
491      tu_descriptor_set_layout_unref(device, pipeline_layout->set[i].layout);
492
493   vk_object_free(&device->vk, pAllocator, pipeline_layout);
494}
495
496#define EMPTY 1
497
498static VkResult
499tu_descriptor_set_create(struct tu_device *device,
500            struct tu_descriptor_pool *pool,
501            struct tu_descriptor_set_layout *layout,
502            const uint32_t *variable_count,
503            struct tu_descriptor_set **out_set)
504{
505   struct tu_descriptor_set *set;
506   unsigned dynamic_offset = sizeof(struct tu_descriptor_set);
507   unsigned mem_size = dynamic_offset + layout->dynamic_offset_size;
508
509   if (pool->host_memory_base) {
510      if (pool->host_memory_end - pool->host_memory_ptr < mem_size)
511         return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
512
513      set = (struct tu_descriptor_set*)pool->host_memory_ptr;
514      pool->host_memory_ptr += mem_size;
515   } else {
516      set = vk_alloc2(&device->vk.alloc, NULL, mem_size, 8,
517                      VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
518
519      if (!set)
520         return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
521   }
522
523   memset(set, 0, mem_size);
524   vk_object_base_init(&device->vk, &set->base, VK_OBJECT_TYPE_DESCRIPTOR_SET);
525
526   if (layout->dynamic_offset_size) {
527      set->dynamic_descriptors = (uint32_t *)((uint8_t*)set + dynamic_offset);
528   }
529
530   set->layout = layout;
531   set->pool = pool;
532   uint32_t layout_size = layout->size;
533   if (variable_count) {
534      assert(layout->has_variable_descriptors);
535      uint32_t stride = layout->binding[layout->binding_count - 1].size;
536      layout_size = layout->binding[layout->binding_count - 1].offset +
537                    *variable_count * stride;
538   }
539
540   if (layout_size) {
541      set->size = layout_size;
542
543      if (!pool->host_memory_base && pool->entry_count == pool->max_entry_count) {
544         vk_object_free(&device->vk, NULL, set);
545         return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
546      }
547
548      /* try to allocate linearly first, so that we don't spend
549       * time looking for gaps if the app only allocates &
550       * resets via the pool. */
551      if (pool->current_offset + layout_size <= pool->size) {
552         set->mapped_ptr = (uint32_t*)(pool_base(pool) + pool->current_offset);
553         set->va = pool->host_bo ? 0 : pool->bo->iova + pool->current_offset;
554
555         if (!pool->host_memory_base) {
556            pool->entries[pool->entry_count].offset = pool->current_offset;
557            pool->entries[pool->entry_count].size = layout_size;
558            pool->entries[pool->entry_count].set = set;
559            pool->entry_count++;
560         }
561         pool->current_offset += layout_size;
562      } else if (!pool->host_memory_base) {
563         uint64_t offset = 0;
564         int index;
565
566         for (index = 0; index < pool->entry_count; ++index) {
567            if (pool->entries[index].offset - offset >= layout_size)
568               break;
569            offset = pool->entries[index].offset + pool->entries[index].size;
570         }
571
572         if (pool->size - offset < layout_size) {
573            vk_object_free(&device->vk, NULL, set);
574            return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
575         }
576
577         set->mapped_ptr = (uint32_t*)(pool_base(pool) + offset);
578         set->va = pool->host_bo ? 0 : pool->bo->iova + offset;
579
580         memmove(&pool->entries[index + 1], &pool->entries[index],
581            sizeof(pool->entries[0]) * (pool->entry_count - index));
582         pool->entries[index].offset = offset;
583         pool->entries[index].size = layout_size;
584         pool->entries[index].set = set;
585         pool->entry_count++;
586      } else
587         return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
588   }
589
590   if (layout->has_immutable_samplers) {
591      for (unsigned i = 0; i < layout->binding_count; ++i) {
592         if (!layout->binding[i].immutable_samplers_offset)
593            continue;
594
595         unsigned offset = layout->binding[i].offset / 4;
596         if (layout->binding[i].type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
597            offset += A6XX_TEX_CONST_DWORDS;
598
599         const struct tu_sampler *samplers =
600            (const struct tu_sampler *)((const char *)layout +
601                               layout->binding[i].immutable_samplers_offset);
602         for (unsigned j = 0; j < layout->binding[i].array_size; ++j) {
603            memcpy(set->mapped_ptr + offset, samplers[j].descriptor,
604                   sizeof(samplers[j].descriptor));
605            offset += layout->binding[i].size / 4;
606         }
607      }
608   }
609
610   tu_descriptor_set_layout_ref(layout);
611   list_addtail(&set->pool_link, &pool->desc_sets);
612
613   *out_set = set;
614   return VK_SUCCESS;
615}
616
617static void
618tu_descriptor_set_destroy(struct tu_device *device,
619             struct tu_descriptor_pool *pool,
620             struct tu_descriptor_set *set,
621             bool free_bo)
622{
623   assert(!pool->host_memory_base);
624
625   if (free_bo && set->size && !pool->host_memory_base) {
626      uint32_t offset = (uint8_t*)set->mapped_ptr - pool_base(pool);
627
628      for (int i = 0; i < pool->entry_count; ++i) {
629         if (pool->entries[i].offset == offset) {
630            memmove(&pool->entries[i], &pool->entries[i+1],
631               sizeof(pool->entries[i]) * (pool->entry_count - i - 1));
632            --pool->entry_count;
633            break;
634         }
635      }
636   }
637
638   vk_object_free(&device->vk, NULL, set);
639}
640
641VKAPI_ATTR VkResult VKAPI_CALL
642tu_CreateDescriptorPool(VkDevice _device,
643                        const VkDescriptorPoolCreateInfo *pCreateInfo,
644                        const VkAllocationCallbacks *pAllocator,
645                        VkDescriptorPool *pDescriptorPool)
646{
647   TU_FROM_HANDLE(tu_device, device, _device);
648   struct tu_descriptor_pool *pool;
649   uint64_t size = sizeof(struct tu_descriptor_pool);
650   uint64_t bo_size = 0, bo_count = 0, dynamic_size = 0;
651   VkResult ret;
652
653   const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
654      vk_find_struct_const( pCreateInfo->pNext,
655         MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
656
657   for (unsigned i = 0; i < pCreateInfo->poolSizeCount; ++i) {
658      if (pCreateInfo->pPoolSizes[i].type != VK_DESCRIPTOR_TYPE_SAMPLER)
659         bo_count += pCreateInfo->pPoolSizes[i].descriptorCount;
660
661      switch(pCreateInfo->pPoolSizes[i].type) {
662      case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
663      case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
664         dynamic_size += descriptor_size(device, pCreateInfo->pPoolSizes[i].type) *
665            pCreateInfo->pPoolSizes[i].descriptorCount;
666         break;
667      case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE:
668         if (mutable_info && i < mutable_info->mutableDescriptorTypeListCount &&
669             mutable_info->pMutableDescriptorTypeLists[i].descriptorTypeCount > 0) {
670            bo_size +=
671               mutable_descriptor_size(device, &mutable_info->pMutableDescriptorTypeLists[i]) *
672                  pCreateInfo->pPoolSizes[i].descriptorCount;
673         } else {
674            /* Allocate the maximum size possible. */
675            bo_size += 2 * A6XX_TEX_CONST_DWORDS * 4 *
676                  pCreateInfo->pPoolSizes[i].descriptorCount;
677         }
678         continue;
679      default:
680         break;
681      }
682
683      bo_size += descriptor_size(device, pCreateInfo->pPoolSizes[i].type) *
684                           pCreateInfo->pPoolSizes[i].descriptorCount;
685   }
686
687   if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
688      uint64_t host_size = pCreateInfo->maxSets * sizeof(struct tu_descriptor_set);
689      host_size += sizeof(struct tu_bo*) * bo_count;
690      host_size += dynamic_size;
691      size += host_size;
692   } else {
693      size += sizeof(struct tu_descriptor_pool_entry) * pCreateInfo->maxSets;
694   }
695
696   pool = vk_object_zalloc(&device->vk, pAllocator, size,
697                          VK_OBJECT_TYPE_DESCRIPTOR_POOL);
698   if (!pool)
699      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
700
701   if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
702      pool->host_memory_base = (uint8_t*)pool + sizeof(struct tu_descriptor_pool);
703      pool->host_memory_ptr = pool->host_memory_base;
704      pool->host_memory_end = (uint8_t*)pool + size;
705   }
706
707   if (bo_size) {
708      if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_HOST_ONLY_BIT_VALVE)) {
709         ret = tu_bo_init_new(device, &pool->bo, bo_size, TU_BO_ALLOC_ALLOW_DUMP);
710         if (ret)
711            goto fail_alloc;
712
713         ret = tu_bo_map(device, pool->bo);
714         if (ret)
715            goto fail_map;
716      } else {
717         pool->host_bo = vk_alloc2(&device->vk.alloc, pAllocator, bo_size, 8,
718                                   VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
719         if (!pool->host_bo) {
720            ret = VK_ERROR_OUT_OF_HOST_MEMORY;
721            goto fail_alloc;
722         }
723      }
724   }
725   pool->size = bo_size;
726   pool->max_entry_count = pCreateInfo->maxSets;
727
728   list_inithead(&pool->desc_sets);
729
730   *pDescriptorPool = tu_descriptor_pool_to_handle(pool);
731   return VK_SUCCESS;
732
733fail_map:
734   tu_bo_finish(device, pool->bo);
735fail_alloc:
736   vk_object_free(&device->vk, pAllocator, pool);
737   return ret;
738}
739
740VKAPI_ATTR void VKAPI_CALL
741tu_DestroyDescriptorPool(VkDevice _device,
742                         VkDescriptorPool _pool,
743                         const VkAllocationCallbacks *pAllocator)
744{
745   TU_FROM_HANDLE(tu_device, device, _device);
746   TU_FROM_HANDLE(tu_descriptor_pool, pool, _pool);
747
748   if (!pool)
749      return;
750
751   list_for_each_entry_safe(struct tu_descriptor_set, set,
752                            &pool->desc_sets, pool_link) {
753      tu_descriptor_set_layout_unref(device, set->layout);
754   }
755
756   if (!pool->host_memory_base) {
757      for(int i = 0; i < pool->entry_count; ++i) {
758         tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
759      }
760   }
761
762   if (pool->size) {
763      if (pool->host_bo)
764         vk_free2(&device->vk.alloc, pAllocator, pool->host_bo);
765      else
766         tu_bo_finish(device, pool->bo);
767   }
768
769   vk_object_free(&device->vk, pAllocator, pool);
770}
771
772VKAPI_ATTR VkResult VKAPI_CALL
773tu_ResetDescriptorPool(VkDevice _device,
774                       VkDescriptorPool descriptorPool,
775                       VkDescriptorPoolResetFlags flags)
776{
777   TU_FROM_HANDLE(tu_device, device, _device);
778   TU_FROM_HANDLE(tu_descriptor_pool, pool, descriptorPool);
779
780   list_for_each_entry_safe(struct tu_descriptor_set, set,
781                            &pool->desc_sets, pool_link) {
782      tu_descriptor_set_layout_unref(device, set->layout);
783   }
784   list_inithead(&pool->desc_sets);
785
786   if (!pool->host_memory_base) {
787      for(int i = 0; i < pool->entry_count; ++i) {
788         tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
789      }
790      pool->entry_count = 0;
791   }
792
793   pool->current_offset = 0;
794   pool->host_memory_ptr = pool->host_memory_base;
795
796   return VK_SUCCESS;
797}
798
799VKAPI_ATTR VkResult VKAPI_CALL
800tu_AllocateDescriptorSets(VkDevice _device,
801                          const VkDescriptorSetAllocateInfo *pAllocateInfo,
802                          VkDescriptorSet *pDescriptorSets)
803{
804   TU_FROM_HANDLE(tu_device, device, _device);
805   TU_FROM_HANDLE(tu_descriptor_pool, pool, pAllocateInfo->descriptorPool);
806
807   VkResult result = VK_SUCCESS;
808   uint32_t i;
809   struct tu_descriptor_set *set = NULL;
810
811   const VkDescriptorSetVariableDescriptorCountAllocateInfo *variable_counts =
812      vk_find_struct_const(pAllocateInfo->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_ALLOCATE_INFO);
813   const uint32_t zero = 0;
814
815   /* allocate a set of buffers for each shader to contain descriptors */
816   for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
817      TU_FROM_HANDLE(tu_descriptor_set_layout, layout,
818             pAllocateInfo->pSetLayouts[i]);
819
820      const uint32_t *variable_count = NULL;
821      if (variable_counts) {
822         if (i < variable_counts->descriptorSetCount)
823            variable_count = variable_counts->pDescriptorCounts + i;
824         else
825            variable_count = &zero;
826      }
827
828      assert(!(layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
829
830      result = tu_descriptor_set_create(device, pool, layout, variable_count, &set);
831      if (result != VK_SUCCESS)
832         break;
833
834      pDescriptorSets[i] = tu_descriptor_set_to_handle(set);
835   }
836
837   if (result != VK_SUCCESS) {
838      tu_FreeDescriptorSets(_device, pAllocateInfo->descriptorPool,
839               i, pDescriptorSets);
840      for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
841         pDescriptorSets[i] = VK_NULL_HANDLE;
842      }
843   }
844   return result;
845}
846
847VKAPI_ATTR VkResult VKAPI_CALL
848tu_FreeDescriptorSets(VkDevice _device,
849                      VkDescriptorPool descriptorPool,
850                      uint32_t count,
851                      const VkDescriptorSet *pDescriptorSets)
852{
853   TU_FROM_HANDLE(tu_device, device, _device);
854   TU_FROM_HANDLE(tu_descriptor_pool, pool, descriptorPool);
855
856   for (uint32_t i = 0; i < count; i++) {
857      TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
858
859      if (set) {
860         tu_descriptor_set_layout_unref(device, set->layout);
861         list_del(&set->pool_link);
862      }
863
864      if (set && !pool->host_memory_base)
865         tu_descriptor_set_destroy(device, pool, set, true);
866   }
867   return VK_SUCCESS;
868}
869
870static void
871write_texel_buffer_descriptor(uint32_t *dst, const VkBufferView buffer_view)
872{
873   if (buffer_view == VK_NULL_HANDLE) {
874      memset(dst, 0, A6XX_TEX_CONST_DWORDS * sizeof(uint32_t));
875   } else {
876      TU_FROM_HANDLE(tu_buffer_view, view, buffer_view);
877
878      memcpy(dst, view->descriptor, sizeof(view->descriptor));
879   }
880}
881
882static uint32_t get_range(struct tu_buffer *buf, VkDeviceSize offset,
883                          VkDeviceSize range)
884{
885   if (range == VK_WHOLE_SIZE) {
886      return buf->size - offset;
887   } else {
888      return range;
889   }
890}
891
892static void
893write_buffer_descriptor(const struct tu_device *device,
894                        uint32_t *dst,
895                        const VkDescriptorBufferInfo *buffer_info)
896{
897   bool storage_16bit = device->physical_device->info->a6xx.storage_16bit;
898   /* newer a6xx allows using 16-bit descriptor for both 16-bit and 32-bit
899    * access, but we need to keep a 32-bit descriptor for readonly access via
900    * isam.
901    */
902   unsigned descriptors = storage_16bit ? 2 : 1;
903   if (buffer_info->buffer == VK_NULL_HANDLE) {
904      memset(dst, 0, descriptors * A6XX_TEX_CONST_DWORDS * sizeof(uint32_t));
905      return;
906   }
907
908   TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
909
910   assert((buffer_info->offset & 63) == 0); /* minStorageBufferOffsetAlignment */
911   uint64_t va = buffer->iova + buffer_info->offset;
912   uint32_t range = get_range(buffer, buffer_info->offset, buffer_info->range);
913
914   for (unsigned i = 0; i < descriptors; i++) {
915      if (storage_16bit && i == 0) {
916         dst[0] = A6XX_TEX_CONST_0_TILE_MODE(TILE6_LINEAR) | A6XX_TEX_CONST_0_FMT(FMT6_16_UINT);
917         dst[1] = DIV_ROUND_UP(range, 2);
918      } else {
919         dst[0] = A6XX_TEX_CONST_0_TILE_MODE(TILE6_LINEAR) | A6XX_TEX_CONST_0_FMT(FMT6_32_UINT);
920         dst[1] = DIV_ROUND_UP(range, 4);
921      }
922      dst[2] =
923         A6XX_TEX_CONST_2_BUFFER | A6XX_TEX_CONST_2_TYPE(A6XX_TEX_BUFFER);
924      dst[3] = 0;
925      dst[4] = A6XX_TEX_CONST_4_BASE_LO(va);
926      dst[5] = A6XX_TEX_CONST_5_BASE_HI(va >> 32);
927      for (int j = 6; j < A6XX_TEX_CONST_DWORDS; j++)
928         dst[j] = 0;
929      dst += A6XX_TEX_CONST_DWORDS;
930   }
931}
932
933static void
934write_ubo_descriptor(uint32_t *dst, const VkDescriptorBufferInfo *buffer_info)
935{
936   if (buffer_info->buffer == VK_NULL_HANDLE) {
937      dst[0] = dst[1] = 0;
938      return;
939   }
940
941   TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
942
943   uint32_t range = get_range(buffer, buffer_info->offset, buffer_info->range);
944   /* The HW range is in vec4 units */
945   range = ALIGN_POT(range, 16) / 16;
946   uint64_t va = buffer->iova + buffer_info->offset;
947
948   dst[0] = A6XX_UBO_0_BASE_LO(va);
949   dst[1] = A6XX_UBO_1_BASE_HI(va >> 32) | A6XX_UBO_1_SIZE(range);
950}
951
952static void
953write_image_descriptor(uint32_t *dst,
954                       VkDescriptorType descriptor_type,
955                       const VkDescriptorImageInfo *image_info)
956{
957   if (image_info->imageView == VK_NULL_HANDLE) {
958      memset(dst, 0, A6XX_TEX_CONST_DWORDS * sizeof(uint32_t));
959      return;
960   }
961
962   TU_FROM_HANDLE(tu_image_view, iview, image_info->imageView);
963
964   if (descriptor_type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) {
965      memcpy(dst, iview->view.storage_descriptor, sizeof(iview->view.storage_descriptor));
966   } else {
967      memcpy(dst, iview->view.descriptor, sizeof(iview->view.descriptor));
968   }
969}
970
971static void
972write_combined_image_sampler_descriptor(uint32_t *dst,
973                                        VkDescriptorType descriptor_type,
974                                        const VkDescriptorImageInfo *image_info,
975                                        bool has_sampler)
976{
977   write_image_descriptor(dst, descriptor_type, image_info);
978   /* copy over sampler state */
979   if (has_sampler) {
980      TU_FROM_HANDLE(tu_sampler, sampler, image_info->sampler);
981      memcpy(dst + A6XX_TEX_CONST_DWORDS, sampler->descriptor, sizeof(sampler->descriptor));
982   }
983}
984
985static void
986write_sampler_descriptor(uint32_t *dst, const VkDescriptorImageInfo *image_info)
987{
988   TU_FROM_HANDLE(tu_sampler, sampler, image_info->sampler);
989
990   memcpy(dst, sampler->descriptor, sizeof(sampler->descriptor));
991}
992
993/* note: this is used with immutable samplers in push descriptors */
994static void
995write_sampler_push(uint32_t *dst, const struct tu_sampler *sampler)
996{
997   memcpy(dst, sampler->descriptor, sizeof(sampler->descriptor));
998}
999
1000void
1001tu_update_descriptor_sets(const struct tu_device *device,
1002                          VkDescriptorSet dstSetOverride,
1003                          uint32_t descriptorWriteCount,
1004                          const VkWriteDescriptorSet *pDescriptorWrites,
1005                          uint32_t descriptorCopyCount,
1006                          const VkCopyDescriptorSet *pDescriptorCopies)
1007{
1008   uint32_t i, j;
1009   for (i = 0; i < descriptorWriteCount; i++) {
1010      const VkWriteDescriptorSet *writeset = &pDescriptorWrites[i];
1011      TU_FROM_HANDLE(tu_descriptor_set, set, dstSetOverride ?: writeset->dstSet);
1012      const struct tu_descriptor_set_binding_layout *binding_layout =
1013         set->layout->binding + writeset->dstBinding;
1014      uint32_t *ptr = set->mapped_ptr;
1015      if (writeset->descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
1016          writeset->descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
1017         ptr = set->dynamic_descriptors;
1018         ptr += binding_layout->dynamic_offset_offset / 4;
1019      } else {
1020         ptr = set->mapped_ptr;
1021         ptr += binding_layout->offset / 4;
1022      }
1023
1024      /* for immutable samplers with push descriptors: */
1025      const bool copy_immutable_samplers =
1026         dstSetOverride && binding_layout->immutable_samplers_offset;
1027      const struct tu_sampler *samplers =
1028         tu_immutable_samplers(set->layout, binding_layout);
1029
1030      ptr += (binding_layout->size / 4) * writeset->dstArrayElement;
1031      for (j = 0; j < writeset->descriptorCount; ++j) {
1032         switch(writeset->descriptorType) {
1033         case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1034         case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1035            write_ubo_descriptor(ptr, writeset->pBufferInfo + j);
1036            break;
1037         case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
1038         case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1039            write_buffer_descriptor(device, ptr, writeset->pBufferInfo + j);
1040            break;
1041         case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1042         case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1043            write_texel_buffer_descriptor(ptr, writeset->pTexelBufferView[j]);
1044            break;
1045         case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1046         case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
1047            write_image_descriptor(ptr, writeset->descriptorType, writeset->pImageInfo + j);
1048            break;
1049         case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1050            write_combined_image_sampler_descriptor(ptr,
1051                                                    writeset->descriptorType,
1052                                                    writeset->pImageInfo + j,
1053                                                    !binding_layout->immutable_samplers_offset);
1054
1055            if (copy_immutable_samplers)
1056               write_sampler_push(ptr + A6XX_TEX_CONST_DWORDS, &samplers[writeset->dstArrayElement + j]);
1057            break;
1058         case VK_DESCRIPTOR_TYPE_SAMPLER:
1059            if (!binding_layout->immutable_samplers_offset)
1060               write_sampler_descriptor(ptr, writeset->pImageInfo + j);
1061            else if (copy_immutable_samplers)
1062               write_sampler_push(ptr, &samplers[writeset->dstArrayElement + j]);
1063            break;
1064         case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1065            /* nothing in descriptor set - framebuffer state is used instead */
1066            if (unlikely(device->instance->debug_flags & TU_DEBUG_DYNAMIC))
1067               write_image_descriptor(ptr, writeset->descriptorType, writeset->pImageInfo + j);
1068            break;
1069         default:
1070            unreachable("unimplemented descriptor type");
1071            break;
1072         }
1073         ptr += binding_layout->size / 4;
1074      }
1075   }
1076
1077   for (i = 0; i < descriptorCopyCount; i++) {
1078      const VkCopyDescriptorSet *copyset = &pDescriptorCopies[i];
1079      TU_FROM_HANDLE(tu_descriptor_set, src_set,
1080                       copyset->srcSet);
1081      TU_FROM_HANDLE(tu_descriptor_set, dst_set,
1082                       copyset->dstSet);
1083      const struct tu_descriptor_set_binding_layout *src_binding_layout =
1084         src_set->layout->binding + copyset->srcBinding;
1085      const struct tu_descriptor_set_binding_layout *dst_binding_layout =
1086         dst_set->layout->binding + copyset->dstBinding;
1087      uint32_t *src_ptr = src_set->mapped_ptr;
1088      uint32_t *dst_ptr = dst_set->mapped_ptr;
1089      if (src_binding_layout->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
1090          src_binding_layout->type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
1091         src_ptr = src_set->dynamic_descriptors;
1092         dst_ptr = dst_set->dynamic_descriptors;
1093         src_ptr += src_binding_layout->dynamic_offset_offset / 4;
1094         dst_ptr += dst_binding_layout->dynamic_offset_offset / 4;
1095      } else {
1096         src_ptr = src_set->mapped_ptr;
1097         dst_ptr = dst_set->mapped_ptr;
1098         src_ptr += src_binding_layout->offset / 4;
1099         dst_ptr += dst_binding_layout->offset / 4;
1100      }
1101
1102      src_ptr += src_binding_layout->size * copyset->srcArrayElement / 4;
1103      dst_ptr += dst_binding_layout->size * copyset->dstArrayElement / 4;
1104
1105      /* In case of copies between mutable descriptor types
1106       * and non-mutable descriptor types.
1107       */
1108      uint32_t copy_size = MIN2(src_binding_layout->size, dst_binding_layout->size);
1109
1110      for (j = 0; j < copyset->descriptorCount; ++j) {
1111         memcpy(dst_ptr, src_ptr, copy_size);
1112
1113         src_ptr += src_binding_layout->size / 4;
1114         dst_ptr += dst_binding_layout->size / 4;
1115      }
1116   }
1117}
1118
1119VKAPI_ATTR void VKAPI_CALL
1120tu_UpdateDescriptorSets(VkDevice _device,
1121                        uint32_t descriptorWriteCount,
1122                        const VkWriteDescriptorSet *pDescriptorWrites,
1123                        uint32_t descriptorCopyCount,
1124                        const VkCopyDescriptorSet *pDescriptorCopies)
1125{
1126   TU_FROM_HANDLE(tu_device, device, _device);
1127   tu_update_descriptor_sets(device, VK_NULL_HANDLE,
1128                             descriptorWriteCount, pDescriptorWrites,
1129                             descriptorCopyCount, pDescriptorCopies);
1130}
1131
1132VKAPI_ATTR VkResult VKAPI_CALL
1133tu_CreateDescriptorUpdateTemplate(
1134   VkDevice _device,
1135   const VkDescriptorUpdateTemplateCreateInfo *pCreateInfo,
1136   const VkAllocationCallbacks *pAllocator,
1137   VkDescriptorUpdateTemplate *pDescriptorUpdateTemplate)
1138{
1139   TU_FROM_HANDLE(tu_device, device, _device);
1140   struct tu_descriptor_set_layout *set_layout = NULL;
1141   const uint32_t entry_count = pCreateInfo->descriptorUpdateEntryCount;
1142   const size_t size =
1143      sizeof(struct tu_descriptor_update_template) +
1144      sizeof(struct tu_descriptor_update_template_entry) * entry_count;
1145   struct tu_descriptor_update_template *templ;
1146
1147   templ = vk_object_alloc(&device->vk, pAllocator, size,
1148                           VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE);
1149   if (!templ)
1150      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1151
1152   templ->entry_count = entry_count;
1153
1154   if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR) {
1155      TU_FROM_HANDLE(tu_pipeline_layout, pipeline_layout, pCreateInfo->pipelineLayout);
1156
1157      /* descriptorSetLayout should be ignored for push descriptors
1158       * and instead it refers to pipelineLayout and set.
1159       */
1160      assert(pCreateInfo->set < MAX_SETS);
1161      set_layout = pipeline_layout->set[pCreateInfo->set].layout;
1162
1163      templ->bind_point = pCreateInfo->pipelineBindPoint;
1164   } else {
1165      TU_FROM_HANDLE(tu_descriptor_set_layout, _set_layout,
1166                     pCreateInfo->descriptorSetLayout);
1167      set_layout = _set_layout;
1168   }
1169
1170   for (uint32_t i = 0; i < entry_count; i++) {
1171      const VkDescriptorUpdateTemplateEntry *entry = &pCreateInfo->pDescriptorUpdateEntries[i];
1172
1173      const struct tu_descriptor_set_binding_layout *binding_layout =
1174         set_layout->binding + entry->dstBinding;
1175      uint32_t dst_offset, dst_stride;
1176      const struct tu_sampler *immutable_samplers = NULL;
1177
1178      /* dst_offset is an offset into dynamic_descriptors when the descriptor
1179       * is dynamic, and an offset into mapped_ptr otherwise.
1180       */
1181      switch (entry->descriptorType) {
1182      case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1183      case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
1184         dst_offset = binding_layout->dynamic_offset_offset / 4;
1185         break;
1186      case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1187      case VK_DESCRIPTOR_TYPE_SAMPLER:
1188         if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR &&
1189             binding_layout->immutable_samplers_offset) {
1190            immutable_samplers =
1191               tu_immutable_samplers(set_layout, binding_layout) + entry->dstArrayElement;
1192         }
1193         FALLTHROUGH;
1194      default:
1195         dst_offset = binding_layout->offset / 4;
1196      }
1197
1198      dst_offset += (binding_layout->size * entry->dstArrayElement) / 4;
1199      dst_stride = binding_layout->size / 4;
1200
1201      templ->entry[i] = (struct tu_descriptor_update_template_entry) {
1202         .descriptor_type = entry->descriptorType,
1203         .descriptor_count = entry->descriptorCount,
1204         .src_offset = entry->offset,
1205         .src_stride = entry->stride,
1206         .dst_offset = dst_offset,
1207         .dst_stride = dst_stride,
1208         .has_sampler = !binding_layout->immutable_samplers_offset,
1209         .immutable_samplers = immutable_samplers,
1210      };
1211   }
1212
1213   *pDescriptorUpdateTemplate =
1214      tu_descriptor_update_template_to_handle(templ);
1215
1216   return VK_SUCCESS;
1217}
1218
1219VKAPI_ATTR void VKAPI_CALL
1220tu_DestroyDescriptorUpdateTemplate(
1221   VkDevice _device,
1222   VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1223   const VkAllocationCallbacks *pAllocator)
1224{
1225   TU_FROM_HANDLE(tu_device, device, _device);
1226   TU_FROM_HANDLE(tu_descriptor_update_template, templ,
1227                  descriptorUpdateTemplate);
1228
1229   if (!templ)
1230      return;
1231
1232   vk_object_free(&device->vk, pAllocator, templ);
1233}
1234
1235void
1236tu_update_descriptor_set_with_template(
1237   const struct tu_device *device,
1238   struct tu_descriptor_set *set,
1239   VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1240   const void *pData)
1241{
1242   TU_FROM_HANDLE(tu_descriptor_update_template, templ,
1243                  descriptorUpdateTemplate);
1244
1245   for (uint32_t i = 0; i < templ->entry_count; i++) {
1246      uint32_t *ptr = set->mapped_ptr;
1247      const void *src = ((const char *) pData) + templ->entry[i].src_offset;
1248      const struct tu_sampler *samplers = templ->entry[i].immutable_samplers;
1249
1250      ptr += templ->entry[i].dst_offset;
1251      unsigned dst_offset = templ->entry[i].dst_offset;
1252      for (unsigned j = 0; j < templ->entry[i].descriptor_count; ++j) {
1253         switch(templ->entry[i].descriptor_type) {
1254         case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: {
1255            assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1256            write_ubo_descriptor(set->dynamic_descriptors + dst_offset, src);
1257            break;
1258         }
1259         case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1260            write_ubo_descriptor(ptr, src);
1261            break;
1262         case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1263            assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1264            write_buffer_descriptor(device, set->dynamic_descriptors + dst_offset, src);
1265            break;
1266         }
1267         case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1268            write_buffer_descriptor(device, ptr, src);
1269            break;
1270         case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1271         case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1272            write_texel_buffer_descriptor(ptr, *(VkBufferView *) src);
1273            break;
1274         case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1275         case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: {
1276            write_image_descriptor(ptr, templ->entry[i].descriptor_type,  src);
1277            break;
1278         }
1279         case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1280            write_combined_image_sampler_descriptor(ptr,
1281                                                    templ->entry[i].descriptor_type,
1282                                                    src,
1283                                                    templ->entry[i].has_sampler);
1284            if (samplers)
1285               write_sampler_push(ptr + A6XX_TEX_CONST_DWORDS, &samplers[j]);
1286            break;
1287         case VK_DESCRIPTOR_TYPE_SAMPLER:
1288            if (templ->entry[i].has_sampler)
1289               write_sampler_descriptor(ptr, src);
1290            else if (samplers)
1291               write_sampler_push(ptr, &samplers[j]);
1292            break;
1293         case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1294            /* nothing in descriptor set - framebuffer state is used instead */
1295            if (unlikely(device->instance->debug_flags & TU_DEBUG_DYNAMIC))
1296               write_image_descriptor(ptr, templ->entry[i].descriptor_type, src);
1297            break;
1298         default:
1299            unreachable("unimplemented descriptor type");
1300            break;
1301         }
1302         src = (char *) src + templ->entry[i].src_stride;
1303         ptr += templ->entry[i].dst_stride;
1304         dst_offset += templ->entry[i].dst_stride;
1305      }
1306   }
1307}
1308
1309VKAPI_ATTR void VKAPI_CALL
1310tu_UpdateDescriptorSetWithTemplate(
1311   VkDevice _device,
1312   VkDescriptorSet descriptorSet,
1313   VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1314   const void *pData)
1315{
1316   TU_FROM_HANDLE(tu_device, device, _device);
1317   TU_FROM_HANDLE(tu_descriptor_set, set, descriptorSet);
1318
1319   tu_update_descriptor_set_with_template(device, set, descriptorUpdateTemplate, pData);
1320}
1321
1322VKAPI_ATTR VkResult VKAPI_CALL
1323tu_CreateSamplerYcbcrConversion(
1324   VkDevice _device,
1325   const VkSamplerYcbcrConversionCreateInfo *pCreateInfo,
1326   const VkAllocationCallbacks *pAllocator,
1327   VkSamplerYcbcrConversion *pYcbcrConversion)
1328{
1329   TU_FROM_HANDLE(tu_device, device, _device);
1330   struct tu_sampler_ycbcr_conversion *conversion;
1331
1332   conversion = vk_object_alloc(&device->vk, pAllocator, sizeof(*conversion),
1333                                VK_OBJECT_TYPE_SAMPLER_YCBCR_CONVERSION);
1334   if (!conversion)
1335      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1336
1337   conversion->format = pCreateInfo->format;
1338   conversion->ycbcr_model = pCreateInfo->ycbcrModel;
1339   conversion->ycbcr_range = pCreateInfo->ycbcrRange;
1340   conversion->components = pCreateInfo->components;
1341   conversion->chroma_offsets[0] = pCreateInfo->xChromaOffset;
1342   conversion->chroma_offsets[1] = pCreateInfo->yChromaOffset;
1343   conversion->chroma_filter = pCreateInfo->chromaFilter;
1344
1345   *pYcbcrConversion = tu_sampler_ycbcr_conversion_to_handle(conversion);
1346   return VK_SUCCESS;
1347}
1348
1349VKAPI_ATTR void VKAPI_CALL
1350tu_DestroySamplerYcbcrConversion(VkDevice _device,
1351                                 VkSamplerYcbcrConversion ycbcrConversion,
1352                                 const VkAllocationCallbacks *pAllocator)
1353{
1354   TU_FROM_HANDLE(tu_device, device, _device);
1355   TU_FROM_HANDLE(tu_sampler_ycbcr_conversion, ycbcr_conversion, ycbcrConversion);
1356
1357   if (!ycbcr_conversion)
1358      return;
1359
1360   vk_object_free(&device->vk, pAllocator, ycbcr_conversion);
1361}
1362