1/*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 * SPDX-License-Identifier: MIT
5 *
6 * based in part on anv driver which is:
7 * Copyright © 2015 Intel Corporation
8 */
9
10#include "tu_image.h"
11
12#include "fdl/fd6_format_table.h"
13
14#include "util/debug.h"
15#include "util/format/u_format.h"
16#include "vk_util.h"
17#include "drm-uapi/drm_fourcc.h"
18
19#include "tu_android.h"
20#include "tu_cs.h"
21#include "tu_descriptor_set.h"
22#include "tu_device.h"
23#include "tu_formats.h"
24
25uint32_t
26tu6_plane_count(VkFormat format)
27{
28   switch (format) {
29   default:
30      return 1;
31   case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
32   case VK_FORMAT_D32_SFLOAT_S8_UINT:
33      return 2;
34   case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
35      return 3;
36   }
37}
38
39enum pipe_format
40tu6_plane_format(VkFormat format, uint32_t plane)
41{
42   switch (format) {
43   case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
44      return plane ? PIPE_FORMAT_R8G8_UNORM : PIPE_FORMAT_Y8_UNORM;
45   case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
46      return PIPE_FORMAT_R8_UNORM;
47   case VK_FORMAT_D32_SFLOAT_S8_UINT:
48      return plane ? PIPE_FORMAT_S8_UINT : PIPE_FORMAT_Z32_FLOAT;
49   default:
50      return tu_vk_format_to_pipe_format(format);
51   }
52}
53
54uint32_t
55tu6_plane_index(VkFormat format, VkImageAspectFlags aspect_mask)
56{
57   switch (aspect_mask) {
58   default:
59      assert(aspect_mask != VK_IMAGE_ASPECT_MEMORY_PLANE_3_BIT_EXT);
60      return 0;
61   case VK_IMAGE_ASPECT_PLANE_1_BIT:
62   case VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT:
63      return 1;
64   case VK_IMAGE_ASPECT_PLANE_2_BIT:
65   case VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT:
66      return 2;
67   case VK_IMAGE_ASPECT_STENCIL_BIT:
68      return format == VK_FORMAT_D32_SFLOAT_S8_UINT;
69   }
70}
71
72enum pipe_format
73tu_format_for_aspect(enum pipe_format format, VkImageAspectFlags aspect_mask)
74{
75   switch (format) {
76   case PIPE_FORMAT_Z24_UNORM_S8_UINT:
77      if (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT)
78         return PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8;
79      if (aspect_mask & VK_IMAGE_ASPECT_STENCIL_BIT) {
80         if (aspect_mask & VK_IMAGE_ASPECT_DEPTH_BIT)
81            return PIPE_FORMAT_Z24_UNORM_S8_UINT;
82         else
83            return PIPE_FORMAT_X24S8_UINT;
84      } else {
85         return PIPE_FORMAT_Z24X8_UNORM;
86      }
87   case PIPE_FORMAT_Z24X8_UNORM:
88      if (aspect_mask & VK_IMAGE_ASPECT_COLOR_BIT)
89         return PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8;
90      return PIPE_FORMAT_Z24X8_UNORM;
91   default:
92      return format;
93   }
94}
95
96static bool
97tu_is_r8g8(enum pipe_format format)
98{
99   return (util_format_get_blocksize(format) == 2) &&
100          (util_format_get_nr_components(format) == 2);
101}
102
103static bool
104tu_is_r8g8_compatible(enum pipe_format format)
105{
106   return (util_format_get_blocksize(format) == 2) &&
107          !util_format_is_depth_or_stencil(format);
108}
109
110void
111tu_cs_image_ref(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer)
112{
113   tu_cs_emit(cs, iview->PITCH);
114   tu_cs_emit(cs, iview->layer_size >> 6);
115   tu_cs_emit_qw(cs, iview->base_addr + iview->layer_size * layer);
116}
117
118void
119tu_cs_image_stencil_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
120{
121   tu_cs_emit(cs, iview->stencil_PITCH);
122   tu_cs_emit(cs, iview->stencil_layer_size >> 6);
123   tu_cs_emit_qw(cs, iview->stencil_base_addr + iview->stencil_layer_size * layer);
124}
125
126void
127tu_cs_image_depth_ref(struct tu_cs *cs, const struct tu_image_view *iview, uint32_t layer)
128{
129   tu_cs_emit(cs, iview->depth_PITCH);
130   tu_cs_emit(cs, iview->depth_layer_size >> 6);
131   tu_cs_emit_qw(cs, iview->depth_base_addr + iview->depth_layer_size * layer);
132}
133
134void
135tu_cs_image_ref_2d(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer, bool src)
136{
137   tu_cs_emit_qw(cs, iview->base_addr + iview->layer_size * layer);
138   /* SP_PS_2D_SRC_PITCH has shifted pitch field */
139   tu_cs_emit(cs, iview->PITCH << (src ? 9 : 0));
140}
141
142void
143tu_cs_image_flag_ref(struct tu_cs *cs, const struct fdl6_view *iview, uint32_t layer)
144{
145   tu_cs_emit_qw(cs, iview->ubwc_addr + iview->ubwc_layer_size * layer);
146   tu_cs_emit(cs, iview->FLAG_BUFFER_PITCH);
147}
148
149static void
150tu_image_view_init(struct tu_device *device,
151                   struct tu_image_view *iview,
152                   const VkImageViewCreateInfo *pCreateInfo,
153                   bool has_z24uint_s8uint)
154{
155   TU_FROM_HANDLE(tu_image, image, pCreateInfo->image);
156   const VkImageSubresourceRange *range = &pCreateInfo->subresourceRange;
157   VkFormat vk_format = pCreateInfo->format;
158   VkImageAspectFlagBits aspect_mask = pCreateInfo->subresourceRange.aspectMask;
159
160   const struct VkSamplerYcbcrConversionInfo *ycbcr_conversion =
161      vk_find_struct_const(pCreateInfo->pNext, SAMPLER_YCBCR_CONVERSION_INFO);
162   const struct tu_sampler_ycbcr_conversion *conversion = ycbcr_conversion ?
163      tu_sampler_ycbcr_conversion_from_handle(ycbcr_conversion->conversion) : NULL;
164
165   vk_image_view_init(&device->vk, &iview->vk, false, pCreateInfo);
166
167   iview->image = image;
168
169   const struct fdl_layout *layouts[3];
170
171   layouts[0] = &image->layout[tu6_plane_index(image->vk.format, aspect_mask)];
172
173   enum pipe_format format;
174   if (aspect_mask != VK_IMAGE_ASPECT_COLOR_BIT)
175      format = tu6_plane_format(vk_format, tu6_plane_index(vk_format, aspect_mask));
176   else
177      format = tu_vk_format_to_pipe_format(vk_format);
178
179   if (image->vk.format == VK_FORMAT_G8_B8R8_2PLANE_420_UNORM &&
180       aspect_mask == VK_IMAGE_ASPECT_PLANE_0_BIT) {
181      if (vk_format == VK_FORMAT_R8_UNORM) {
182         /* The 0'th plane of this format has a different UBWC compression. */
183         format = PIPE_FORMAT_Y8_UNORM;
184      } else {
185         /* If the user wants to reinterpret this plane, then they should've
186          * set MUTABLE_FORMAT_BIT which should disable UBWC and tiling.
187          */
188         assert(!layouts[0]->ubwc);
189      }
190   }
191
192   if (aspect_mask == VK_IMAGE_ASPECT_COLOR_BIT &&
193       (vk_format == VK_FORMAT_G8_B8R8_2PLANE_420_UNORM ||
194        vk_format == VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM)) {
195      layouts[1] = &image->layout[1];
196      layouts[2] = &image->layout[2];
197   }
198
199   struct fdl_view_args args = {};
200   args.iova = image->iova;
201   args.base_array_layer = range->baseArrayLayer;
202   args.base_miplevel = range->baseMipLevel;
203   args.layer_count = vk_image_subresource_layer_count(&image->vk, range);
204   args.level_count = vk_image_subresource_level_count(&image->vk, range);
205   args.min_lod_clamp = iview->vk.min_lod;
206   args.format = tu_format_for_aspect(format, aspect_mask);
207   vk_component_mapping_to_pipe_swizzle(pCreateInfo->components, args.swiz);
208   if (conversion) {
209      unsigned char conversion_swiz[4], create_swiz[4];
210      memcpy(create_swiz, args.swiz, sizeof(create_swiz));
211      vk_component_mapping_to_pipe_swizzle(conversion->components,
212                                           conversion_swiz);
213      util_format_compose_swizzles(create_swiz, conversion_swiz, args.swiz);
214   }
215
216   switch (pCreateInfo->viewType) {
217   case VK_IMAGE_VIEW_TYPE_1D:
218   case VK_IMAGE_VIEW_TYPE_1D_ARRAY:
219      args.type = FDL_VIEW_TYPE_1D;
220      break;
221   case VK_IMAGE_VIEW_TYPE_2D:
222   case VK_IMAGE_VIEW_TYPE_2D_ARRAY:
223      args.type = FDL_VIEW_TYPE_2D;
224      break;
225   case VK_IMAGE_VIEW_TYPE_CUBE:
226   case VK_IMAGE_VIEW_TYPE_CUBE_ARRAY:
227      args.type = FDL_VIEW_TYPE_CUBE;
228      break;
229   case VK_IMAGE_VIEW_TYPE_3D:
230      args.type = FDL_VIEW_TYPE_3D;
231      break;
232   default:
233      unreachable("unknown view type");
234   }
235
236   STATIC_ASSERT((unsigned)VK_CHROMA_LOCATION_COSITED_EVEN == (unsigned)FDL_CHROMA_LOCATION_COSITED_EVEN);
237   STATIC_ASSERT((unsigned)VK_CHROMA_LOCATION_MIDPOINT == (unsigned)FDL_CHROMA_LOCATION_MIDPOINT);
238   if (conversion) {
239      args.chroma_offsets[0] = (enum fdl_chroma_location) conversion->chroma_offsets[0];
240      args.chroma_offsets[1] = (enum fdl_chroma_location) conversion->chroma_offsets[1];
241   }
242
243   fdl6_view_init(&iview->view, layouts, &args, has_z24uint_s8uint);
244
245   if (image->vk.format == VK_FORMAT_D32_SFLOAT_S8_UINT) {
246      struct fdl_layout *layout = &image->layout[0];
247      iview->depth_base_addr = image->iova +
248         fdl_surface_offset(layout, range->baseMipLevel, range->baseArrayLayer);
249      iview->depth_layer_size = fdl_layer_stride(layout, range->baseMipLevel);
250      iview->depth_PITCH = A6XX_RB_DEPTH_BUFFER_PITCH(fdl_pitch(layout, range->baseMipLevel)).value;
251
252      layout = &image->layout[1];
253      iview->stencil_base_addr = image->iova +
254         fdl_surface_offset(layout, range->baseMipLevel, range->baseArrayLayer);
255      iview->stencil_layer_size = fdl_layer_stride(layout, range->baseMipLevel);
256      iview->stencil_PITCH = A6XX_RB_STENCIL_BUFFER_PITCH(fdl_pitch(layout, range->baseMipLevel)).value;
257   }
258}
259
260bool
261tiling_possible(VkFormat format)
262{
263   if (format == VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM ||
264       format == VK_FORMAT_G8B8G8R8_422_UNORM ||
265       format == VK_FORMAT_B8G8R8G8_422_UNORM)
266      return false;
267
268   return true;
269}
270
271bool
272ubwc_possible(VkFormat format, VkImageType type, VkImageUsageFlags usage,
273              VkImageUsageFlags stencil_usage, const struct fd_dev_info *info,
274              VkSampleCountFlagBits samples, bool use_z24uint_s8uint)
275{
276   /* no UBWC with compressed formats, E5B9G9R9, S8_UINT
277    * (S8_UINT because separate stencil doesn't have UBWC-enable bit)
278    */
279   if (vk_format_is_compressed(format) ||
280       format == VK_FORMAT_E5B9G9R9_UFLOAT_PACK32 ||
281       format == VK_FORMAT_S8_UINT)
282      return false;
283
284   /* In copy_format, we treat snorm as unorm to avoid clamping.  But snorm
285    * and unorm are UBWC incompatible for special values such as all 0's or
286    * all 1's.  Disable UBWC for snorm.
287    */
288   if (vk_format_is_snorm(format))
289      return false;
290
291   if (!info->a6xx.has_8bpp_ubwc &&
292       (format == VK_FORMAT_R8_UNORM ||
293        format == VK_FORMAT_R8_SNORM ||
294        format == VK_FORMAT_R8_UINT ||
295        format == VK_FORMAT_R8_SINT ||
296        format == VK_FORMAT_R8_SRGB))
297      return false;
298
299   if (type == VK_IMAGE_TYPE_3D) {
300      tu_finishme("UBWC with 3D textures");
301      return false;
302   }
303
304   /* Disable UBWC for storage images.
305    *
306    * The closed GL driver skips UBWC for storage images (and additionally
307    * uses linear for writeonly images).  We seem to have image tiling working
308    * in freedreno in general, so turnip matches that.  freedreno also enables
309    * UBWC on images, but it's not really tested due to the lack of
310    * UBWC-enabled mipmaps in freedreno currently.  Just match the closed GL
311    * behavior of no UBWC.
312   */
313   if ((usage | stencil_usage) & VK_IMAGE_USAGE_STORAGE_BIT)
314      return false;
315
316   /* Disable UBWC for D24S8 on A630 in some cases
317    *
318    * VK_IMAGE_ASPECT_STENCIL_BIT image view requires to be able to sample
319    * from the stencil component as UINT, however no format allows this
320    * on a630 (the special FMT6_Z24_UINT_S8_UINT format is missing)
321    *
322    * It must be sampled as FMT6_8_8_8_8_UINT, which is not UBWC-compatible
323    *
324    * If we wish to get the border colors correct without knowing the format
325    * when creating the sampler, we also have to use the A630 workaround.
326    *
327    * Additionally, the special AS_R8G8B8A8 format is broken without UBWC,
328    * so we have to fallback to 8_8_8_8_UNORM when UBWC is disabled
329    */
330   if (!use_z24uint_s8uint &&
331       format == VK_FORMAT_D24_UNORM_S8_UINT &&
332       (stencil_usage & (VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_INPUT_ATTACHMENT_BIT)))
333      return false;
334
335   if (!info->a6xx.has_z24uint_s8uint && samples > VK_SAMPLE_COUNT_1_BIT)
336      return false;
337
338   return true;
339}
340
341static VkResult
342tu_image_init(struct tu_device *device, struct tu_image *image,
343              const VkImageCreateInfo *pCreateInfo, uint64_t modifier,
344              const VkSubresourceLayout *plane_layouts)
345{
346   vk_image_init(&device->vk, &image->vk, pCreateInfo);
347   image->vk.drm_format_mod = modifier;
348
349   enum a6xx_tile_mode tile_mode = TILE6_3;
350   bool ubwc_enabled = true;
351
352   /* use linear tiling if requested */
353   if (pCreateInfo->tiling == VK_IMAGE_TILING_LINEAR || modifier == DRM_FORMAT_MOD_LINEAR) {
354      tile_mode = TILE6_LINEAR;
355      ubwc_enabled = false;
356   }
357
358   /* Force linear tiling for formats with "fake" optimalTilingFeatures */
359   if (!tiling_possible(image->vk.format)) {
360      tile_mode = TILE6_LINEAR;
361      ubwc_enabled = false;
362   }
363
364   /* No sense in tiling a 1D image, you'd just waste space and cache locality. */
365   if (pCreateInfo->imageType == VK_IMAGE_TYPE_1D) {
366      tile_mode = TILE6_LINEAR;
367      ubwc_enabled = false;
368   }
369
370   enum pipe_format format =
371      tu_vk_format_to_pipe_format(image->vk.format);
372   /* Whether a view of the image with an R8G8 format could be made. */
373   bool has_r8g8 = tu_is_r8g8(format);
374
375   /* Mutable images can be reinterpreted as any other compatible format.
376    * This is a problem with UBWC (compression for different formats is different),
377    * but also tiling ("swap" affects how tiled formats are stored in memory)
378    * Depth and stencil formats cannot be reintepreted as another format, and
379    * cannot be linear with sysmem rendering, so don't fall back for those.
380    *
381    * TODO:
382    * - if the fmt_list contains only formats which are swapped, but compatible
383    *   with each other (B8G8R8A8_UNORM and B8G8R8A8_UINT for example), then
384    *   tiling is still possible
385    * - figure out which UBWC compressions are compatible to keep it enabled
386    */
387   if ((pCreateInfo->flags & VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT) &&
388       !vk_format_is_depth_or_stencil(image->vk.format)) {
389      const VkImageFormatListCreateInfo *fmt_list =
390         vk_find_struct_const(pCreateInfo->pNext, IMAGE_FORMAT_LIST_CREATE_INFO);
391      bool may_be_swapped = true;
392      /* Whether a view of the image with a non-R8G8 but R8G8 compatible format
393       * could be made.
394       */
395      bool has_r8g8_compatible = false;
396      if (fmt_list) {
397         may_be_swapped = false;
398         has_r8g8_compatible = !has_r8g8 && tu_is_r8g8_compatible(format);
399         for (uint32_t i = 0; i < fmt_list->viewFormatCount; i++) {
400            enum pipe_format format =
401               tu_vk_format_to_pipe_format(fmt_list->pViewFormats[i]);
402            bool is_r8g8 = tu_is_r8g8(format);
403            has_r8g8 = has_r8g8 || is_r8g8;
404            has_r8g8_compatible = has_r8g8_compatible ||
405                                  (!is_r8g8 && tu_is_r8g8_compatible(format));
406
407            if (tu6_format_texture(format, TILE6_LINEAR).swap) {
408               may_be_swapped = true;
409               break;
410            }
411         }
412      } else {
413         /* If there is no format list it could be reinterpreted as
414          * any compatible format.
415          */
416         has_r8g8 = tu_is_r8g8_compatible(format);
417         has_r8g8_compatible = has_r8g8;
418      }
419
420      if (may_be_swapped)
421         tile_mode = TILE6_LINEAR;
422
423      /* R8G8 have a different block width/height and height alignment from other
424       * formats that would normally be compatible (like R16), and so if we are
425       * trying to, for example, sample R16 as R8G8 we need to demote to linear.
426       */
427      if (has_r8g8 && has_r8g8_compatible)
428         tile_mode = TILE6_LINEAR;
429
430      ubwc_enabled = false;
431   }
432
433   if (!ubwc_possible(image->vk.format, pCreateInfo->imageType,
434                      pCreateInfo->usage, image->vk.stencil_usage,
435                      device->physical_device->info, pCreateInfo->samples,
436                      device->use_z24uint_s8uint))
437      ubwc_enabled = false;
438
439   /* expect UBWC enabled if we asked for it */
440   if (modifier == DRM_FORMAT_MOD_QCOM_COMPRESSED)
441      assert(ubwc_enabled);
442   else if (device->physical_device->instance->debug_flags & TU_DEBUG_NOUBWC)
443      ubwc_enabled = false;
444
445   /* Non-UBWC tiled R8G8 is probably buggy since media formats are always
446    * either linear or UBWC. There is no simple test to reproduce the bug.
447    * However it was observed in the wild leading to an unrecoverable hang
448    * on a650/a660.
449    */
450   if (has_r8g8 && tile_mode == TILE6_3 && !ubwc_enabled) {
451      tile_mode = TILE6_LINEAR;
452   }
453
454   for (uint32_t i = 0; i < tu6_plane_count(image->vk.format); i++) {
455      struct fdl_layout *layout = &image->layout[i];
456      enum pipe_format format = tu6_plane_format(image->vk.format, i);
457      uint32_t width0 = pCreateInfo->extent.width;
458      uint32_t height0 = pCreateInfo->extent.height;
459
460      if (i > 0) {
461         switch (image->vk.format) {
462         case VK_FORMAT_G8_B8R8_2PLANE_420_UNORM:
463         case VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM:
464            /* half width/height on chroma planes */
465            width0 = (width0 + 1) >> 1;
466            height0 = (height0 + 1) >> 1;
467            break;
468         case VK_FORMAT_D32_SFLOAT_S8_UINT:
469            /* no UBWC for separate stencil */
470            ubwc_enabled = false;
471            break;
472         default:
473            break;
474         }
475      }
476
477      struct fdl_explicit_layout plane_layout;
478
479      if (plane_layouts) {
480         /* only expect simple 2D images for now */
481         if (pCreateInfo->mipLevels != 1 ||
482            pCreateInfo->arrayLayers != 1 ||
483            pCreateInfo->extent.depth != 1)
484            return vk_error(device, VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT);
485
486         plane_layout.offset = plane_layouts[i].offset;
487         plane_layout.pitch = plane_layouts[i].rowPitch;
488         /* note: use plane_layouts[0].arrayPitch to support array formats */
489      }
490
491      layout->tile_mode = tile_mode;
492      layout->ubwc = ubwc_enabled;
493
494      if (!fdl6_layout(layout, format,
495                       pCreateInfo->samples,
496                       width0, height0,
497                       pCreateInfo->extent.depth,
498                       pCreateInfo->mipLevels,
499                       pCreateInfo->arrayLayers,
500                       pCreateInfo->imageType == VK_IMAGE_TYPE_3D,
501                       plane_layouts ? &plane_layout : NULL)) {
502         assert(plane_layouts); /* can only fail with explicit layout */
503         return vk_error(device, VK_ERROR_INVALID_DRM_FORMAT_MODIFIER_PLANE_LAYOUT_EXT);
504      }
505
506      if (device->instance->debug_flags & TU_DEBUG_LAYOUT)
507         fdl_dump_layout(layout);
508
509      /* fdl6_layout can't take explicit offset without explicit pitch
510       * add offset manually for extra layouts for planes
511       */
512      if (!plane_layouts && i > 0) {
513         uint32_t offset = ALIGN_POT(image->total_size, 4096);
514         for (int i = 0; i < pCreateInfo->mipLevels; i++) {
515            layout->slices[i].offset += offset;
516            layout->ubwc_slices[i].offset += offset;
517         }
518         layout->size += offset;
519      }
520
521      image->total_size = MAX2(image->total_size, layout->size);
522   }
523
524   const struct util_format_description *desc = util_format_description(image->layout[0].format);
525   if (util_format_has_depth(desc) && !(device->instance->debug_flags & TU_DEBUG_NOLRZ))
526   {
527      /* Depth plane is the first one */
528      struct fdl_layout *layout = &image->layout[0];
529      unsigned width = layout->width0;
530      unsigned height = layout->height0;
531
532      /* LRZ buffer is super-sampled */
533      switch (layout->nr_samples) {
534      case 4:
535         width *= 2;
536         FALLTHROUGH;
537      case 2:
538         height *= 2;
539         break;
540      default:
541         break;
542      }
543
544      unsigned lrz_pitch  = align(DIV_ROUND_UP(width, 8), 32);
545      unsigned lrz_height = align(DIV_ROUND_UP(height, 8), 16);
546
547      image->lrz_height = lrz_height;
548      image->lrz_pitch = lrz_pitch;
549      image->lrz_offset = image->total_size;
550      unsigned lrz_size = lrz_pitch * lrz_height * 2;
551      image->total_size += lrz_size;
552
553      unsigned nblocksx = DIV_ROUND_UP(DIV_ROUND_UP(width, 8), 16);
554      unsigned nblocksy = DIV_ROUND_UP(DIV_ROUND_UP(height, 8), 4);
555
556      /* Fast-clear buffer is 1bit/block */
557      image->lrz_fc_size = DIV_ROUND_UP(nblocksx * nblocksy, 8);
558
559      /* Fast-clear buffer cannot be larger than 512 bytes (HW limitation) */
560      bool has_lrz_fc = image->lrz_fc_size <= 512 &&
561         device->physical_device->info->a6xx.enable_lrz_fast_clear &&
562         !unlikely(device->physical_device->instance->debug_flags & TU_DEBUG_NOLRZFC);
563
564      if (has_lrz_fc || device->physical_device->info->a6xx.has_lrz_dir_tracking) {
565         image->lrz_fc_offset = image->total_size;
566         image->total_size += 512;
567
568         if (device->physical_device->info->a6xx.has_lrz_dir_tracking) {
569            /* Direction tracking uses 1 byte */
570            image->total_size += 1;
571            /* GRAS_LRZ_DEPTH_VIEW needs 5 bytes: 4 for view data and 1 for padding */
572            image->total_size += 5;
573         }
574      }
575
576      if (!has_lrz_fc) {
577         image->lrz_fc_size = 0;
578      }
579   } else {
580      image->lrz_height = 0;
581   }
582
583   return VK_SUCCESS;
584}
585
586VKAPI_ATTR VkResult VKAPI_CALL
587tu_CreateImage(VkDevice _device,
588               const VkImageCreateInfo *pCreateInfo,
589               const VkAllocationCallbacks *alloc,
590               VkImage *pImage)
591{
592   uint64_t modifier = DRM_FORMAT_MOD_INVALID;
593   const VkSubresourceLayout *plane_layouts = NULL;
594
595   TU_FROM_HANDLE(tu_device, device, _device);
596   struct tu_image *image =
597      vk_object_zalloc(&device->vk, alloc, sizeof(*image), VK_OBJECT_TYPE_IMAGE);
598
599   if (!image)
600      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
601
602   if (pCreateInfo->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) {
603      const VkImageDrmFormatModifierListCreateInfoEXT *mod_info =
604         vk_find_struct_const(pCreateInfo->pNext,
605                              IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
606      const VkImageDrmFormatModifierExplicitCreateInfoEXT *drm_explicit_info =
607         vk_find_struct_const(pCreateInfo->pNext,
608                              IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT);
609
610      assert(mod_info || drm_explicit_info);
611
612      if (mod_info) {
613         modifier = DRM_FORMAT_MOD_LINEAR;
614         for (unsigned i = 0; i < mod_info->drmFormatModifierCount; i++) {
615            if (mod_info->pDrmFormatModifiers[i] == DRM_FORMAT_MOD_QCOM_COMPRESSED)
616               modifier = DRM_FORMAT_MOD_QCOM_COMPRESSED;
617         }
618      } else {
619         modifier = drm_explicit_info->drmFormatModifier;
620         assert(modifier == DRM_FORMAT_MOD_LINEAR ||
621                modifier == DRM_FORMAT_MOD_QCOM_COMPRESSED);
622         plane_layouts = drm_explicit_info->pPlaneLayouts;
623      }
624   } else {
625      const struct wsi_image_create_info *wsi_info =
626         vk_find_struct_const(pCreateInfo->pNext, WSI_IMAGE_CREATE_INFO_MESA);
627      if (wsi_info && wsi_info->scanout)
628         modifier = DRM_FORMAT_MOD_LINEAR;
629   }
630
631#ifdef ANDROID
632   const VkNativeBufferANDROID *gralloc_info =
633      vk_find_struct_const(pCreateInfo->pNext, NATIVE_BUFFER_ANDROID);
634   int dma_buf;
635   if (gralloc_info) {
636      VkResult result = tu_gralloc_info(device, gralloc_info, &dma_buf, &modifier);
637      if (result != VK_SUCCESS)
638         return result;
639   }
640#endif
641
642   VkResult result = tu_image_init(device, image, pCreateInfo, modifier,
643                                   plane_layouts);
644   if (result != VK_SUCCESS) {
645      vk_object_free(&device->vk, alloc, image);
646      return result;
647   }
648
649   *pImage = tu_image_to_handle(image);
650
651#ifdef ANDROID
652   if (gralloc_info)
653      return tu_import_memory_from_gralloc_handle(_device, dma_buf, alloc,
654                                                  *pImage);
655#endif
656   return VK_SUCCESS;
657}
658
659VKAPI_ATTR void VKAPI_CALL
660tu_DestroyImage(VkDevice _device,
661                VkImage _image,
662                const VkAllocationCallbacks *pAllocator)
663{
664   TU_FROM_HANDLE(tu_device, device, _device);
665   TU_FROM_HANDLE(tu_image, image, _image);
666
667   if (!image)
668      return;
669
670#ifdef ANDROID
671   if (image->owned_memory != VK_NULL_HANDLE)
672      tu_FreeMemory(_device, image->owned_memory, pAllocator);
673#endif
674
675   vk_object_free(&device->vk, pAllocator, image);
676}
677
678static void
679tu_get_image_memory_requirements(struct tu_image *image,
680                                 VkMemoryRequirements2 *pMemoryRequirements)
681{
682   pMemoryRequirements->memoryRequirements = (VkMemoryRequirements) {
683      .memoryTypeBits = 1,
684      .alignment = image->layout[0].base_align,
685      .size = image->total_size
686   };
687
688   vk_foreach_struct(ext, pMemoryRequirements->pNext) {
689      switch (ext->sType) {
690      case VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS: {
691         VkMemoryDedicatedRequirements *req =
692            (VkMemoryDedicatedRequirements *) ext;
693         req->requiresDedicatedAllocation =
694            image->vk.external_handle_types != 0;
695         req->prefersDedicatedAllocation = req->requiresDedicatedAllocation;
696         break;
697      }
698      default:
699         break;
700      }
701   }
702}
703
704VKAPI_ATTR void VKAPI_CALL
705tu_GetImageMemoryRequirements2(VkDevice device,
706                               const VkImageMemoryRequirementsInfo2 *pInfo,
707                               VkMemoryRequirements2 *pMemoryRequirements)
708{
709   TU_FROM_HANDLE(tu_image, image, pInfo->image);
710
711   tu_get_image_memory_requirements(image, pMemoryRequirements);
712}
713
714VKAPI_ATTR void VKAPI_CALL
715tu_GetImageSparseMemoryRequirements2(
716   VkDevice device,
717   const VkImageSparseMemoryRequirementsInfo2 *pInfo,
718   uint32_t *pSparseMemoryRequirementCount,
719   VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
720{
721   tu_stub();
722}
723
724VKAPI_ATTR void VKAPI_CALL
725tu_GetDeviceImageMemoryRequirements(
726   VkDevice _device,
727   const VkDeviceImageMemoryRequirements *pInfo,
728   VkMemoryRequirements2 *pMemoryRequirements)
729{
730   TU_FROM_HANDLE(tu_device, device, _device);
731
732   struct tu_image image = {0};
733
734   tu_image_init(device, &image, pInfo->pCreateInfo, DRM_FORMAT_MOD_INVALID,
735                 NULL);
736
737   tu_get_image_memory_requirements(&image, pMemoryRequirements);
738}
739
740VKAPI_ATTR void VKAPI_CALL
741tu_GetDeviceImageSparseMemoryRequirements(
742    VkDevice device,
743    const VkDeviceImageMemoryRequirements *pInfo,
744    uint32_t *pSparseMemoryRequirementCount,
745    VkSparseImageMemoryRequirements2 *pSparseMemoryRequirements)
746{
747   tu_stub();
748}
749
750VKAPI_ATTR void VKAPI_CALL
751tu_GetImageSubresourceLayout(VkDevice _device,
752                             VkImage _image,
753                             const VkImageSubresource *pSubresource,
754                             VkSubresourceLayout *pLayout)
755{
756   TU_FROM_HANDLE(tu_image, image, _image);
757
758   struct fdl_layout *layout =
759      &image->layout[tu6_plane_index(image->vk.format, pSubresource->aspectMask)];
760   const struct fdl_slice *slice = layout->slices + pSubresource->mipLevel;
761
762   pLayout->offset =
763      fdl_surface_offset(layout, pSubresource->mipLevel, pSubresource->arrayLayer);
764   pLayout->rowPitch = fdl_pitch(layout, pSubresource->mipLevel);
765   pLayout->arrayPitch = fdl_layer_stride(layout, pSubresource->mipLevel);
766   pLayout->depthPitch = slice->size0;
767   pLayout->size = pLayout->depthPitch * layout->depth0;
768
769   if (fdl_ubwc_enabled(layout, pSubresource->mipLevel)) {
770      /* UBWC starts at offset 0 */
771      pLayout->offset = 0;
772      /* UBWC scanout won't match what the kernel wants if we have levels/layers */
773      assert(image->vk.mip_levels == 1 && image->vk.array_layers == 1);
774   }
775}
776
777VKAPI_ATTR VkResult VKAPI_CALL
778tu_CreateImageView(VkDevice _device,
779                   const VkImageViewCreateInfo *pCreateInfo,
780                   const VkAllocationCallbacks *pAllocator,
781                   VkImageView *pView)
782{
783   TU_FROM_HANDLE(tu_device, device, _device);
784   struct tu_image_view *view;
785
786   view = vk_object_alloc(&device->vk, pAllocator, sizeof(*view),
787                          VK_OBJECT_TYPE_IMAGE_VIEW);
788   if (view == NULL)
789      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
790
791   tu_image_view_init(device, view, pCreateInfo, device->use_z24uint_s8uint);
792
793   *pView = tu_image_view_to_handle(view);
794
795   return VK_SUCCESS;
796}
797
798VKAPI_ATTR void VKAPI_CALL
799tu_DestroyImageView(VkDevice _device,
800                    VkImageView _iview,
801                    const VkAllocationCallbacks *pAllocator)
802{
803   TU_FROM_HANDLE(tu_device, device, _device);
804   TU_FROM_HANDLE(tu_image_view, iview, _iview);
805
806   if (!iview)
807      return;
808
809   vk_object_free(&device->vk, pAllocator, iview);
810}
811
812void
813tu_buffer_view_init(struct tu_buffer_view *view,
814                    struct tu_device *device,
815                    const VkBufferViewCreateInfo *pCreateInfo)
816{
817   TU_FROM_HANDLE(tu_buffer, buffer, pCreateInfo->buffer);
818
819   view->buffer = buffer;
820
821   uint32_t range;
822   if (pCreateInfo->range == VK_WHOLE_SIZE)
823      range = buffer->size - pCreateInfo->offset;
824   else
825      range = pCreateInfo->range;
826
827   uint8_t swiz[4] = { PIPE_SWIZZLE_X, PIPE_SWIZZLE_Y, PIPE_SWIZZLE_Z,
828                       PIPE_SWIZZLE_W };
829
830   fdl6_buffer_view_init(
831      view->descriptor, tu_vk_format_to_pipe_format(pCreateInfo->format),
832      swiz, buffer->iova + pCreateInfo->offset, range);
833}
834
835VKAPI_ATTR VkResult VKAPI_CALL
836tu_CreateBufferView(VkDevice _device,
837                    const VkBufferViewCreateInfo *pCreateInfo,
838                    const VkAllocationCallbacks *pAllocator,
839                    VkBufferView *pView)
840{
841   TU_FROM_HANDLE(tu_device, device, _device);
842   struct tu_buffer_view *view;
843
844   view = vk_object_alloc(&device->vk, pAllocator, sizeof(*view),
845                          VK_OBJECT_TYPE_BUFFER_VIEW);
846   if (!view)
847      return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
848
849   tu_buffer_view_init(view, device, pCreateInfo);
850
851   *pView = tu_buffer_view_to_handle(view);
852
853   return VK_SUCCESS;
854}
855
856VKAPI_ATTR void VKAPI_CALL
857tu_DestroyBufferView(VkDevice _device,
858                     VkBufferView bufferView,
859                     const VkAllocationCallbacks *pAllocator)
860{
861   TU_FROM_HANDLE(tu_device, device, _device);
862   TU_FROM_HANDLE(tu_buffer_view, view, bufferView);
863
864   if (!view)
865      return;
866
867   vk_object_free(&device->vk, pAllocator, view);
868}
869