1/*
2 * Copyright © 2021 Collabora Ltd.
3 *
4 * derived from tu_private.h driver which is:
5 * Copyright © 2016 Red Hat.
6 * Copyright © 2016 Bas Nieuwenhuizen
7 * Copyright © 2015 Intel Corporation
8 *
9 * Permission is hereby granted, free of charge, to any person obtaining a
10 * copy of this software and associated documentation files (the "Software"),
11 * to deal in the Software without restriction, including without limitation
12 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13 * and/or sell copies of the Software, and to permit persons to whom the
14 * Software is furnished to do so, subject to the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the next
17 * paragraph) shall be included in all copies or substantial portions of the
18 * Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
23 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26 * DEALINGS IN THE SOFTWARE.
27 */
28
29#ifndef PANVK_PRIVATE_H
30#define PANVK_PRIVATE_H
31
32#include <assert.h>
33#include <pthread.h>
34#include <stdbool.h>
35#include <stdint.h>
36#include <stdio.h>
37#include <stdlib.h>
38#include <string.h>
39#ifdef HAVE_VALGRIND
40#include <memcheck.h>
41#include <valgrind.h>
42#define VG(x) x
43#else
44#define VG(x)
45#endif
46
47#include "c11/threads.h"
48#include "compiler/shader_enums.h"
49#include "util/list.h"
50#include "util/macros.h"
51#include "vk_alloc.h"
52#include "vk_buffer.h"
53#include "vk_command_buffer.h"
54#include "vk_command_pool.h"
55#include "vk_descriptor_set_layout.h"
56#include "vk_device.h"
57#include "vk_image.h"
58#include "vk_instance.h"
59#include "vk_log.h"
60#include "vk_object.h"
61#include "vk_physical_device.h"
62#include "vk_pipeline_layout.h"
63#include "vk_queue.h"
64#include "vk_sync.h"
65#include "wsi_common.h"
66
67#include "drm-uapi/panfrost_drm.h"
68
69#include "pan_blend.h"
70#include "pan_cs.h"
71#include "pan_device.h"
72#include "panvk_mempool.h"
73#include "pan_texture.h"
74#include "pan_scoreboard.h"
75#include "vk_extensions.h"
76#include "panvk_varyings.h"
77
78/* Pre-declarations needed for WSI entrypoints */
79struct wl_surface;
80struct wl_display;
81typedef struct xcb_connection_t xcb_connection_t;
82typedef uint32_t xcb_visualid_t;
83typedef uint32_t xcb_window_t;
84
85#include <vulkan/vk_android_native_buffer.h>
86#include <vulkan/vk_icd.h>
87#include <vulkan/vulkan.h>
88
89#include "panvk_entrypoints.h"
90
91#define MAX_BIND_POINTS 2 /* compute + graphics */
92#define MAX_VBS 16
93#define MAX_VERTEX_ATTRIBS 16
94#define MAX_RTS 8
95#define MAX_VSC_PIPES 32
96#define MAX_VIEWPORTS 1
97#define MAX_SCISSORS 16
98#define MAX_DISCARD_RECTANGLES 4
99#define MAX_PUSH_CONSTANTS_SIZE 128
100#define MAX_PUSH_DESCRIPTORS 32
101#define MAX_DYNAMIC_UNIFORM_BUFFERS 16
102#define MAX_DYNAMIC_STORAGE_BUFFERS 8
103#define MAX_DYNAMIC_BUFFERS                                                  \
104   (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
105#define MAX_SAMPLES_LOG2 4
106#define NUM_META_FS_KEYS 13
107#define PANVK_MAX_DRM_DEVICES 1
108#define MAX_VIEWS 8
109
110#define NUM_DEPTH_CLEAR_PIPELINES 3
111
112#define PANVK_SYSVAL_UBO_INDEX 0
113#define PANVK_PUSH_CONST_UBO_INDEX 1
114#define PANVK_NUM_BUILTIN_UBOS 2
115
116#define panvk_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
117
118void
119panvk_logi(const char *format, ...) panvk_printflike(1, 2);
120void
121panvk_logi_v(const char *format, va_list va);
122
123#define panvk_stub() assert(!"stub")
124
125#define PANVK_META_COPY_BUF2IMG_NUM_FORMATS 12
126#define PANVK_META_COPY_IMG2BUF_NUM_FORMATS 12
127#define PANVK_META_COPY_IMG2IMG_NUM_FORMATS 14
128#define PANVK_META_COPY_NUM_TEX_TYPES 5
129#define PANVK_META_COPY_BUF2BUF_NUM_BLKSIZES 5
130
131static inline unsigned
132panvk_meta_copy_tex_type(unsigned dim, bool isarray)
133{
134   assert(dim > 0 && dim <= 3);
135   assert(dim < 3 || !isarray);
136   return (((dim - 1) << 1) | (isarray ? 1 : 0));
137}
138
139struct panvk_meta {
140   struct panvk_pool bin_pool;
141   struct panvk_pool desc_pool;
142
143   /* Access to the blitter pools are protected by the blitter
144    * shader/rsd locks. They can't be merged with other binary/desc
145    * pools unless we patch pan_blitter.c to external pool locks.
146    */
147   struct {
148      struct panvk_pool bin_pool;
149      struct panvk_pool desc_pool;
150   } blitter;
151
152   struct {
153      struct {
154         mali_ptr shader;
155         struct pan_shader_info shader_info;
156      } color[3]; /* 3 base types */
157   } clear_attachment;
158
159   struct {
160      struct {
161         mali_ptr rsd;
162      } buf2img[PANVK_META_COPY_BUF2IMG_NUM_FORMATS];
163      struct {
164         mali_ptr rsd;
165      } img2buf[PANVK_META_COPY_NUM_TEX_TYPES][PANVK_META_COPY_IMG2BUF_NUM_FORMATS];
166      struct {
167         mali_ptr rsd;
168      } img2img[2][PANVK_META_COPY_NUM_TEX_TYPES][PANVK_META_COPY_IMG2IMG_NUM_FORMATS];
169      struct {
170         mali_ptr rsd;
171      } buf2buf[PANVK_META_COPY_BUF2BUF_NUM_BLKSIZES];
172      struct {
173         mali_ptr rsd;
174      } fillbuf;
175   } copy;
176};
177
178struct panvk_physical_device {
179   struct vk_physical_device vk;
180
181   /* The API agnostic device object. */
182   struct panfrost_device pdev;
183
184   struct panvk_instance *instance;
185
186   char path[20];
187   char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
188   uint8_t driver_uuid[VK_UUID_SIZE];
189   uint8_t device_uuid[VK_UUID_SIZE];
190   uint8_t cache_uuid[VK_UUID_SIZE];
191
192   struct vk_sync_type drm_syncobj_type;
193   const struct vk_sync_type *sync_types[2];
194
195   struct wsi_device wsi_device;
196   struct panvk_meta meta;
197
198   int master_fd;
199};
200
201enum panvk_debug_flags {
202   PANVK_DEBUG_STARTUP = 1 << 0,
203   PANVK_DEBUG_NIR = 1 << 1,
204   PANVK_DEBUG_TRACE = 1 << 2,
205   PANVK_DEBUG_SYNC = 1 << 3,
206   PANVK_DEBUG_AFBC = 1 << 4,
207   PANVK_DEBUG_LINEAR = 1 << 5,
208   PANVK_DEBUG_DUMP = 1 << 6,
209};
210
211struct panvk_instance {
212   struct vk_instance vk;
213
214   uint32_t api_version;
215   int physical_device_count;
216   struct panvk_physical_device physical_devices[PANVK_MAX_DRM_DEVICES];
217
218   enum panvk_debug_flags debug_flags;
219};
220
221VkResult
222panvk_wsi_init(struct panvk_physical_device *physical_device);
223void
224panvk_wsi_finish(struct panvk_physical_device *physical_device);
225
226bool
227panvk_instance_extension_supported(const char *name);
228uint32_t
229panvk_physical_device_api_version(struct panvk_physical_device *dev);
230bool
231panvk_physical_device_extension_supported(struct panvk_physical_device *dev,
232                                       const char *name);
233
234struct panvk_pipeline_cache {
235   struct vk_object_base base;
236   VkAllocationCallbacks alloc;
237};
238
239/* queue types */
240#define PANVK_QUEUE_GENERAL 0
241
242#define PANVK_MAX_QUEUE_FAMILIES 1
243
244struct panvk_queue {
245   struct vk_queue vk;
246   struct panvk_device *device;
247   uint32_t sync;
248};
249
250struct panvk_device {
251   struct vk_device vk;
252
253   struct vk_device_dispatch_table cmd_dispatch;
254
255   struct panvk_instance *instance;
256
257   struct panvk_queue *queues[PANVK_MAX_QUEUE_FAMILIES];
258   int queue_count[PANVK_MAX_QUEUE_FAMILIES];
259
260   struct panvk_physical_device *physical_device;
261   int _lost;
262};
263
264VkResult _panvk_device_set_lost(struct panvk_device *device,
265                                const char *file, int line,
266                                const char *msg, ...) PRINTFLIKE(4, 5);
267#define panvk_device_set_lost(dev, ...) \
268   _panvk_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
269
270static inline bool
271panvk_device_is_lost(struct panvk_device *device)
272{
273   return unlikely(p_atomic_read(&device->_lost));
274}
275
276#define TILER_DESC_WORDS 56
277
278struct panvk_batch {
279   struct list_head node;
280   struct util_dynarray jobs;
281   struct util_dynarray event_ops;
282   struct pan_scoreboard scoreboard;
283   struct {
284      const struct panvk_framebuffer *info;
285      struct panfrost_ptr desc;
286   } fb;
287   struct {
288      struct panfrost_bo *src, *dst;
289   } blit;
290   struct panfrost_ptr tls;
291   mali_ptr fragment_job;
292   struct {
293      struct pan_tiler_context ctx;
294      struct panfrost_ptr descs;
295      uint32_t templ[TILER_DESC_WORDS];
296   } tiler;
297   struct pan_tls_info tlsinfo;
298   unsigned wls_total_size;
299   bool issued;
300};
301
302enum panvk_event_op_type {
303   PANVK_EVENT_OP_SET,
304   PANVK_EVENT_OP_RESET,
305   PANVK_EVENT_OP_WAIT,
306};
307
308struct panvk_event_op {
309   enum panvk_event_op_type type;
310   struct panvk_event *event;
311};
312
313struct panvk_device_memory {
314   struct vk_object_base base;
315   struct panfrost_bo *bo;
316};
317
318struct panvk_buffer_desc {
319   struct panvk_buffer *buffer;
320   VkDeviceSize offset;
321   VkDeviceSize size;
322};
323
324struct panvk_descriptor_set {
325   struct vk_object_base base;
326   struct panvk_descriptor_pool *pool;
327   const struct panvk_descriptor_set_layout *layout;
328   struct panvk_buffer_desc *dyn_ssbos;
329   void *ubos;
330   struct panvk_buffer_desc *dyn_ubos;
331   void *samplers;
332   void *textures;
333   void *img_attrib_bufs;
334   uint32_t *img_fmts;
335
336   struct panfrost_bo *desc_bo;
337};
338
339#define MAX_SETS 4
340
341struct panvk_descriptor_set_binding_layout {
342   VkDescriptorType type;
343
344   /* Number of array elements in this binding */
345   unsigned array_size;
346
347   /* Indices in the desc arrays */
348   union {
349      struct {
350         union {
351            unsigned sampler_idx;
352            unsigned img_idx;
353         };
354         unsigned tex_idx;
355      };
356      unsigned dyn_ssbo_idx;
357      unsigned ubo_idx;
358      unsigned dyn_ubo_idx;
359   };
360
361   /* Offset into the descriptor UBO where this binding starts */
362   uint32_t desc_ubo_offset;
363
364   /* Stride between descriptors in this binding in the UBO */
365   uint16_t desc_ubo_stride;
366
367   /* Shader stages affected by this set+binding */
368   uint16_t shader_stages;
369
370   struct panvk_sampler **immutable_samplers;
371};
372
373struct panvk_descriptor_set_layout {
374   struct vk_descriptor_set_layout vk;
375
376   /* Shader stages affected by this descriptor set */
377   uint16_t shader_stages;
378
379   unsigned num_samplers;
380   unsigned num_textures;
381   unsigned num_ubos;
382   unsigned num_dyn_ubos;
383   unsigned num_dyn_ssbos;
384   unsigned num_imgs;
385
386   /* Size of the descriptor UBO */
387   uint32_t desc_ubo_size;
388
389   /* Index of the descriptor UBO */
390   unsigned desc_ubo_index;
391
392   /* Number of bindings in this descriptor set */
393   uint32_t binding_count;
394
395   /* Bindings in this descriptor set */
396   struct panvk_descriptor_set_binding_layout bindings[0];
397};
398
399static inline const struct panvk_descriptor_set_layout *
400vk_to_panvk_descriptor_set_layout(const struct vk_descriptor_set_layout *layout)
401{
402   return container_of(layout, const struct panvk_descriptor_set_layout, vk);
403}
404
405struct panvk_pipeline_layout {
406   struct vk_pipeline_layout vk;
407
408   unsigned char sha1[20];
409
410   unsigned num_samplers;
411   unsigned num_textures;
412   unsigned num_ubos;
413   unsigned num_dyn_ubos;
414   unsigned num_dyn_ssbos;
415   uint32_t num_imgs;
416   uint32_t num_sets;
417
418   struct {
419      uint32_t size;
420   } push_constants;
421
422   struct {
423      unsigned sampler_offset;
424      unsigned tex_offset;
425      unsigned ubo_offset;
426      unsigned dyn_ubo_offset;
427      unsigned dyn_ssbo_offset;
428      unsigned img_offset;
429   } sets[MAX_SETS];
430};
431
432static unsigned
433panvk_pipeline_layout_ubo_start(const struct panvk_pipeline_layout *layout,
434                                unsigned set, bool is_dynamic)
435{
436   const struct panvk_descriptor_set_layout *set_layout =
437      vk_to_panvk_descriptor_set_layout(layout->vk.set_layouts[set]);
438
439   unsigned offset = PANVK_NUM_BUILTIN_UBOS +
440                     layout->sets[set].ubo_offset +
441                     layout->sets[set].dyn_ubo_offset;
442
443   if (is_dynamic)
444      offset += set_layout->num_ubos;
445
446   return offset;
447}
448
449static unsigned
450panvk_pipeline_layout_ubo_index(const struct panvk_pipeline_layout *layout,
451                                unsigned set, unsigned binding,
452                                unsigned array_index)
453{
454   const struct panvk_descriptor_set_layout *set_layout =
455      vk_to_panvk_descriptor_set_layout(layout->vk.set_layouts[set]);
456   const struct panvk_descriptor_set_binding_layout *binding_layout =
457      &set_layout->bindings[binding];
458
459   const bool is_dynamic =
460      binding_layout->type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC;
461   const uint32_t ubo_idx = is_dynamic ? binding_layout->dyn_ubo_idx :
462                                         binding_layout->ubo_idx;
463
464   return panvk_pipeline_layout_ubo_start(layout, set, is_dynamic) +
465          ubo_idx + array_index;
466}
467
468struct panvk_desc_pool_counters {
469   unsigned samplers;
470   unsigned combined_image_samplers;
471   unsigned sampled_images;
472   unsigned storage_images;
473   unsigned uniform_texel_bufs;
474   unsigned storage_texel_bufs;
475   unsigned input_attachments;
476   unsigned uniform_bufs;
477   unsigned storage_bufs;
478   unsigned uniform_dyn_bufs;
479   unsigned storage_dyn_bufs;
480   unsigned sets;
481};
482
483struct panvk_descriptor_pool {
484   struct vk_object_base base;
485   struct panvk_desc_pool_counters max;
486   struct panvk_desc_pool_counters cur;
487   struct panvk_descriptor_set *sets;
488};
489
490struct panvk_buffer {
491   struct vk_buffer vk;
492
493   struct panfrost_bo *bo;
494   VkDeviceSize bo_offset;
495};
496
497static inline mali_ptr
498panvk_buffer_gpu_ptr(const struct panvk_buffer *buffer, uint64_t offset)
499{
500   if (buffer->bo == NULL)
501      return 0;
502
503   return buffer->bo->ptr.gpu + buffer->bo_offset + offset;
504}
505
506static inline uint64_t
507panvk_buffer_range(const struct panvk_buffer *buffer,
508                   uint64_t offset, uint64_t range)
509{
510   if (buffer->bo == NULL)
511      return 0;
512
513   return vk_buffer_range(&buffer->vk, offset, range);
514}
515
516enum panvk_dynamic_state_bits {
517   PANVK_DYNAMIC_VIEWPORT = 1 << 0,
518   PANVK_DYNAMIC_SCISSOR = 1 << 1,
519   PANVK_DYNAMIC_LINE_WIDTH = 1 << 2,
520   PANVK_DYNAMIC_DEPTH_BIAS = 1 << 3,
521   PANVK_DYNAMIC_BLEND_CONSTANTS = 1 << 4,
522   PANVK_DYNAMIC_DEPTH_BOUNDS = 1 << 5,
523   PANVK_DYNAMIC_STENCIL_COMPARE_MASK = 1 << 6,
524   PANVK_DYNAMIC_STENCIL_WRITE_MASK = 1 << 7,
525   PANVK_DYNAMIC_STENCIL_REFERENCE = 1 << 8,
526   PANVK_DYNAMIC_DISCARD_RECTANGLE = 1 << 9,
527   PANVK_DYNAMIC_SSBO = 1 << 10,
528   PANVK_DYNAMIC_VERTEX_INSTANCE_OFFSETS = 1 << 11,
529   PANVK_DYNAMIC_ALL = (1 << 12) - 1,
530};
531
532/* This has to match nir_address_format_64bit_bounded_global */
533struct panvk_ssbo_addr {
534   uint64_t base_addr;
535   uint32_t size;
536   uint32_t zero; /* Must be zero! */
537};
538
539union panvk_sysval_vec4 {
540   float f32[4];
541   uint32_t u32[4];
542};
543
544struct panvk_sysvals {
545   union {
546      struct {
547         /* Only for graphics */
548         union panvk_sysval_vec4 viewport_scale;
549         union panvk_sysval_vec4 viewport_offset;
550         union {
551            union panvk_sysval_vec4 vertex_instance_offsets;
552            struct {
553               uint32_t first_vertex;
554               uint32_t base_vertex;
555               uint32_t base_instance;
556            };
557         };
558         union panvk_sysval_vec4 blend_constants;
559      };
560
561      struct {
562         /* Only for compute */
563         union panvk_sysval_vec4 num_work_groups;
564         union panvk_sysval_vec4 local_group_size;
565      };
566   };
567
568   /* The back-end compiler doesn't know about any sysvals after this point */
569
570   struct panvk_ssbo_addr dyn_ssbos[MAX_DYNAMIC_STORAGE_BUFFERS];
571};
572
573struct panvk_descriptor_state {
574   uint32_t dirty;
575   const struct panvk_descriptor_set *sets[MAX_SETS];
576   struct panvk_sysvals sysvals;
577   struct {
578      struct panvk_buffer_desc ubos[MAX_DYNAMIC_UNIFORM_BUFFERS];
579      struct panvk_buffer_desc ssbos[MAX_DYNAMIC_STORAGE_BUFFERS];
580   } dyn;
581   mali_ptr sysvals_ptr;
582   mali_ptr ubos;
583   mali_ptr textures;
584   mali_ptr samplers;
585   mali_ptr push_constants;
586   mali_ptr vs_attribs;
587   mali_ptr vs_attrib_bufs;
588   mali_ptr non_vs_attribs;
589   mali_ptr non_vs_attrib_bufs;
590};
591
592#define INVOCATION_DESC_WORDS 2
593
594struct panvk_draw_info {
595   unsigned first_index;
596   unsigned index_count;
597   unsigned index_size;
598   unsigned first_vertex;
599   unsigned vertex_count;
600   unsigned vertex_range;
601   unsigned padded_vertex_count;
602   unsigned first_instance;
603   unsigned instance_count;
604   int vertex_offset;
605   unsigned offset_start;
606   uint32_t invocation[INVOCATION_DESC_WORDS];
607   struct {
608      mali_ptr varyings;
609      mali_ptr attributes;
610      mali_ptr attribute_bufs;
611      mali_ptr push_constants;
612   } stages[MESA_SHADER_STAGES];
613   mali_ptr varying_bufs;
614   mali_ptr textures;
615   mali_ptr samplers;
616   mali_ptr ubos;
617   mali_ptr position;
618   mali_ptr indices;
619   union {
620      mali_ptr psiz;
621      float line_width;
622   };
623   mali_ptr tls;
624   mali_ptr fb;
625   const struct pan_tiler_context *tiler_ctx;
626   mali_ptr fs_rsd;
627   mali_ptr viewport;
628   struct {
629      struct panfrost_ptr vertex;
630      struct panfrost_ptr tiler;
631   } jobs;
632};
633
634struct panvk_dispatch_info {
635   struct pan_compute_dim wg_count;
636   mali_ptr attributes;
637   mali_ptr attribute_bufs;
638   mali_ptr tsd;
639   mali_ptr ubos;
640   mali_ptr push_uniforms;
641   mali_ptr textures;
642   mali_ptr samplers;
643};
644
645struct panvk_attrib_info {
646   unsigned buf;
647   unsigned offset;
648   enum pipe_format format;
649};
650
651struct panvk_attrib_buf_info {
652   bool special;
653   union {
654      struct {
655         unsigned stride;
656         bool per_instance;
657         uint32_t instance_divisor;
658      };
659      unsigned special_id;
660   };
661};
662
663struct panvk_attribs_info {
664   struct panvk_attrib_info attrib[PAN_MAX_ATTRIBUTE];
665   unsigned attrib_count;
666   struct panvk_attrib_buf_info buf[PAN_MAX_ATTRIBUTE];
667   unsigned buf_count;
668};
669
670struct panvk_attrib_buf {
671   mali_ptr address;
672   unsigned size;
673};
674
675struct panvk_cmd_state {
676   uint32_t dirty;
677
678   struct panvk_varyings_info varyings;
679   mali_ptr fs_rsd;
680
681   struct {
682      float constants[4];
683   } blend;
684
685   struct {
686      struct {
687         float constant_factor;
688         float clamp;
689         float slope_factor;
690      } depth_bias;
691      float line_width;
692   } rast;
693
694   struct {
695      struct panvk_attrib_buf bufs[MAX_VBS];
696      unsigned count;
697   } vb;
698
699   /* Index buffer */
700   struct {
701      struct panvk_buffer *buffer;
702      uint64_t offset;
703      uint8_t index_size;
704      uint32_t first_vertex, base_vertex, base_instance;
705   } ib;
706
707   struct {
708      struct {
709         uint8_t compare_mask;
710         uint8_t write_mask;
711         uint8_t ref;
712      } s_front, s_back;
713   } zs;
714
715   struct {
716      struct pan_fb_info info;
717      bool crc_valid[MAX_RTS];
718   } fb;
719
720   const struct panvk_render_pass *pass;
721   const struct panvk_subpass *subpass;
722   const struct panvk_framebuffer *framebuffer;
723   VkRect2D render_area;
724
725   struct panvk_clear_value *clear;
726
727   mali_ptr vpd;
728   VkViewport viewport;
729   VkRect2D scissor;
730
731   struct panvk_batch *batch;
732};
733
734struct panvk_cmd_pool {
735   struct vk_command_pool vk;
736   struct list_head active_cmd_buffers;
737   struct list_head free_cmd_buffers;
738   struct panvk_bo_pool desc_bo_pool;
739   struct panvk_bo_pool varying_bo_pool;
740   struct panvk_bo_pool tls_bo_pool;
741};
742
743enum panvk_cmd_buffer_status {
744   PANVK_CMD_BUFFER_STATUS_INVALID,
745   PANVK_CMD_BUFFER_STATUS_INITIAL,
746   PANVK_CMD_BUFFER_STATUS_RECORDING,
747   PANVK_CMD_BUFFER_STATUS_EXECUTABLE,
748   PANVK_CMD_BUFFER_STATUS_PENDING,
749};
750
751struct panvk_cmd_bind_point_state {
752   struct panvk_descriptor_state desc_state;
753   const struct panvk_pipeline *pipeline;
754};
755
756struct panvk_cmd_buffer {
757   struct vk_command_buffer vk;
758
759   struct panvk_device *device;
760
761   struct panvk_cmd_pool *pool;
762   struct list_head pool_link;
763   struct panvk_pool desc_pool;
764   struct panvk_pool varying_pool;
765   struct panvk_pool tls_pool;
766   struct list_head batches;
767
768   VkCommandBufferUsageFlags usage_flags;
769   enum panvk_cmd_buffer_status status;
770
771   struct panvk_cmd_state state;
772   uint32_t queue_family_index;
773
774   uint8_t push_constants[MAX_PUSH_CONSTANTS_SIZE];
775   VkShaderStageFlags push_constant_stages;
776   struct panvk_descriptor_set meta_push_descriptors;
777
778   struct panvk_cmd_bind_point_state bind_points[MAX_BIND_POINTS];
779
780   VkResult record_result;
781};
782
783#define panvk_cmd_get_bind_point_state(cmdbuf, bindpoint) \
784        &(cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_ ## bindpoint]
785
786#define panvk_cmd_get_pipeline(cmdbuf, bindpoint) \
787        (cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_ ## bindpoint].pipeline
788
789#define panvk_cmd_get_desc_state(cmdbuf, bindpoint) \
790        &(cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_ ## bindpoint].desc_state
791
792struct panvk_batch *
793panvk_cmd_open_batch(struct panvk_cmd_buffer *cmdbuf);
794
795void
796panvk_cmd_fb_info_set_subpass(struct panvk_cmd_buffer *cmdbuf);
797
798void
799panvk_cmd_fb_info_init(struct panvk_cmd_buffer *cmdbuf);
800
801void
802panvk_cmd_preload_fb_after_batch_split(struct panvk_cmd_buffer *cmdbuf);
803
804void
805panvk_pack_color(struct panvk_clear_value *out,
806                 const VkClearColorValue *in,
807                 enum pipe_format format);
808
809struct panvk_event {
810   struct vk_object_base base;
811   uint32_t syncobj;
812};
813
814struct panvk_shader {
815   struct pan_shader_info info;
816   struct util_dynarray binary;
817   unsigned sysval_ubo;
818   struct pan_compute_dim local_size;
819   bool has_img_access;
820};
821
822struct panvk_shader *
823panvk_shader_create(struct panvk_device *dev,
824                    gl_shader_stage stage,
825                    const VkPipelineShaderStageCreateInfo *stage_info,
826                    const struct panvk_pipeline_layout *layout,
827                    unsigned sysval_ubo,
828                    struct pan_blend_state *blend_state,
829                    bool static_blend_constants,
830                    const VkAllocationCallbacks *alloc);
831
832void
833panvk_shader_destroy(struct panvk_device *dev,
834                     struct panvk_shader *shader,
835                     const VkAllocationCallbacks *alloc);
836
837#define RSD_WORDS 16
838#define BLEND_DESC_WORDS 4
839
840struct panvk_pipeline {
841   struct vk_object_base base;
842
843   struct panvk_varyings_info varyings;
844   struct panvk_attribs_info attribs;
845
846   const struct panvk_pipeline_layout *layout;
847
848   unsigned active_stages;
849
850   uint32_t dynamic_state_mask;
851
852   struct panfrost_bo *binary_bo;
853   struct panfrost_bo *state_bo;
854
855   mali_ptr vpd;
856   mali_ptr rsds[MESA_SHADER_STAGES];
857
858   /* shader stage bit is set of the stage accesses storage images */
859   uint32_t img_access_mask;
860
861   unsigned num_ubos;
862
863   struct {
864      unsigned ubo_idx;
865      struct panfrost_sysvals ids;
866   } sysvals[MESA_SHADER_STAGES];
867
868   unsigned tls_size;
869   unsigned wls_size;
870
871   struct {
872      mali_ptr address;
873      struct pan_shader_info info;
874      uint32_t rsd_template[RSD_WORDS];
875      bool required;
876      bool dynamic_rsd;
877      uint8_t rt_mask;
878   } fs;
879
880   struct {
881      struct pan_compute_dim local_size;
882   } cs;
883
884   struct {
885      unsigned topology;
886      bool writes_point_size;
887      bool primitive_restart;
888   } ia;
889
890   struct {
891      bool clamp_depth;
892      float line_width;
893      struct {
894         bool enable;
895         float constant_factor;
896         float clamp;
897         float slope_factor;
898      } depth_bias;
899      bool front_ccw;
900      bool cull_front_face;
901      bool cull_back_face;
902      bool enable;
903   } rast;
904
905   struct {
906      bool z_test;
907      bool z_write;
908      unsigned z_compare_func;
909      bool s_test;
910      struct {
911         unsigned fail_op;
912         unsigned pass_op;
913         unsigned z_fail_op;
914         unsigned compare_func;
915         uint8_t compare_mask;
916         uint8_t write_mask;
917         uint8_t ref;
918      } s_front, s_back;
919   } zs;
920
921   struct {
922      uint8_t rast_samples;
923      uint8_t min_samples;
924      uint16_t sample_mask;
925      bool alpha_to_coverage;
926      bool alpha_to_one;
927   } ms;
928
929   struct {
930      struct pan_blend_state state;
931      uint32_t bd_template[8][BLEND_DESC_WORDS];
932      struct {
933         uint8_t index;
934         uint16_t bifrost_factor;
935      } constant[8];
936      bool reads_dest;
937   } blend;
938
939   VkViewport viewport;
940   VkRect2D scissor;
941};
942
943#define PANVK_MAX_PLANES 1
944
945struct panvk_image {
946   struct vk_image vk;
947
948   struct pan_image pimage;
949};
950
951unsigned
952panvk_image_get_plane_size(const struct panvk_image *image, unsigned plane);
953
954unsigned
955panvk_image_get_total_size(const struct panvk_image *image);
956
957#define TEXTURE_DESC_WORDS 8
958#define ATTRIB_BUF_DESC_WORDS 4
959
960struct panvk_image_view {
961   struct vk_image_view vk;
962
963   struct pan_image_view pview;
964
965   struct panfrost_bo *bo;
966   struct {
967      uint32_t tex[TEXTURE_DESC_WORDS];
968      uint32_t img_attrib_buf[ATTRIB_BUF_DESC_WORDS * 2];
969   } descs;
970};
971
972#define SAMPLER_DESC_WORDS 8
973
974struct panvk_sampler {
975   struct vk_object_base base;
976   uint32_t desc[SAMPLER_DESC_WORDS];
977};
978
979struct panvk_buffer_view {
980   struct vk_object_base base;
981   struct panfrost_bo *bo;
982   struct {
983      uint32_t tex[TEXTURE_DESC_WORDS];
984      uint32_t img_attrib_buf[ATTRIB_BUF_DESC_WORDS * 2];
985   } descs;
986   enum pipe_format fmt;
987   uint32_t elems;
988};
989
990struct panvk_attachment_info {
991   struct panvk_image_view *iview;
992};
993
994struct panvk_framebuffer {
995   struct vk_object_base base;
996
997   uint32_t width;
998   uint32_t height;
999   uint32_t layers;
1000
1001   uint32_t attachment_count;
1002   struct panvk_attachment_info attachments[0];
1003};
1004
1005struct panvk_clear_value {
1006   union {
1007      uint32_t color[4];
1008      struct {
1009         float depth;
1010         uint8_t stencil;
1011      };
1012   };
1013};
1014
1015struct panvk_subpass_attachment {
1016   uint32_t idx;
1017   VkImageLayout layout;
1018   bool clear;
1019   bool preload;
1020};
1021
1022struct panvk_subpass {
1023   uint32_t input_count;
1024   uint32_t color_count;
1025   struct panvk_subpass_attachment *input_attachments;
1026   uint8_t active_color_attachments;
1027   struct panvk_subpass_attachment *color_attachments;
1028   struct panvk_subpass_attachment *resolve_attachments;
1029   struct panvk_subpass_attachment zs_attachment;
1030
1031   uint32_t view_mask;
1032};
1033
1034struct panvk_render_pass_attachment {
1035   VkAttachmentDescriptionFlags flags;
1036   enum pipe_format format;
1037   unsigned samples;
1038   VkAttachmentLoadOp load_op;
1039   VkAttachmentStoreOp store_op;
1040   VkAttachmentLoadOp stencil_load_op;
1041   VkAttachmentStoreOp stencil_store_op;
1042   VkImageLayout initial_layout;
1043   VkImageLayout final_layout;
1044   unsigned view_mask;
1045   unsigned first_used_in_subpass;
1046};
1047
1048struct panvk_render_pass {
1049   struct vk_object_base base;
1050
1051   uint32_t attachment_count;
1052   uint32_t subpass_count;
1053   struct panvk_subpass_attachment *subpass_attachments;
1054   struct panvk_render_pass_attachment *attachments;
1055   struct panvk_subpass subpasses[0];
1056};
1057
1058VK_DEFINE_HANDLE_CASTS(panvk_cmd_buffer, vk.base, VkCommandBuffer, VK_OBJECT_TYPE_COMMAND_BUFFER)
1059VK_DEFINE_HANDLE_CASTS(panvk_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE)
1060VK_DEFINE_HANDLE_CASTS(panvk_instance, vk.base, VkInstance, VK_OBJECT_TYPE_INSTANCE)
1061VK_DEFINE_HANDLE_CASTS(panvk_physical_device, vk.base, VkPhysicalDevice, VK_OBJECT_TYPE_PHYSICAL_DEVICE)
1062VK_DEFINE_HANDLE_CASTS(panvk_queue, vk.base, VkQueue, VK_OBJECT_TYPE_QUEUE)
1063
1064VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_cmd_pool, vk.base, VkCommandPool, VK_OBJECT_TYPE_COMMAND_POOL)
1065VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_buffer, vk.base, VkBuffer, VK_OBJECT_TYPE_BUFFER)
1066VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_buffer_view, base, VkBufferView, VK_OBJECT_TYPE_BUFFER_VIEW)
1067VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_pool, base, VkDescriptorPool, VK_OBJECT_TYPE_DESCRIPTOR_POOL)
1068VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_set, base, VkDescriptorSet, VK_OBJECT_TYPE_DESCRIPTOR_SET)
1069VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_set_layout, vk.base,
1070                               VkDescriptorSetLayout, VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT)
1071VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_device_memory, base, VkDeviceMemory, VK_OBJECT_TYPE_DEVICE_MEMORY)
1072VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_event, base, VkEvent, VK_OBJECT_TYPE_EVENT)
1073VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_framebuffer, base, VkFramebuffer, VK_OBJECT_TYPE_FRAMEBUFFER)
1074VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_image, vk.base, VkImage, VK_OBJECT_TYPE_IMAGE)
1075VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_image_view, vk.base, VkImageView, VK_OBJECT_TYPE_IMAGE_VIEW);
1076VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline_cache, base, VkPipelineCache, VK_OBJECT_TYPE_PIPELINE_CACHE)
1077VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline, base, VkPipeline, VK_OBJECT_TYPE_PIPELINE)
1078VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline_layout, vk.base, VkPipelineLayout, VK_OBJECT_TYPE_PIPELINE_LAYOUT)
1079VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_render_pass, base, VkRenderPass, VK_OBJECT_TYPE_RENDER_PASS)
1080VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_sampler, base, VkSampler, VK_OBJECT_TYPE_SAMPLER)
1081
1082#define panvk_arch_name(name, version) panvk_## version ## _ ## name
1083
1084#define panvk_arch_dispatch(arch, name, ...) \
1085do { \
1086   switch (arch) { \
1087   case 6: panvk_arch_name(name, v6)(__VA_ARGS__); break; \
1088   case 7: panvk_arch_name(name, v7)(__VA_ARGS__); break; \
1089   default: unreachable("Invalid arch"); \
1090   } \
1091} while (0)
1092
1093#ifdef PAN_ARCH
1094#if PAN_ARCH == 6
1095#define panvk_per_arch(name) panvk_arch_name(name, v6)
1096#elif PAN_ARCH == 7
1097#define panvk_per_arch(name) panvk_arch_name(name, v7)
1098#endif
1099#include "panvk_vX_cmd_buffer.h"
1100#include "panvk_vX_cs.h"
1101#include "panvk_vX_device.h"
1102#include "panvk_vX_meta.h"
1103#else
1104#define PAN_ARCH 6
1105#define panvk_per_arch(name) panvk_arch_name(name, v6)
1106#include "panvk_vX_cmd_buffer.h"
1107#include "panvk_vX_cs.h"
1108#include "panvk_vX_device.h"
1109#include "panvk_vX_meta.h"
1110#undef PAN_ARCH
1111#undef panvk_per_arch
1112#define PAN_ARCH 7
1113#define panvk_per_arch(name) panvk_arch_name(name, v7)
1114#include "panvk_vX_cmd_buffer.h"
1115#include "panvk_vX_cs.h"
1116#include "panvk_vX_device.h"
1117#include "panvk_vX_meta.h"
1118#undef PAN_ARCH
1119#undef panvk_per_arch
1120#endif
1121
1122#ifdef PAN_ARCH
1123bool
1124panvk_per_arch(blend_needs_lowering)(const struct panfrost_device *dev,
1125                                     const struct pan_blend_state *state,
1126                                     unsigned rt);
1127
1128struct panvk_shader *
1129panvk_per_arch(shader_create)(struct panvk_device *dev,
1130                              gl_shader_stage stage,
1131                              const VkPipelineShaderStageCreateInfo *stage_info,
1132                              const struct panvk_pipeline_layout *layout,
1133                              unsigned sysval_ubo,
1134                              struct pan_blend_state *blend_state,
1135                              bool static_blend_constants,
1136                              const VkAllocationCallbacks *alloc);
1137struct nir_shader;
1138
1139bool
1140panvk_per_arch(nir_lower_descriptors)(struct nir_shader *nir,
1141                                      struct panvk_device *dev,
1142                                      const struct panvk_pipeline_layout *layout,
1143                                      bool *has_img_access_out);
1144#endif
1145
1146#endif /* PANVK_PRIVATE_H */
1147