1/*
2 * Copyright 2018 Collabora Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24#ifndef ZINK_CONTEXT_H
25#define ZINK_CONTEXT_H
26
27#define ZINK_FBFETCH_BINDING 6 //COMPUTE+1
28#define ZINK_SHADER_COUNT (PIPE_SHADER_TYPES - 1)
29
30#define ZINK_DEFAULT_MAX_DESCS 5000
31#define ZINK_DEFAULT_DESC_CLAMP (ZINK_DEFAULT_MAX_DESCS * 0.9)
32#define ZINK_MAX_SHADER_IMAGES 32
33#define ZINK_MAX_BINDLESS_HANDLES 1024
34
35#include "zink_clear.h"
36#include "zink_pipeline.h"
37#include "zink_batch.h"
38#include "zink_compiler.h"
39#include "zink_descriptors.h"
40#include "zink_surface.h"
41
42#include "pipe/p_context.h"
43#include "pipe/p_state.h"
44#include "util/u_rect.h"
45#include "util/u_threaded_context.h"
46#include "util/u_idalloc.h"
47#include "util/slab.h"
48#include "util/list.h"
49#include "util/u_dynarray.h"
50#include "vk_enum_to_str.h"
51
52#include <vulkan/vulkan.h>
53
54#define GFX_SHADER_BITS (VK_PIPELINE_STAGE_VERTEX_SHADER_BIT | \
55                         VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT | \
56                         VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT | \
57                         VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT | \
58                         VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT)
59
60#define pipe_buffer_write "use tc_buffer_write to avoid breaking threaded context"
61
62#ifdef __cplusplus
63extern "C" {
64#endif
65
66struct blitter_context;
67struct list_head;
68
69struct zink_blend_state;
70struct zink_depth_stencil_alpha_state;
71struct zink_gfx_program;
72struct zink_rasterizer_state;
73struct zink_resource;
74struct zink_vertex_elements_state;
75
76enum zink_blit_flags {
77   ZINK_BLIT_NORMAL = 1 << 0,
78   ZINK_BLIT_SAVE_FS = 1 << 1,
79   ZINK_BLIT_SAVE_FB = 1 << 2,
80   ZINK_BLIT_SAVE_TEXTURES = 1 << 3,
81   ZINK_BLIT_NO_COND_RENDER = 1 << 4,
82};
83
84struct zink_sampler_state {
85   VkSampler sampler;
86   VkSampler sampler_clamped;
87   uint32_t hash;
88   struct zink_descriptor_refs desc_set_refs;
89   struct zink_batch_usage *batch_uses;
90   bool custom_border_color;
91   bool emulate_nonseamless;
92};
93
94struct zink_buffer_view {
95   struct pipe_reference reference;
96   struct pipe_resource *pres;
97   VkBufferViewCreateInfo bvci;
98   VkBufferView buffer_view;
99   uint32_t hash;
100   struct zink_batch_usage *batch_uses;
101   struct zink_descriptor_refs desc_set_refs;
102};
103
104struct zink_sampler_view {
105   struct pipe_sampler_view base;
106   union {
107      struct zink_surface *image_view;
108      struct zink_buffer_view *buffer_view;
109   };
110   struct zink_surface *cube_array;
111};
112
113struct zink_image_view {
114   struct pipe_image_view base;
115   union {
116      struct zink_surface *surface;
117      struct zink_buffer_view *buffer_view;
118   };
119};
120
121static inline struct zink_sampler_view *
122zink_sampler_view(struct pipe_sampler_view *pview)
123{
124   return (struct zink_sampler_view *)pview;
125}
126
127struct zink_so_target {
128   struct pipe_stream_output_target base;
129   struct pipe_resource *counter_buffer;
130   VkDeviceSize counter_buffer_offset;
131   uint32_t stride;
132   bool counter_buffer_valid;
133};
134
135static inline struct zink_so_target *
136zink_so_target(struct pipe_stream_output_target *so_target)
137{
138   return (struct zink_so_target *)so_target;
139}
140
141struct zink_viewport_state {
142   struct pipe_viewport_state viewport_states[PIPE_MAX_VIEWPORTS];
143   struct pipe_scissor_state scissor_states[PIPE_MAX_VIEWPORTS];
144   uint8_t num_viewports;
145};
146
147
148struct zink_descriptor_surface {
149   union {
150      struct zink_surface *surface;
151      struct zink_buffer_view *bufferview;
152   };
153   bool is_buffer;
154};
155
156struct zink_bindless_descriptor {
157   struct zink_descriptor_surface ds;
158   struct zink_sampler_state *sampler;
159   uint32_t handle;
160   uint32_t access; //PIPE_ACCESS_...
161};
162
163struct zink_rendering_info {
164   VkPipelineRenderingCreateInfo info;
165   unsigned id;
166};
167
168static inline struct zink_resource *
169zink_descriptor_surface_resource(struct zink_descriptor_surface *ds)
170{
171   return ds->is_buffer ? (struct zink_resource*)ds->bufferview->pres : (struct zink_resource*)ds->surface->base.texture;
172}
173
174typedef void (*pipe_draw_vbo_func)(struct pipe_context *pipe,
175                                   const struct pipe_draw_info *info,
176                                   unsigned drawid_offset,
177                                   const struct pipe_draw_indirect_info *indirect,
178                                   const struct pipe_draw_start_count_bias *draws,
179                                   unsigned num_draws);
180typedef void (*pipe_draw_vertex_state_func)(struct pipe_context *ctx,
181                                            struct pipe_vertex_state *vstate,
182                                            uint32_t partial_velem_mask,
183                                            struct pipe_draw_vertex_state_info info,
184                                            const struct pipe_draw_start_count_bias *draws,
185                                            unsigned num_draws);
186typedef void (*pipe_launch_grid_func)(struct pipe_context *pipe, const struct pipe_grid_info *info);
187
188typedef enum {
189   ZINK_NO_MULTIDRAW,
190   ZINK_MULTIDRAW,
191} zink_multidraw;
192
193typedef enum {
194   ZINK_NO_DYNAMIC_STATE,
195   ZINK_DYNAMIC_STATE,
196   ZINK_DYNAMIC_STATE2,
197   ZINK_DYNAMIC_VERTEX_INPUT,
198} zink_dynamic_state;
199
200struct zink_context {
201   struct pipe_context base;
202   struct threaded_context *tc;
203   struct slab_child_pool transfer_pool;
204   struct slab_child_pool transfer_pool_unsync;
205   struct blitter_context *blitter;
206
207   unsigned flags;
208
209   pipe_draw_vbo_func draw_vbo[2]; //batch changed
210   pipe_draw_vertex_state_func draw_state[2]; //batch changed
211   pipe_launch_grid_func launch_grid[2]; //batch changed
212
213   struct pipe_device_reset_callback reset;
214
215   struct zink_fence *deferred_fence;
216   struct zink_fence *last_fence; //the last command buffer submitted
217   struct zink_batch_state *batch_states; //list of submitted batch states: ordered by increasing timeline id
218   unsigned batch_states_count; //number of states in `batch_states`
219   struct util_dynarray free_batch_states; //unused batch states
220   bool oom_flush;
221   bool oom_stall;
222   struct zink_batch batch;
223
224   unsigned shader_has_inlinable_uniforms_mask;
225   unsigned inlinable_uniforms_valid_mask;
226
227   struct pipe_constant_buffer ubos[PIPE_SHADER_TYPES][PIPE_MAX_CONSTANT_BUFFERS];
228   struct pipe_shader_buffer ssbos[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_BUFFERS];
229   uint32_t writable_ssbos[PIPE_SHADER_TYPES];
230   struct zink_image_view image_views[PIPE_SHADER_TYPES][ZINK_MAX_SHADER_IMAGES];
231
232   uint32_t transient_attachments;
233   struct pipe_framebuffer_state fb_state;
234   struct hash_table framebuffer_cache;
235
236   struct zink_vertex_elements_state *element_state;
237   struct zink_rasterizer_state *rast_state;
238   struct zink_depth_stencil_alpha_state *dsa_state;
239
240   struct hash_table desc_set_layouts[ZINK_DESCRIPTOR_TYPES];
241   struct set desc_pool_keys[ZINK_DESCRIPTOR_TYPES];
242   bool pipeline_changed[2]; //gfx, compute
243
244   struct zink_shader *gfx_stages[ZINK_SHADER_COUNT];
245   struct zink_shader *last_vertex_stage;
246   bool shader_reads_drawid;
247   bool shader_reads_basevertex;
248   struct zink_gfx_pipeline_state gfx_pipeline_state;
249   /* there are 5 gfx stages, but VS and FS are assumed to be always present,
250    * thus only 3 stages need to be considered, giving 2^3 = 8 program caches.
251    */
252   struct hash_table program_cache[8];
253   uint32_t gfx_hash;
254   struct zink_gfx_program *curr_program;
255
256   struct zink_descriptor_data *dd;
257
258   struct zink_shader *compute_stage;
259   struct zink_compute_pipeline_state compute_pipeline_state;
260   struct hash_table compute_program_cache;
261   struct zink_compute_program *curr_compute;
262
263   unsigned shader_stages : ZINK_SHADER_COUNT; /* mask of bound gfx shader stages */
264   unsigned dirty_shader_stages : 6; /* mask of changed shader stages */
265   bool last_vertex_stage_dirty;
266
267   struct {
268      VkRenderingAttachmentInfo attachments[PIPE_MAX_COLOR_BUFS + 2]; //+depth, +stencil
269      VkRenderingInfo info;
270   } dynamic_fb;
271   uint32_t fb_layer_mismatch; //bitmask
272   unsigned depth_bias_scale_factor;
273   struct set rendering_state_cache;
274   struct set render_pass_state_cache;
275   struct hash_table *render_pass_cache;
276   bool new_swapchain;
277   VkExtent2D swapchain_size;
278   bool fb_changed;
279   bool rp_changed; //force renderpass restart
280   bool rp_layout_changed; //renderpass changed, maybe restart
281   bool rp_loadop_changed; //renderpass changed, don't restart
282
283   struct zink_framebuffer *framebuffer;
284   struct zink_framebuffer_clear fb_clears[PIPE_MAX_COLOR_BUFS + 1];
285   uint16_t clears_enabled;
286   uint16_t rp_clears_enabled;
287   uint16_t void_clears;
288   uint16_t fbfetch_outputs;
289   struct zink_resource *needs_present;
290
291   struct pipe_vertex_buffer vertex_buffers[PIPE_MAX_ATTRIBS];
292   bool vertex_buffers_dirty;
293
294   struct zink_sampler_state *sampler_states[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
295   struct pipe_sampler_view *sampler_views[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
296
297   struct zink_viewport_state vp_state;
298   bool vp_state_changed;
299   bool scissor_changed;
300
301   float blend_constants[4];
302
303   bool sample_locations_changed;
304   VkSampleLocationEXT vk_sample_locations[PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE * PIPE_MAX_SAMPLE_LOCATION_GRID_SIZE];
305   uint8_t sample_locations[2 * 4 * 8 * 16];
306
307   struct pipe_stencil_ref stencil_ref;
308
309   union {
310      struct {
311         float default_inner_level[2];
312         float default_outer_level[4];
313      };
314      float tess_levels[6];
315   };
316
317   struct zink_vk_query *curr_xfb_queries[PIPE_MAX_VERTEX_STREAMS];
318
319   struct list_head query_pools;
320   struct list_head suspended_queries;
321   struct list_head primitives_generated_queries;
322   struct zink_query *vertices_query;
323   bool disable_color_writes;
324   bool primitives_generated_active;
325   bool queries_disabled, render_condition_active;
326   struct {
327      struct zink_query *query;
328      bool inverted;
329      bool active; //this is the internal vk state
330   } render_condition;
331
332   struct pipe_resource *dummy_vertex_buffer;
333   struct pipe_resource *dummy_xfb_buffer;
334   struct pipe_surface *dummy_surface[7];
335   struct zink_buffer_view *dummy_bufferview;
336
337   unsigned buffer_rebind_counter;
338   unsigned image_rebind_counter;
339
340   struct {
341      /* descriptor info */
342      VkDescriptorBufferInfo ubos[PIPE_SHADER_TYPES][PIPE_MAX_CONSTANT_BUFFERS];
343      uint32_t push_valid;
344      uint8_t num_ubos[PIPE_SHADER_TYPES];
345
346      VkDescriptorBufferInfo ssbos[PIPE_SHADER_TYPES][PIPE_MAX_SHADER_BUFFERS];
347      uint8_t num_ssbos[PIPE_SHADER_TYPES];
348
349      VkDescriptorImageInfo textures[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
350      VkBufferView tbos[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
351      uint32_t emulate_nonseamless[PIPE_SHADER_TYPES];
352      uint32_t cubes[PIPE_SHADER_TYPES];
353      uint8_t num_samplers[PIPE_SHADER_TYPES];
354      uint8_t num_sampler_views[PIPE_SHADER_TYPES];
355
356      VkDescriptorImageInfo images[PIPE_SHADER_TYPES][ZINK_MAX_SHADER_IMAGES];
357      VkBufferView texel_images[PIPE_SHADER_TYPES][ZINK_MAX_SHADER_IMAGES];
358      uint8_t num_images[PIPE_SHADER_TYPES];
359
360      VkDescriptorImageInfo fbfetch;
361
362      struct zink_resource *descriptor_res[ZINK_DESCRIPTOR_TYPES][PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
363      struct zink_descriptor_surface sampler_surfaces[PIPE_SHADER_TYPES][PIPE_MAX_SAMPLERS];
364      struct zink_descriptor_surface image_surfaces[PIPE_SHADER_TYPES][ZINK_MAX_SHADER_IMAGES];
365
366      struct {
367         struct util_idalloc tex_slots;
368         struct util_idalloc img_slots;
369         struct hash_table tex_handles;
370         struct hash_table img_handles;
371         VkBufferView *buffer_infos; //tex, img
372         VkDescriptorImageInfo *img_infos; //tex, img
373         struct util_dynarray updates;
374         struct util_dynarray resident;
375      } bindless[2];  //img, buffer
376      union {
377         bool bindless_dirty[2]; //tex, img
378         uint16_t any_bindless_dirty;
379      };
380      bool bindless_refs_dirty;
381   } di;
382   struct set *need_barriers[2]; //gfx, compute
383   struct set update_barriers[2][2]; //[gfx, compute][current, next]
384   uint8_t barrier_set_idx[2];
385   unsigned memory_barrier;
386
387   uint32_t num_so_targets;
388   struct pipe_stream_output_target *so_targets[PIPE_MAX_SO_OUTPUTS];
389   bool dirty_so_targets;
390
391   bool gfx_dirty;
392
393   bool is_device_lost;
394   bool primitive_restart;
395   bool vertex_state_changed : 1;
396   bool blend_state_changed : 1;
397   bool rast_state_changed : 1;
398   bool dsa_state_changed : 1;
399   bool stencil_ref_changed : 1;
400   bool rasterizer_discard_changed : 1;
401};
402
403static inline struct zink_context *
404zink_context(struct pipe_context *context)
405{
406   return (struct zink_context *)context;
407}
408
409static inline bool
410zink_fb_clear_enabled(const struct zink_context *ctx, unsigned idx)
411{
412   if (idx == PIPE_MAX_COLOR_BUFS)
413      return ctx->clears_enabled & PIPE_CLEAR_DEPTHSTENCIL;
414   return ctx->clears_enabled & (PIPE_CLEAR_COLOR0 << idx);
415}
416
417void
418zink_fence_wait(struct pipe_context *ctx);
419
420void
421zink_wait_on_batch(struct zink_context *ctx, uint64_t batch_id);
422
423bool
424zink_check_batch_completion(struct zink_context *ctx, uint64_t batch_id);
425VkCommandBuffer
426zink_get_cmdbuf(struct zink_context *ctx, struct zink_resource *src, struct zink_resource *dst);
427void
428zink_flush_queue(struct zink_context *ctx);
429void
430zink_update_fbfetch(struct zink_context *ctx);
431bool
432zink_resource_access_is_write(VkAccessFlags flags);
433
434void
435zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline);
436bool
437zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
438bool
439zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
440void
441zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res,
442                      VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
443
444bool
445zink_resource_needs_barrier(struct zink_resource *res, VkImageLayout layout, VkAccessFlags flags, VkPipelineStageFlags pipeline);
446void
447zink_update_descriptor_refs(struct zink_context *ctx, bool compute);
448void
449zink_init_vk_sample_locations(struct zink_context *ctx, VkSampleLocationsInfoEXT *loc);
450
451void
452zink_batch_rp(struct zink_context *ctx);
453
454void
455zink_batch_no_rp(struct zink_context *ctx);
456
457VkImageView
458zink_prep_fb_attachment(struct zink_context *ctx, struct zink_surface *surf, unsigned i);
459void
460zink_update_vk_sample_locations(struct zink_context *ctx);
461
462static inline VkPipelineStageFlags
463zink_pipeline_flags_from_pipe_stage(enum pipe_shader_type pstage)
464{
465   switch (pstage) {
466   case PIPE_SHADER_VERTEX:
467      return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
468   case PIPE_SHADER_FRAGMENT:
469      return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
470   case PIPE_SHADER_GEOMETRY:
471      return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
472   case PIPE_SHADER_TESS_CTRL:
473      return VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT;
474   case PIPE_SHADER_TESS_EVAL:
475      return VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT;
476   case PIPE_SHADER_COMPUTE:
477      return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
478   default:
479      unreachable("unknown shader stage");
480   }
481}
482
483void
484zink_rebind_all_buffers(struct zink_context *ctx);
485void
486zink_rebind_all_images(struct zink_context *ctx);
487
488void
489zink_flush_memory_barrier(struct zink_context *ctx, bool is_compute);
490void
491zink_init_draw_functions(struct zink_context *ctx, struct zink_screen *screen);
492void
493zink_init_grid_functions(struct zink_context *ctx);
494struct zink_context *
495zink_tc_context_unwrap(struct pipe_context *pctx);
496#ifdef __cplusplus
497}
498#endif
499
500#ifndef __cplusplus
501VkPipelineStageFlags
502zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage);
503
504VkShaderStageFlagBits
505zink_shader_stage(enum pipe_shader_type type);
506
507struct pipe_context *
508zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags);
509
510void
511zink_context_query_init(struct pipe_context *ctx);
512
513void
514zink_blit_begin(struct zink_context *ctx, enum zink_blit_flags flags);
515
516void
517zink_blit(struct pipe_context *pctx,
518          const struct pipe_blit_info *info);
519
520bool
521zink_blit_region_fills(struct u_rect region, unsigned width, unsigned height);
522
523bool
524zink_blit_region_covers(struct u_rect region, struct u_rect covers);
525
526static inline struct u_rect
527zink_rect_from_box(const struct pipe_box *box)
528{
529   return (struct u_rect){box->x, box->x + box->width, box->y, box->y + box->height};
530}
531
532static inline VkComponentSwizzle
533zink_component_mapping(enum pipe_swizzle swizzle)
534{
535   switch (swizzle) {
536   case PIPE_SWIZZLE_X: return VK_COMPONENT_SWIZZLE_R;
537   case PIPE_SWIZZLE_Y: return VK_COMPONENT_SWIZZLE_G;
538   case PIPE_SWIZZLE_Z: return VK_COMPONENT_SWIZZLE_B;
539   case PIPE_SWIZZLE_W: return VK_COMPONENT_SWIZZLE_A;
540   case PIPE_SWIZZLE_0: return VK_COMPONENT_SWIZZLE_ZERO;
541   case PIPE_SWIZZLE_1: return VK_COMPONENT_SWIZZLE_ONE;
542   case PIPE_SWIZZLE_NONE: return VK_COMPONENT_SWIZZLE_IDENTITY; // ???
543   default:
544      unreachable("unexpected swizzle");
545   }
546}
547
548enum pipe_swizzle
549zink_clamp_void_swizzle(const struct util_format_description *desc, enum pipe_swizzle swizzle);
550
551bool
552zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res);
553
554void
555zink_rebind_framebuffer(struct zink_context *ctx, struct zink_resource *res);
556bool
557zink_use_dummy_attachments(const struct zink_context *ctx);
558void
559zink_set_color_write_enables(struct zink_context *ctx);
560void
561zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
562                 unsigned dst_offset, unsigned src_offset, unsigned size);
563
564void
565zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
566                       unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
567                       unsigned src_level, const struct pipe_box *src_box, enum pipe_map_flags map_flags);
568
569void
570zink_destroy_buffer_view(struct zink_screen *screen, struct zink_buffer_view *buffer_view);
571
572void
573debug_describe_zink_buffer_view(char *buf, const struct zink_buffer_view *ptr);
574
575static inline void
576zink_buffer_view_reference(struct zink_screen *screen,
577                           struct zink_buffer_view **dst,
578                           struct zink_buffer_view *src)
579{
580   struct zink_buffer_view *old_dst = dst ? *dst : NULL;
581
582   if (pipe_reference_described(old_dst ? &old_dst->reference : NULL, &src->reference,
583                                (debug_reference_descriptor)debug_describe_zink_buffer_view))
584      zink_destroy_buffer_view(screen, old_dst);
585   if (dst) *dst = src;
586}
587#endif
588
589#endif
590