1/*
2 * Copyright 2021 Alyssa Rosenzweig
3 * Copyright (C) 2019-2021 Collabora, Ltd.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25#ifndef AGX_STATE_H
26#define AGX_STATE_H
27
28#include "gallium/include/pipe/p_context.h"
29#include "gallium/include/pipe/p_state.h"
30#include "gallium/include/pipe/p_screen.h"
31#include "gallium/auxiliary/util/u_blitter.h"
32#include "asahi/lib/agx_pack.h"
33#include "asahi/lib/agx_bo.h"
34#include "asahi/lib/agx_device.h"
35#include "asahi/lib/pool.h"
36#include "asahi/compiler/agx_compile.h"
37#include "compiler/nir/nir_lower_blend.h"
38#include "util/hash_table.h"
39#include "util/bitset.h"
40
41struct agx_streamout_target {
42   struct pipe_stream_output_target base;
43   uint32_t offset;
44};
45
46struct agx_streamout {
47   struct pipe_stream_output_target *targets[PIPE_MAX_SO_BUFFERS];
48   unsigned num_targets;
49};
50
51static inline struct agx_streamout_target *
52agx_so_target(struct pipe_stream_output_target *target)
53{
54   return (struct agx_streamout_target *)target;
55}
56
57struct agx_compiled_shader {
58   /* Mapped executable memory */
59   struct agx_bo *bo;
60
61   /* Varying descriptor (TODO: is this the right place?) */
62   uint64_t varyings;
63
64   /* Metadata returned from the compiler */
65   struct agx_shader_info info;
66};
67
68struct agx_uncompiled_shader {
69   struct pipe_shader_state base;
70   struct nir_shader *nir;
71   struct hash_table *variants;
72
73   /* Set on VS, passed to FS for linkage */
74   unsigned base_varying;
75};
76
77struct agx_stage {
78   struct agx_uncompiled_shader *shader;
79   uint32_t dirty;
80
81   struct pipe_constant_buffer cb[PIPE_MAX_CONSTANT_BUFFERS];
82   uint32_t cb_mask;
83
84   /* Need full CSOs for u_blitter */
85   struct agx_sampler_state *samplers[PIPE_MAX_SAMPLERS];
86   struct agx_sampler_view *textures[PIPE_MAX_SHADER_SAMPLER_VIEWS];
87
88   unsigned sampler_count, texture_count;
89};
90
91/* Uploaded scissor or depth bias descriptors */
92struct agx_array {
93      struct agx_bo *bo;
94      unsigned count;
95};
96
97struct agx_batch {
98   unsigned width, height, nr_cbufs;
99   struct pipe_surface *cbufs[8];
100   struct pipe_surface *zsbuf;
101
102   /* PIPE_CLEAR_* bitmask */
103   uint32_t clear, draw;
104
105   float clear_color[4];
106   double clear_depth;
107   unsigned clear_stencil;
108
109   /* Resource list requirements, represented as a bit set indexed by BO
110    * handles (GEM handles on Linux, or IOGPU's equivalent on macOS) */
111   BITSET_WORD bo_list[256];
112
113   struct agx_pool pool, pipeline_pool;
114   struct agx_bo *encoder;
115   uint8_t *encoder_current;
116
117   struct agx_array scissor, depth_bias;
118};
119
120struct agx_zsa {
121   struct pipe_depth_stencil_alpha_state base;
122   struct agx_rasterizer_face_packed front, back;
123};
124
125struct agx_blend {
126   bool logicop_enable, blend_enable;
127
128   union {
129      nir_lower_blend_rt rt[8];
130      unsigned logicop_func;
131   };
132};
133
134struct asahi_shader_key {
135   struct agx_shader_key base;
136   struct agx_blend blend;
137   unsigned nr_cbufs;
138   enum pipe_format rt_formats[PIPE_MAX_COLOR_BUFS];
139};
140
141enum agx_dirty {
142   AGX_DIRTY_VERTEX   = BITFIELD_BIT(0),
143   AGX_DIRTY_VIEWPORT = BITFIELD_BIT(1),
144   AGX_DIRTY_SCISSOR_ZBIAS  = BITFIELD_BIT(2),
145};
146
147struct agx_context {
148   struct pipe_context base;
149   struct agx_compiled_shader *vs, *fs;
150   uint32_t dirty;
151
152   struct agx_batch *batch;
153
154   struct pipe_vertex_buffer vertex_buffers[PIPE_MAX_ATTRIBS];
155   uint32_t vb_mask;
156
157   struct agx_stage stage[PIPE_SHADER_TYPES];
158   struct agx_attribute *attributes;
159   struct agx_rasterizer *rast;
160   struct agx_zsa zs;
161   struct agx_blend *blend;
162   struct pipe_blend_color blend_color;
163   struct pipe_viewport_state viewport;
164   struct pipe_scissor_state scissor;
165   struct pipe_stencil_ref stencil_ref;
166   struct agx_streamout streamout;
167   uint16_t sample_mask;
168   struct pipe_framebuffer_state framebuffer;
169
170   struct pipe_query *cond_query;
171   bool cond_cond;
172   enum pipe_render_cond_flag cond_mode;
173
174   bool is_noop;
175
176   uint8_t render_target[8][AGX_RENDER_TARGET_LENGTH];
177
178   struct blitter_context *blitter;
179};
180
181static inline struct agx_context *
182agx_context(struct pipe_context *pctx)
183{
184   return (struct agx_context *) pctx;
185}
186
187struct agx_rasterizer {
188   struct pipe_rasterizer_state base;
189   uint8_t cull[AGX_CULL_LENGTH];
190   uint8_t line_width;
191};
192
193struct agx_query {
194   unsigned	query;
195};
196
197struct agx_sampler_state {
198   struct pipe_sampler_state base;
199
200   /* Prepared descriptor */
201   struct agx_bo *desc;
202};
203
204struct agx_sampler_view {
205   struct pipe_sampler_view base;
206
207   /* Prepared descriptor */
208   struct agx_bo *desc;
209};
210
211struct agx_screen {
212   struct pipe_screen pscreen;
213   struct agx_device dev;
214   struct sw_winsys *winsys;
215};
216
217static inline struct agx_screen *
218agx_screen(struct pipe_screen *p)
219{
220   return (struct agx_screen *)p;
221}
222
223static inline struct agx_device *
224agx_device(struct pipe_screen *p)
225{
226   return &(agx_screen(p)->dev);
227}
228
229/* TODO: UABI, fake for macOS */
230#ifndef DRM_FORMAT_MOD_LINEAR
231#define DRM_FORMAT_MOD_LINEAR 1
232#endif
233#define DRM_FORMAT_MOD_APPLE_64X64_MORTON_ORDER (2)
234
235struct agx_resource {
236   struct pipe_resource	base;
237   uint64_t modifier;
238
239   /* Should probably be part of the modifier. Affects the tiling algorithm, or
240    * something like that.
241    */
242   bool mipmapped;
243
244   /* Hardware backing */
245   struct agx_bo *bo;
246
247   /* Software backing (XXX) */
248   struct sw_displaytarget	*dt;
249   unsigned dt_stride;
250
251   BITSET_DECLARE(data_valid, PIPE_MAX_TEXTURE_LEVELS);
252
253   struct {
254      unsigned offset;
255      unsigned line_stride;
256      unsigned size;
257   } slices[PIPE_MAX_TEXTURE_LEVELS];
258
259   /* Bytes from one miptree to the next */
260   unsigned array_stride;
261
262   /* Metal does not support packed depth/stencil formats; presumably AGX does
263    * not either. Instead, we create separate depth and stencil resources,
264    * managed by u_transfer_helper.  We provide the illusion of packed
265    * resources.
266    */
267   struct agx_resource *separate_stencil;
268
269   /* Internal format, since many depth/stencil formats are emulated. */
270   enum pipe_format internal_format;
271};
272
273static inline struct agx_resource *
274agx_resource(struct pipe_resource *pctx)
275{
276   return (struct agx_resource *) pctx;
277}
278
279/*
280 * Within a resource containing multiple layers and multiple mip levels,
281 * returns the offset from the start of the backing BO of a given level/slice.
282 */
283static inline uint32_t
284agx_texture_offset(struct agx_resource *rsrc, unsigned level, unsigned z)
285{
286   return rsrc->slices[level].offset + (z * rsrc->array_stride);
287}
288
289static inline void *
290agx_map_texture_cpu(struct agx_resource *rsrc, unsigned level, unsigned z)
291{
292   return ((uint8_t *) rsrc->bo->ptr.cpu) + agx_texture_offset(rsrc, level, z);
293}
294
295static inline uint64_t
296agx_map_texture_gpu(struct agx_resource *rsrc, unsigned level, unsigned z)
297{
298   return rsrc->bo->ptr.gpu + (uint64_t) agx_texture_offset(rsrc, level, z);
299}
300
301struct agx_transfer {
302   struct pipe_transfer base;
303   void *map;
304   struct {
305      struct pipe_resource *rsrc;
306      struct pipe_box box;
307   } staging;
308};
309
310static inline struct agx_transfer *
311agx_transfer(struct pipe_transfer *p)
312{
313   return (struct agx_transfer *)p;
314}
315
316uint64_t
317agx_push_location(struct agx_context *ctx, struct agx_push push,
318                  enum pipe_shader_type stage);
319
320uint64_t
321agx_build_clear_pipeline(struct agx_context *ctx, uint32_t code, uint64_t clear_buf);
322
323uint64_t
324agx_build_store_pipeline(struct agx_context *ctx, uint32_t code,
325                         uint64_t render_target);
326
327uint64_t
328agx_build_reload_pipeline(struct agx_context *ctx, uint32_t code, struct pipe_surface *surf);
329
330/* Add a BO to a batch. This needs to be amortized O(1) since it's called in
331 * hot paths. To achieve this we model BO lists by bit sets */
332
333static inline void
334agx_batch_add_bo(struct agx_batch *batch, struct agx_bo *bo)
335{
336   if (unlikely(bo->handle > (sizeof(batch->bo_list) * 8)))
337      unreachable("todo: growable");
338
339   BITSET_SET(batch->bo_list, bo->handle);
340}
341
342/* Blit shaders */
343void agx_blit(struct pipe_context *pipe,
344              const struct pipe_blit_info *info);
345
346void agx_internal_shaders(struct agx_device *dev);
347
348#endif
349