1/*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23/**
24 * @file iris_draw.c
25 *
26 * The main driver hooks for drawing and launching compute shaders.
27 */
28
29#include <stdio.h>
30#include <errno.h>
31#include "pipe/p_defines.h"
32#include "pipe/p_state.h"
33#include "pipe/p_context.h"
34#include "pipe/p_screen.h"
35#include "util/u_draw.h"
36#include "util/u_inlines.h"
37#include "util/u_transfer.h"
38#include "util/u_upload_mgr.h"
39#include "intel/compiler/brw_compiler.h"
40#include "intel/compiler/brw_eu_defines.h"
41#include "compiler/shader_info.h"
42#include "iris_context.h"
43#include "iris_defines.h"
44
45static bool
46prim_is_points_or_lines(const struct pipe_draw_info *draw)
47{
48   /* We don't need to worry about adjacency - it can only be used with
49    * geometry shaders, and we don't care about this info when GS is on.
50    */
51   return draw->mode == PIPE_PRIM_POINTS ||
52          draw->mode == PIPE_PRIM_LINES ||
53          draw->mode == PIPE_PRIM_LINE_LOOP ||
54          draw->mode == PIPE_PRIM_LINE_STRIP;
55}
56
57/**
58 * Record the current primitive mode and restart information, flagging
59 * related packets as dirty if necessary.
60 *
61 * This must be called before updating compiled shaders, because the patch
62 * information informs the TCS key.
63 */
64static void
65iris_update_draw_info(struct iris_context *ice,
66                      const struct pipe_draw_info *info)
67{
68   struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen;
69   const struct intel_device_info *devinfo = &screen->devinfo;
70   const struct brw_compiler *compiler = screen->compiler;
71
72   if (ice->state.prim_mode != info->mode) {
73      ice->state.prim_mode = info->mode;
74      ice->state.dirty |= IRIS_DIRTY_VF_TOPOLOGY;
75
76
77      /* For XY Clip enables */
78      bool points_or_lines = prim_is_points_or_lines(info);
79      if (points_or_lines != ice->state.prim_is_points_or_lines) {
80         ice->state.prim_is_points_or_lines = points_or_lines;
81         ice->state.dirty |= IRIS_DIRTY_CLIP;
82      }
83   }
84
85   if (info->mode == PIPE_PRIM_PATCHES &&
86       ice->state.vertices_per_patch != ice->state.patch_vertices) {
87      ice->state.vertices_per_patch = ice->state.patch_vertices;
88      ice->state.dirty |= IRIS_DIRTY_VF_TOPOLOGY;
89
90      /* 8_PATCH TCS needs this for key->input_vertices */
91      if (compiler->use_tcs_8_patch)
92         ice->state.stage_dirty |= IRIS_STAGE_DIRTY_UNCOMPILED_TCS;
93
94      /* Flag constants dirty for gl_PatchVerticesIn if needed. */
95      const struct shader_info *tcs_info =
96         iris_get_shader_info(ice, MESA_SHADER_TESS_CTRL);
97      if (tcs_info &&
98          BITSET_TEST(tcs_info->system_values_read, SYSTEM_VALUE_VERTICES_IN)) {
99         ice->state.stage_dirty |= IRIS_STAGE_DIRTY_CONSTANTS_TCS;
100         ice->state.shaders[MESA_SHADER_TESS_CTRL].sysvals_need_upload = true;
101      }
102   }
103
104   /* Track restart_index changes only if primitive_restart is true */
105   const unsigned cut_index = info->primitive_restart ? info->restart_index :
106                                                        ice->state.cut_index;
107   if (ice->state.primitive_restart != info->primitive_restart ||
108       ice->state.cut_index != cut_index) {
109      ice->state.dirty |= IRIS_DIRTY_VF;
110      ice->state.cut_index = cut_index;
111      ice->state.dirty |=
112         ((ice->state.primitive_restart != info->primitive_restart) &&
113          devinfo->verx10 >= 125) ? IRIS_DIRTY_VFG : 0;
114      ice->state.primitive_restart = info->primitive_restart;
115   }
116}
117
118/**
119 * Update shader draw parameters, flagging VF packets as dirty if necessary.
120 */
121static void
122iris_update_draw_parameters(struct iris_context *ice,
123                            const struct pipe_draw_info *info,
124                            unsigned drawid_offset,
125                            const struct pipe_draw_indirect_info *indirect,
126                            const struct pipe_draw_start_count_bias *draw)
127{
128   bool changed = false;
129
130   if (ice->state.vs_uses_draw_params) {
131      struct iris_state_ref *draw_params = &ice->draw.draw_params;
132
133      if (indirect && indirect->buffer) {
134         pipe_resource_reference(&draw_params->res, indirect->buffer);
135         draw_params->offset =
136            indirect->offset + (info->index_size ? 12 : 8);
137
138         changed = true;
139         ice->draw.params_valid = false;
140      } else {
141         int firstvertex = info->index_size ? draw->index_bias : draw->start;
142
143         if (!ice->draw.params_valid ||
144             ice->draw.params.firstvertex != firstvertex ||
145             ice->draw.params.baseinstance != info->start_instance) {
146
147            changed = true;
148            ice->draw.params.firstvertex = firstvertex;
149            ice->draw.params.baseinstance = info->start_instance;
150            ice->draw.params_valid = true;
151
152            u_upload_data(ice->ctx.const_uploader, 0,
153                          sizeof(ice->draw.params), 4, &ice->draw.params,
154                          &draw_params->offset, &draw_params->res);
155         }
156      }
157   }
158
159   if (ice->state.vs_uses_derived_draw_params) {
160      struct iris_state_ref *derived_params = &ice->draw.derived_draw_params;
161      int is_indexed_draw = info->index_size ? -1 : 0;
162
163      if (ice->draw.derived_params.drawid != drawid_offset ||
164          ice->draw.derived_params.is_indexed_draw != is_indexed_draw) {
165
166         changed = true;
167         ice->draw.derived_params.drawid = drawid_offset;
168         ice->draw.derived_params.is_indexed_draw = is_indexed_draw;
169
170         u_upload_data(ice->ctx.const_uploader, 0,
171                       sizeof(ice->draw.derived_params), 4,
172                       &ice->draw.derived_params,
173                       &derived_params->offset, &derived_params->res);
174      }
175   }
176
177   if (changed) {
178      ice->state.dirty |= IRIS_DIRTY_VERTEX_BUFFERS |
179                          IRIS_DIRTY_VERTEX_ELEMENTS |
180                          IRIS_DIRTY_VF_SGVS;
181   }
182}
183
184static void
185iris_indirect_draw_vbo(struct iris_context *ice,
186                       const struct pipe_draw_info *dinfo,
187                       unsigned drawid_offset,
188                       const struct pipe_draw_indirect_info *dindirect,
189                       const struct pipe_draw_start_count_bias *draw)
190{
191   struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
192   struct pipe_draw_info info = *dinfo;
193   struct pipe_draw_indirect_info indirect = *dindirect;
194
195   iris_emit_buffer_barrier_for(batch, iris_resource_bo(indirect.buffer),
196                                IRIS_DOMAIN_VF_READ);
197
198   if (indirect.indirect_draw_count) {
199      struct iris_bo *draw_count_bo =
200         iris_resource_bo(indirect.indirect_draw_count);
201      iris_emit_buffer_barrier_for(batch, draw_count_bo,
202                                   IRIS_DOMAIN_OTHER_READ);
203
204      if (ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT) {
205         /* Upload MI_PREDICATE_RESULT to GPR15.*/
206         batch->screen->vtbl.load_register_reg64(batch, CS_GPR(15), MI_PREDICATE_RESULT);
207      }
208   }
209
210   const uint64_t orig_dirty = ice->state.dirty;
211   const uint64_t orig_stage_dirty = ice->state.stage_dirty;
212
213   for (int i = 0; i < indirect.draw_count; i++) {
214      iris_batch_maybe_flush(batch, 1500);
215
216      iris_update_draw_parameters(ice, &info, drawid_offset + i, &indirect, draw);
217
218      batch->screen->vtbl.upload_render_state(ice, batch, &info, drawid_offset + i, &indirect, draw);
219
220      ice->state.dirty &= ~IRIS_ALL_DIRTY_FOR_RENDER;
221      ice->state.stage_dirty &= ~IRIS_ALL_STAGE_DIRTY_FOR_RENDER;
222
223      indirect.offset += indirect.stride;
224   }
225
226   if (indirect.indirect_draw_count &&
227       ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT) {
228      /* Restore MI_PREDICATE_RESULT. */
229      batch->screen->vtbl.load_register_reg64(batch, MI_PREDICATE_RESULT, CS_GPR(15));
230   }
231
232   /* Put this back for post-draw resolves, we'll clear it again after. */
233   ice->state.dirty = orig_dirty;
234   ice->state.stage_dirty = orig_stage_dirty;
235}
236
237static void
238iris_simple_draw_vbo(struct iris_context *ice,
239                     const struct pipe_draw_info *draw,
240                     unsigned drawid_offset,
241                     const struct pipe_draw_indirect_info *indirect,
242                     const struct pipe_draw_start_count_bias *sc)
243{
244   struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
245
246   iris_batch_maybe_flush(batch, 1500);
247
248   iris_update_draw_parameters(ice, draw, drawid_offset, indirect, sc);
249
250   batch->screen->vtbl.upload_render_state(ice, batch, draw, drawid_offset, indirect, sc);
251}
252
253/**
254 * The pipe->draw_vbo() driver hook.  Performs a draw on the GPU.
255 */
256void
257iris_draw_vbo(struct pipe_context *ctx, const struct pipe_draw_info *info,
258              unsigned drawid_offset,
259              const struct pipe_draw_indirect_info *indirect,
260              const struct pipe_draw_start_count_bias *draws,
261              unsigned num_draws)
262{
263   if (num_draws > 1) {
264      util_draw_multi(ctx, info, drawid_offset, indirect, draws, num_draws);
265      return;
266   }
267
268   if (!indirect && (!draws[0].count || !info->instance_count))
269      return;
270
271   struct iris_context *ice = (struct iris_context *) ctx;
272   struct iris_screen *screen = (struct iris_screen*)ice->ctx.screen;
273   const struct intel_device_info *devinfo = &screen->devinfo;
274   struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
275
276   if (ice->state.predicate == IRIS_PREDICATE_STATE_DONT_RENDER)
277      return;
278
279   if (INTEL_DEBUG(DEBUG_REEMIT)) {
280      ice->state.dirty |= IRIS_ALL_DIRTY_FOR_RENDER;
281      ice->state.stage_dirty |= IRIS_ALL_STAGE_DIRTY_FOR_RENDER;
282   }
283
284   iris_update_draw_info(ice, info);
285
286   if (devinfo->ver == 9)
287      gfx9_toggle_preemption(ice, batch, info);
288
289   iris_update_compiled_shaders(ice);
290
291   if (ice->state.dirty & IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES) {
292      bool draw_aux_buffer_disabled[BRW_MAX_DRAW_BUFFERS] = { };
293      for (gl_shader_stage stage = 0; stage < MESA_SHADER_COMPUTE; stage++) {
294         if (ice->shaders.prog[stage])
295            iris_predraw_resolve_inputs(ice, batch, draw_aux_buffer_disabled,
296                                        stage, true);
297      }
298      iris_predraw_resolve_framebuffer(ice, batch, draw_aux_buffer_disabled);
299   }
300
301   if (ice->state.dirty & IRIS_DIRTY_RENDER_MISC_BUFFER_FLUSHES) {
302      for (gl_shader_stage stage = 0; stage < MESA_SHADER_COMPUTE; stage++)
303         iris_predraw_flush_buffers(ice, batch, stage);
304   }
305
306   iris_binder_reserve_3d(ice);
307
308   batch->screen->vtbl.update_binder_address(batch, &ice->state.binder);
309
310   iris_handle_always_flush_cache(batch);
311
312   if (indirect && indirect->buffer)
313      iris_indirect_draw_vbo(ice, info, drawid_offset, indirect, &draws[0]);
314   else
315      iris_simple_draw_vbo(ice, info, drawid_offset, indirect, &draws[0]);
316
317   iris_handle_always_flush_cache(batch);
318
319   iris_postdraw_update_resolve_tracking(ice, batch);
320
321   ice->state.dirty &= ~IRIS_ALL_DIRTY_FOR_RENDER;
322   ice->state.stage_dirty &= ~IRIS_ALL_STAGE_DIRTY_FOR_RENDER;
323}
324
325static void
326iris_update_grid_size_resource(struct iris_context *ice,
327                               const struct pipe_grid_info *grid)
328{
329   const struct iris_screen *screen = (void *) ice->ctx.screen;
330   const struct isl_device *isl_dev = &screen->isl_dev;
331   struct iris_state_ref *grid_ref = &ice->state.grid_size;
332   struct iris_state_ref *state_ref = &ice->state.grid_surf_state;
333
334   const struct iris_compiled_shader *shader = ice->shaders.prog[MESA_SHADER_COMPUTE];
335   bool grid_needs_surface = shader->bt.used_mask[IRIS_SURFACE_GROUP_CS_WORK_GROUPS];
336   bool grid_updated = false;
337
338   if (grid->indirect) {
339      pipe_resource_reference(&grid_ref->res, grid->indirect);
340      grid_ref->offset = grid->indirect_offset;
341
342      /* Zero out the grid size so that the next non-indirect grid launch will
343       * re-upload it properly.
344       */
345      memset(ice->state.last_grid, 0, sizeof(ice->state.last_grid));
346      grid_updated = true;
347   } else if (memcmp(ice->state.last_grid, grid->grid, sizeof(grid->grid)) != 0) {
348      memcpy(ice->state.last_grid, grid->grid, sizeof(grid->grid));
349      u_upload_data(ice->state.dynamic_uploader, 0, sizeof(grid->grid), 4,
350                    grid->grid, &grid_ref->offset, &grid_ref->res);
351      grid_updated = true;
352   }
353
354   /* If we changed the grid, the old surface state is invalid. */
355   if (grid_updated)
356      pipe_resource_reference(&state_ref->res, NULL);
357
358   /* Skip surface upload if we don't need it or we already have one */
359   if (!grid_needs_surface || state_ref->res)
360      return;
361
362   struct iris_bo *grid_bo = iris_resource_bo(grid_ref->res);
363
364   void *surf_map = NULL;
365   u_upload_alloc(ice->state.surface_uploader, 0, isl_dev->ss.size,
366                  isl_dev->ss.align, &state_ref->offset, &state_ref->res,
367                  &surf_map);
368   state_ref->offset +=
369      iris_bo_offset_from_base_address(iris_resource_bo(state_ref->res));
370   isl_buffer_fill_state(&screen->isl_dev, surf_map,
371                         .address = grid_ref->offset + grid_bo->address,
372                         .size_B = sizeof(grid->grid),
373                         .format = ISL_FORMAT_RAW,
374                         .stride_B = 1,
375                         .mocs = iris_mocs(grid_bo, isl_dev,
376                                           ISL_SURF_USAGE_CONSTANT_BUFFER_BIT));
377
378   ice->state.stage_dirty |= IRIS_STAGE_DIRTY_BINDINGS_CS;
379}
380
381void
382iris_launch_grid(struct pipe_context *ctx, const struct pipe_grid_info *grid)
383{
384   struct iris_context *ice = (struct iris_context *) ctx;
385   struct iris_batch *batch = &ice->batches[IRIS_BATCH_COMPUTE];
386
387   if (ice->state.predicate == IRIS_PREDICATE_STATE_DONT_RENDER)
388      return;
389
390   if (INTEL_DEBUG(DEBUG_REEMIT)) {
391      ice->state.dirty |= IRIS_ALL_DIRTY_FOR_COMPUTE;
392      ice->state.stage_dirty |= IRIS_ALL_STAGE_DIRTY_FOR_COMPUTE;
393   }
394
395   if (ice->state.dirty & IRIS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES)
396      iris_predraw_resolve_inputs(ice, batch, NULL, MESA_SHADER_COMPUTE, false);
397
398   if (ice->state.dirty & IRIS_DIRTY_COMPUTE_MISC_BUFFER_FLUSHES)
399      iris_predraw_flush_buffers(ice, batch, MESA_SHADER_COMPUTE);
400
401   iris_batch_maybe_flush(batch, 1500);
402
403   iris_update_compiled_compute_shader(ice);
404
405   if (memcmp(ice->state.last_block, grid->block, sizeof(grid->block)) != 0) {
406      memcpy(ice->state.last_block, grid->block, sizeof(grid->block));
407      ice->state.stage_dirty |= IRIS_STAGE_DIRTY_CONSTANTS_CS;
408      ice->state.shaders[MESA_SHADER_COMPUTE].sysvals_need_upload = true;
409   }
410
411   if (ice->state.last_grid_dim != grid->work_dim) {
412      ice->state.last_grid_dim = grid->work_dim;
413      ice->state.stage_dirty |= IRIS_STAGE_DIRTY_CONSTANTS_CS;
414      ice->state.shaders[MESA_SHADER_COMPUTE].sysvals_need_upload = true;
415   }
416
417   iris_update_grid_size_resource(ice, grid);
418
419   iris_binder_reserve_compute(ice);
420   batch->screen->vtbl.update_binder_address(batch, &ice->state.binder);
421
422   if (ice->state.compute_predicate) {
423      batch->screen->vtbl.load_register_mem64(batch, MI_PREDICATE_RESULT,
424                                    ice->state.compute_predicate, 0);
425      ice->state.compute_predicate = NULL;
426   }
427
428   iris_handle_always_flush_cache(batch);
429
430   batch->screen->vtbl.upload_compute_state(ice, batch, grid);
431
432   iris_handle_always_flush_cache(batch);
433
434   ice->state.dirty &= ~IRIS_ALL_DIRTY_FOR_COMPUTE;
435   ice->state.stage_dirty &= ~IRIS_ALL_STAGE_DIRTY_FOR_COMPUTE;
436
437   /* Note: since compute shaders can't access the framebuffer, there's
438    * no need to call iris_postdraw_update_resolve_tracking.
439    */
440}
441