1/*
2 * Copyright © 2018 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include "nir.h"
25#include "GL/gl.h"
26#include "linker_util.h"
27#include "gl_nir_linker.h"
28#include "compiler/glsl/ir_uniform.h" /* for gl_uniform_storage */
29#include "main/consts_exts.h"
30#include "main/shader_types.h"
31
32/**
33 * This file do the common link for GLSL atomic counter uniforms, using NIR,
34 * instead of IR as the counter-part glsl/link_uniforms.cpp
35 */
36
37struct active_atomic_counter_uniform {
38   unsigned loc;
39   nir_variable *var;
40};
41
42struct active_atomic_buffer {
43   struct active_atomic_counter_uniform *uniforms;
44   unsigned num_uniforms;
45   unsigned uniform_buffer_size;
46   unsigned stage_counter_references[MESA_SHADER_STAGES];
47   unsigned size;
48};
49
50static void
51add_atomic_counter(const void *ctx,
52                   struct active_atomic_buffer *buffer,
53                   unsigned uniform_loc,
54                   nir_variable *var)
55{
56   if (buffer->num_uniforms >= buffer->uniform_buffer_size) {
57      if (buffer->uniform_buffer_size == 0)
58         buffer->uniform_buffer_size = 1;
59      else
60         buffer->uniform_buffer_size *= 2;
61      buffer->uniforms = reralloc(ctx,
62                                  buffer->uniforms,
63                                  struct active_atomic_counter_uniform,
64                                  buffer->uniform_buffer_size);
65   }
66
67   struct active_atomic_counter_uniform *uniform =
68      buffer->uniforms + buffer->num_uniforms;
69   uniform->loc = uniform_loc;
70   uniform->var = var;
71   buffer->num_uniforms++;
72}
73
74static void
75process_atomic_variable(const struct glsl_type *t,
76                        struct gl_shader_program *prog,
77                        unsigned *uniform_loc,
78                        nir_variable *var,
79                        struct active_atomic_buffer *buffers,
80                        unsigned *num_buffers,
81                        int *offset,
82                        unsigned shader_stage)
83{
84   /* FIXME: Arrays of arrays get counted separately. For example:
85    * x1[3][3][2] = 9 uniforms, 18 atomic counters
86    * x2[3][2]    = 3 uniforms, 6 atomic counters
87    * x3[2]       = 1 uniform, 2 atomic counters
88    *
89    * However this code marks all the counters as active even when they
90    * might not be used.
91    */
92   if (glsl_type_is_array(t) &&
93       glsl_type_is_array(glsl_get_array_element(t))) {
94      for (unsigned i = 0; i < glsl_get_length(t); i++) {
95         process_atomic_variable(glsl_get_array_element(t),
96                                 prog,
97                                 uniform_loc,
98                                 var,
99                                 buffers, num_buffers,
100                                 offset,
101                                 shader_stage);
102      }
103   } else {
104      struct active_atomic_buffer *buf = buffers + var->data.binding;
105      struct gl_uniform_storage *const storage =
106         &prog->data->UniformStorage[*uniform_loc];
107
108      /* If this is the first time the buffer is used, increment
109       * the counter of buffers used.
110       */
111      if (buf->size == 0)
112         (*num_buffers)++;
113
114      add_atomic_counter(buffers, /* ctx */
115                         buf,
116                         *uniform_loc,
117                         var);
118
119      /* When checking for atomic counters we should count every member in
120       * an array as an atomic counter reference.
121       */
122      if (glsl_type_is_array(t))
123         buf->stage_counter_references[shader_stage] += glsl_get_length(t);
124      else
125         buf->stage_counter_references[shader_stage]++;
126      buf->size = MAX2(buf->size, *offset + glsl_atomic_size(t));
127
128      storage->offset = *offset;
129      *offset += glsl_atomic_size(t);
130
131      (*uniform_loc)++;
132   }
133}
134
135static struct active_atomic_buffer *
136find_active_atomic_counters(const struct gl_constants *consts,
137                            struct gl_shader_program *prog,
138                            unsigned *num_buffers)
139{
140   struct active_atomic_buffer *buffers =
141      rzalloc_array(NULL, /* ctx */
142                    struct active_atomic_buffer,
143                    consts->MaxAtomicBufferBindings);
144   *num_buffers = 0;
145
146   for (unsigned i = 0; i < MESA_SHADER_STAGES; ++i) {
147      struct gl_linked_shader *sh = prog->_LinkedShaders[i];
148      if (sh == NULL)
149         continue;
150
151      nir_shader *nir = sh->Program->nir;
152
153      nir_foreach_uniform_variable(var, nir) {
154         if (!glsl_contains_atomic(var->type))
155            continue;
156
157         int offset = var->data.offset;
158         unsigned uniform_loc = var->data.location;
159
160         process_atomic_variable(var->type,
161                                 prog,
162                                 &uniform_loc,
163                                 var,
164                                 buffers,
165                                 num_buffers,
166                                 &offset,
167                                 i);
168      }
169   }
170
171   return buffers;
172}
173
174static bool
175check_atomic_counters_overlap(const nir_variable *x, const nir_variable *y)
176{
177   return ((x->data.offset >= y->data.offset &&
178            x->data.offset < y->data.offset + glsl_atomic_size(y->type)) ||
179           (y->data.offset >= x->data.offset &&
180            y->data.offset < x->data.offset + glsl_atomic_size(x->type)));
181}
182
183static int
184cmp_active_counter_offsets(const void *a, const void *b)
185{
186   const struct active_atomic_counter_uniform *const first =
187      (struct active_atomic_counter_uniform *) a;
188   const struct active_atomic_counter_uniform *const second =
189      (struct active_atomic_counter_uniform *) b;
190
191   return first->var->data.offset - second->var->data.offset;
192}
193
194void
195gl_nir_link_assign_atomic_counter_resources(const struct gl_constants *consts,
196                                            struct gl_shader_program *prog)
197{
198   unsigned num_buffers;
199   unsigned num_atomic_buffers[MESA_SHADER_STAGES] = {0};
200   struct active_atomic_buffer *abs =
201      find_active_atomic_counters(consts, prog, &num_buffers);
202
203   prog->data->AtomicBuffers =
204      rzalloc_array(prog->data, struct gl_active_atomic_buffer, num_buffers);
205   prog->data->NumAtomicBuffers = num_buffers;
206
207   unsigned buffer_idx = 0;
208   for (unsigned binding = 0;
209        binding < consts->MaxAtomicBufferBindings;
210        binding++) {
211
212      /* If the binding was not used, skip.
213       */
214      if (abs[binding].size == 0)
215         continue;
216
217      struct active_atomic_buffer *ab = abs + binding;
218      struct gl_active_atomic_buffer *mab =
219         prog->data->AtomicBuffers + buffer_idx;
220
221      /* Assign buffer-specific fields. */
222      mab->Binding = binding;
223      mab->MinimumSize = ab->size;
224      mab->Uniforms = rzalloc_array(prog->data->AtomicBuffers, GLuint,
225                                    ab->num_uniforms);
226      mab->NumUniforms = ab->num_uniforms;
227
228      /* Assign counter-specific fields. */
229      for (unsigned j = 0; j < ab->num_uniforms; j++) {
230         nir_variable *var = ab->uniforms[j].var;
231         struct gl_uniform_storage *storage =
232            &prog->data->UniformStorage[ab->uniforms[j].loc];
233
234         mab->Uniforms[j] = ab->uniforms[j].loc;
235
236         storage->atomic_buffer_index = buffer_idx;
237         storage->offset = var->data.offset;
238         if (glsl_type_is_array(var->type)) {
239            const struct glsl_type *without_array =
240               glsl_without_array(var->type);
241            storage->array_stride = glsl_atomic_size(without_array);
242         } else {
243            storage->array_stride = 0;
244         }
245         if (!glsl_type_is_matrix(var->type))
246            storage->matrix_stride = 0;
247      }
248
249      /* Assign stage-specific fields. */
250      for (unsigned stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
251         if (ab->stage_counter_references[stage]) {
252            mab->StageReferences[stage] = GL_TRUE;
253            num_atomic_buffers[stage]++;
254         } else {
255            mab->StageReferences[stage] = GL_FALSE;
256         }
257      }
258
259      buffer_idx++;
260   }
261
262   /* Store a list pointers to atomic buffers per stage and store the index
263    * to the intra-stage buffer list in uniform storage.
264    */
265   for (unsigned stage = 0; stage < MESA_SHADER_STAGES; ++stage) {
266      if (prog->_LinkedShaders[stage] == NULL ||
267          num_atomic_buffers[stage] <= 0)
268         continue;
269
270      struct gl_program *gl_prog = prog->_LinkedShaders[stage]->Program;
271      gl_prog->info.num_abos = num_atomic_buffers[stage];
272      gl_prog->sh.AtomicBuffers =
273         rzalloc_array(gl_prog,
274                       struct gl_active_atomic_buffer *,
275                       num_atomic_buffers[stage]);
276
277      gl_prog->nir->info.num_abos = num_atomic_buffers[stage];
278
279      unsigned intra_stage_idx = 0;
280      for (unsigned i = 0; i < num_buffers; i++) {
281         struct gl_active_atomic_buffer *atomic_buffer =
282            &prog->data->AtomicBuffers[i];
283         if (!atomic_buffer->StageReferences[stage])
284            continue;
285
286         gl_prog->sh.AtomicBuffers[intra_stage_idx] = atomic_buffer;
287
288         for (unsigned u = 0; u < atomic_buffer->NumUniforms; u++) {
289            GLuint uniform_loc = atomic_buffer->Uniforms[u];
290            struct gl_opaque_uniform_index *opaque =
291               prog->data->UniformStorage[uniform_loc].opaque + stage;
292            opaque->index = intra_stage_idx;
293            opaque->active = true;
294         }
295
296         intra_stage_idx++;
297      }
298   }
299
300   assert(buffer_idx == num_buffers);
301
302   ralloc_free(abs);
303}
304
305void
306gl_nir_link_check_atomic_counter_resources(const struct gl_constants *consts,
307                                           struct gl_shader_program *prog)
308{
309   unsigned num_buffers;
310   struct active_atomic_buffer *abs =
311      find_active_atomic_counters(consts, prog, &num_buffers);
312   unsigned atomic_counters[MESA_SHADER_STAGES] = {0};
313   unsigned atomic_buffers[MESA_SHADER_STAGES] = {0};
314   unsigned total_atomic_counters = 0;
315   unsigned total_atomic_buffers = 0;
316
317   /* Sum the required resources.  Note that this counts buffers and
318    * counters referenced by several shader stages multiple times
319    * against the combined limit -- That's the behavior the spec
320    * requires.
321    */
322   for (unsigned i = 0; i < consts->MaxAtomicBufferBindings; i++) {
323      if (abs[i].size == 0)
324         continue;
325
326      qsort(abs[i].uniforms, abs[i].num_uniforms,
327            sizeof(struct active_atomic_counter_uniform),
328            cmp_active_counter_offsets);
329
330      for (unsigned j = 1; j < abs[i].num_uniforms; j++) {
331         /* If an overlapping counter found, it must be a reference to the
332          * same counter from a different shader stage.
333          */
334         if (check_atomic_counters_overlap(abs[i].uniforms[j-1].var,
335                                           abs[i].uniforms[j].var)
336             && strcmp(abs[i].uniforms[j-1].var->name,
337                       abs[i].uniforms[j].var->name) != 0) {
338            linker_error(prog, "Atomic counter %s declared at offset %d "
339                         "which is already in use.",
340                         abs[i].uniforms[j].var->name,
341                         abs[i].uniforms[j].var->data.offset);
342         }
343      }
344
345      for (unsigned j = 0; j < MESA_SHADER_STAGES; ++j) {
346         const unsigned n = abs[i].stage_counter_references[j];
347
348         if (n) {
349            atomic_counters[j] += n;
350            total_atomic_counters += n;
351            atomic_buffers[j]++;
352            total_atomic_buffers++;
353         }
354      }
355   }
356
357   /* Check that they are within the supported limits. */
358   for (unsigned i = 0; i < MESA_SHADER_STAGES; i++) {
359      if (atomic_counters[i] > consts->Program[i].MaxAtomicCounters)
360         linker_error(prog, "Too many %s shader atomic counters",
361                      _mesa_shader_stage_to_string(i));
362
363      if (atomic_buffers[i] > consts->Program[i].MaxAtomicBuffers)
364         linker_error(prog, "Too many %s shader atomic counter buffers",
365                      _mesa_shader_stage_to_string(i));
366   }
367
368   if (total_atomic_counters > consts->MaxCombinedAtomicCounters)
369      linker_error(prog, "Too many combined atomic counters");
370
371   if (total_atomic_buffers > consts->MaxCombinedAtomicBuffers)
372      linker_error(prog, "Too many combined atomic buffers");
373
374   ralloc_free(abs);
375}
376