1/*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24#include "util/blob.h"
25#include "util/hash_table.h"
26#include "util/debug.h"
27#include "util/disk_cache.h"
28#include "util/mesa-sha1.h"
29#include "nir/nir_serialize.h"
30#include "anv_private.h"
31#include "nir/nir_xfb_info.h"
32#include "vulkan/util/vk_util.h"
33
34static bool
35anv_shader_bin_serialize(struct vk_pipeline_cache_object *object,
36                         struct blob *blob);
37
38struct vk_pipeline_cache_object *
39anv_shader_bin_deserialize(struct vk_device *device,
40                           const void *key_data, size_t key_size,
41                           struct blob_reader *blob);
42
43static void
44anv_shader_bin_destroy(struct vk_pipeline_cache_object *object)
45{
46   struct anv_device *device =
47      container_of(object->device, struct anv_device, vk);
48   struct anv_shader_bin *shader =
49      container_of(object, struct anv_shader_bin, base);
50
51   anv_state_pool_free(&device->instruction_state_pool, shader->kernel);
52   vk_pipeline_cache_object_finish(&shader->base);
53   vk_free(&device->vk.alloc, shader);
54}
55
56static const struct vk_pipeline_cache_object_ops anv_shader_bin_ops = {
57   .serialize = anv_shader_bin_serialize,
58   .deserialize = anv_shader_bin_deserialize,
59   .destroy = anv_shader_bin_destroy,
60};
61
62const struct vk_pipeline_cache_object_ops *const anv_cache_import_ops[2] = {
63   &anv_shader_bin_ops,
64   NULL
65};
66
67struct anv_shader_bin *
68anv_shader_bin_create(struct anv_device *device,
69                      gl_shader_stage stage,
70                      const void *key_data, uint32_t key_size,
71                      const void *kernel_data, uint32_t kernel_size,
72                      const struct brw_stage_prog_data *prog_data_in,
73                      uint32_t prog_data_size,
74                      const struct brw_compile_stats *stats, uint32_t num_stats,
75                      const nir_xfb_info *xfb_info_in,
76                      const struct anv_pipeline_bind_map *bind_map)
77{
78   VK_MULTIALLOC(ma);
79   VK_MULTIALLOC_DECL(&ma, struct anv_shader_bin, shader, 1);
80   VK_MULTIALLOC_DECL_SIZE(&ma, void, obj_key_data, key_size);
81   VK_MULTIALLOC_DECL_SIZE(&ma, struct brw_stage_prog_data, prog_data,
82                                prog_data_size);
83   VK_MULTIALLOC_DECL(&ma, struct brw_shader_reloc, prog_data_relocs,
84                           prog_data_in->num_relocs);
85   VK_MULTIALLOC_DECL(&ma, uint32_t, prog_data_param, prog_data_in->nr_params);
86
87   VK_MULTIALLOC_DECL_SIZE(&ma, nir_xfb_info, xfb_info,
88                                xfb_info_in == NULL ? 0 :
89                                nir_xfb_info_size(xfb_info_in->output_count));
90
91   VK_MULTIALLOC_DECL(&ma, struct anv_pipeline_binding, surface_to_descriptor,
92                           bind_map->surface_count);
93   VK_MULTIALLOC_DECL(&ma, struct anv_pipeline_binding, sampler_to_descriptor,
94                           bind_map->sampler_count);
95
96   if (!vk_multialloc_alloc(&ma, &device->vk.alloc,
97                            VK_SYSTEM_ALLOCATION_SCOPE_DEVICE))
98      return NULL;
99
100   memcpy(obj_key_data, key_data, key_size);
101   vk_pipeline_cache_object_init(&device->vk, &shader->base,
102                                 &anv_shader_bin_ops, obj_key_data, key_size);
103
104   shader->stage = stage;
105
106   shader->kernel =
107      anv_state_pool_alloc(&device->instruction_state_pool, kernel_size, 64);
108   memcpy(shader->kernel.map, kernel_data, kernel_size);
109   shader->kernel_size = kernel_size;
110
111   uint64_t shader_data_addr = INSTRUCTION_STATE_POOL_MIN_ADDRESS +
112                               shader->kernel.offset +
113                               prog_data_in->const_data_offset;
114
115   int rv_count = 0;
116   struct brw_shader_reloc_value reloc_values[5];
117   reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
118      .id = BRW_SHADER_RELOC_CONST_DATA_ADDR_LOW,
119      .value = shader_data_addr,
120   };
121   reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
122      .id = BRW_SHADER_RELOC_CONST_DATA_ADDR_HIGH,
123      .value = shader_data_addr >> 32,
124   };
125   reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
126      .id = BRW_SHADER_RELOC_SHADER_START_OFFSET,
127      .value = shader->kernel.offset,
128   };
129   if (brw_shader_stage_is_bindless(stage)) {
130      const struct brw_bs_prog_data *bs_prog_data =
131         brw_bs_prog_data_const(prog_data_in);
132      uint64_t resume_sbt_addr = INSTRUCTION_STATE_POOL_MIN_ADDRESS +
133                                 shader->kernel.offset +
134                                 bs_prog_data->resume_sbt_offset;
135      reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
136         .id = BRW_SHADER_RELOC_RESUME_SBT_ADDR_LOW,
137         .value = resume_sbt_addr,
138      };
139      reloc_values[rv_count++] = (struct brw_shader_reloc_value) {
140         .id = BRW_SHADER_RELOC_RESUME_SBT_ADDR_HIGH,
141         .value = resume_sbt_addr >> 32,
142      };
143   }
144
145   brw_write_shader_relocs(&device->physical->compiler->isa,
146                           shader->kernel.map, prog_data_in,
147                           reloc_values, rv_count);
148
149   memcpy(prog_data, prog_data_in, prog_data_size);
150   typed_memcpy(prog_data_relocs, prog_data_in->relocs,
151                prog_data_in->num_relocs);
152   prog_data->relocs = prog_data_relocs;
153   memset(prog_data_param, 0,
154          prog_data->nr_params * sizeof(*prog_data_param));
155   prog_data->param = prog_data_param;
156   shader->prog_data = prog_data;
157   shader->prog_data_size = prog_data_size;
158
159   assert(num_stats <= ARRAY_SIZE(shader->stats));
160   typed_memcpy(shader->stats, stats, num_stats);
161   shader->num_stats = num_stats;
162
163   if (xfb_info_in) {
164      *xfb_info = *xfb_info_in;
165      typed_memcpy(xfb_info->outputs, xfb_info_in->outputs,
166                   xfb_info_in->output_count);
167      shader->xfb_info = xfb_info;
168   } else {
169      shader->xfb_info = NULL;
170   }
171
172   shader->bind_map = *bind_map;
173   typed_memcpy(surface_to_descriptor, bind_map->surface_to_descriptor,
174                bind_map->surface_count);
175   shader->bind_map.surface_to_descriptor = surface_to_descriptor;
176   typed_memcpy(sampler_to_descriptor, bind_map->sampler_to_descriptor,
177                bind_map->sampler_count);
178   shader->bind_map.sampler_to_descriptor = sampler_to_descriptor;
179
180   return shader;
181}
182
183static bool
184anv_shader_bin_serialize(struct vk_pipeline_cache_object *object,
185                         struct blob *blob)
186{
187   struct anv_shader_bin *shader =
188      container_of(object, struct anv_shader_bin, base);
189
190   blob_write_uint32(blob, shader->stage);
191
192   blob_write_uint32(blob, shader->kernel_size);
193   blob_write_bytes(blob, shader->kernel.map, shader->kernel_size);
194
195   blob_write_uint32(blob, shader->prog_data_size);
196   blob_write_bytes(blob, shader->prog_data, shader->prog_data_size);
197   blob_write_bytes(blob, shader->prog_data->relocs,
198                    shader->prog_data->num_relocs *
199                    sizeof(shader->prog_data->relocs[0]));
200
201   blob_write_uint32(blob, shader->num_stats);
202   blob_write_bytes(blob, shader->stats,
203                    shader->num_stats * sizeof(shader->stats[0]));
204
205   if (shader->xfb_info) {
206      uint32_t xfb_info_size =
207         nir_xfb_info_size(shader->xfb_info->output_count);
208      blob_write_uint32(blob, xfb_info_size);
209      blob_write_bytes(blob, shader->xfb_info, xfb_info_size);
210   } else {
211      blob_write_uint32(blob, 0);
212   }
213
214   blob_write_bytes(blob, shader->bind_map.surface_sha1,
215                    sizeof(shader->bind_map.surface_sha1));
216   blob_write_bytes(blob, shader->bind_map.sampler_sha1,
217                    sizeof(shader->bind_map.sampler_sha1));
218   blob_write_bytes(blob, shader->bind_map.push_sha1,
219                    sizeof(shader->bind_map.push_sha1));
220   blob_write_uint32(blob, shader->bind_map.surface_count);
221   blob_write_uint32(blob, shader->bind_map.sampler_count);
222   blob_write_bytes(blob, shader->bind_map.surface_to_descriptor,
223                    shader->bind_map.surface_count *
224                    sizeof(*shader->bind_map.surface_to_descriptor));
225   blob_write_bytes(blob, shader->bind_map.sampler_to_descriptor,
226                    shader->bind_map.sampler_count *
227                    sizeof(*shader->bind_map.sampler_to_descriptor));
228   blob_write_bytes(blob, shader->bind_map.push_ranges,
229                    sizeof(shader->bind_map.push_ranges));
230
231   return !blob->out_of_memory;
232}
233
234struct vk_pipeline_cache_object *
235anv_shader_bin_deserialize(struct vk_device *vk_device,
236                           const void *key_data, size_t key_size,
237                           struct blob_reader *blob)
238{
239   struct anv_device *device =
240      container_of(vk_device, struct anv_device, vk);
241
242   gl_shader_stage stage = blob_read_uint32(blob);
243
244   uint32_t kernel_size = blob_read_uint32(blob);
245   const void *kernel_data = blob_read_bytes(blob, kernel_size);
246
247   uint32_t prog_data_size = blob_read_uint32(blob);
248   const void *prog_data_bytes = blob_read_bytes(blob, prog_data_size);
249   if (blob->overrun)
250      return NULL;
251
252   union brw_any_prog_data prog_data;
253   memcpy(&prog_data, prog_data_bytes,
254          MIN2(sizeof(prog_data), prog_data_size));
255   prog_data.base.relocs =
256      blob_read_bytes(blob, prog_data.base.num_relocs *
257                            sizeof(prog_data.base.relocs[0]));
258
259   uint32_t num_stats = blob_read_uint32(blob);
260   const struct brw_compile_stats *stats =
261      blob_read_bytes(blob, num_stats * sizeof(stats[0]));
262
263   const nir_xfb_info *xfb_info = NULL;
264   uint32_t xfb_size = blob_read_uint32(blob);
265   if (xfb_size)
266      xfb_info = blob_read_bytes(blob, xfb_size);
267
268   struct anv_pipeline_bind_map bind_map;
269   blob_copy_bytes(blob, bind_map.surface_sha1, sizeof(bind_map.surface_sha1));
270   blob_copy_bytes(blob, bind_map.sampler_sha1, sizeof(bind_map.sampler_sha1));
271   blob_copy_bytes(blob, bind_map.push_sha1, sizeof(bind_map.push_sha1));
272   bind_map.surface_count = blob_read_uint32(blob);
273   bind_map.sampler_count = blob_read_uint32(blob);
274   bind_map.surface_to_descriptor = (void *)
275      blob_read_bytes(blob, bind_map.surface_count *
276                            sizeof(*bind_map.surface_to_descriptor));
277   bind_map.sampler_to_descriptor = (void *)
278      blob_read_bytes(blob, bind_map.sampler_count *
279                            sizeof(*bind_map.sampler_to_descriptor));
280   blob_copy_bytes(blob, bind_map.push_ranges, sizeof(bind_map.push_ranges));
281
282   if (blob->overrun)
283      return NULL;
284
285   struct anv_shader_bin *shader =
286      anv_shader_bin_create(device, stage,
287                            key_data, key_size,
288                            kernel_data, kernel_size,
289                            &prog_data.base, prog_data_size,
290                            stats, num_stats, xfb_info, &bind_map);
291   if (shader == NULL)
292      return NULL;
293
294   return &shader->base;
295}
296
297struct anv_shader_bin *
298anv_device_search_for_kernel(struct anv_device *device,
299                             struct vk_pipeline_cache *cache,
300                             const void *key_data, uint32_t key_size,
301                             bool *user_cache_hit)
302{
303   /* Use the default pipeline cache if none is specified */
304   if (cache == NULL)
305      cache = device->default_pipeline_cache;
306
307   bool cache_hit = false;
308   struct vk_pipeline_cache_object *object =
309      vk_pipeline_cache_lookup_object(cache, key_data, key_size,
310                                      &anv_shader_bin_ops, &cache_hit);
311   if (user_cache_hit != NULL) {
312      *user_cache_hit = object != NULL && cache_hit &&
313                        cache != device->default_pipeline_cache;
314   }
315   if (object == NULL)
316      return NULL;
317
318   return container_of(object, struct anv_shader_bin, base);
319}
320
321struct anv_shader_bin *
322anv_device_upload_kernel(struct anv_device *device,
323                         struct vk_pipeline_cache *cache,
324                         gl_shader_stage stage,
325                         const void *key_data, uint32_t key_size,
326                         const void *kernel_data, uint32_t kernel_size,
327                         const struct brw_stage_prog_data *prog_data,
328                         uint32_t prog_data_size,
329                         const struct brw_compile_stats *stats,
330                         uint32_t num_stats,
331                         const nir_xfb_info *xfb_info,
332                         const struct anv_pipeline_bind_map *bind_map)
333{
334   /* Use the default pipeline cache if none is specified */
335   if (cache == NULL)
336      cache = device->default_pipeline_cache;
337
338   struct anv_shader_bin *shader =
339      anv_shader_bin_create(device, stage,
340                            key_data, key_size,
341                            kernel_data, kernel_size,
342                            prog_data, prog_data_size,
343                            stats, num_stats,
344                            xfb_info, bind_map);
345   if (shader == NULL)
346      return NULL;
347
348   struct vk_pipeline_cache_object *cached =
349      vk_pipeline_cache_add_object(cache, &shader->base);
350
351   return container_of(cached, struct anv_shader_bin, base);
352}
353
354#define SHA1_KEY_SIZE 20
355
356struct nir_shader *
357anv_device_search_for_nir(struct anv_device *device,
358                          struct vk_pipeline_cache *cache,
359                          const nir_shader_compiler_options *nir_options,
360                          unsigned char sha1_key[SHA1_KEY_SIZE],
361                          void *mem_ctx)
362{
363   if (cache == NULL)
364      cache = device->default_pipeline_cache;
365
366   return vk_pipeline_cache_lookup_nir(cache, sha1_key, SHA1_KEY_SIZE,
367                                       nir_options, NULL, mem_ctx);
368}
369
370void
371anv_device_upload_nir(struct anv_device *device,
372                      struct vk_pipeline_cache *cache,
373                      const struct nir_shader *nir,
374                      unsigned char sha1_key[SHA1_KEY_SIZE])
375{
376   if (cache == NULL)
377      cache = device->default_pipeline_cache;
378
379   vk_pipeline_cache_add_nir(cache, sha1_key, SHA1_KEY_SIZE, nir);
380}
381