1/*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23#include <stdio.h>
24#include <time.h>
25#include "pipe/p_defines.h"
26#include "pipe/p_state.h"
27#include "util/debug.h"
28#include "util/ralloc.h"
29#include "util/u_inlines.h"
30#include "util/format/u_format.h"
31#include "util/u_upload_mgr.h"
32#include "drm-uapi/i915_drm.h"
33#include "iris_context.h"
34#include "iris_resource.h"
35#include "iris_screen.h"
36#include "iris_utrace.h"
37#include "common/intel_defines.h"
38#include "common/intel_sample_positions.h"
39
40/**
41 * The pipe->set_debug_callback() driver hook.
42 */
43static void
44iris_set_debug_callback(struct pipe_context *ctx,
45                        const struct util_debug_callback *cb)
46{
47   struct iris_context *ice = (struct iris_context *)ctx;
48   struct iris_screen *screen = (struct iris_screen *)ctx->screen;
49
50   util_queue_finish(&screen->shader_compiler_queue);
51
52   if (cb)
53      ice->dbg = *cb;
54   else
55      memset(&ice->dbg, 0, sizeof(ice->dbg));
56}
57
58/**
59 * Called from the batch module when it detects a GPU hang.
60 *
61 * In this case, we've lost our GEM context, and can't rely on any existing
62 * state on the GPU.  We must mark everything dirty and wipe away any saved
63 * assumptions about the last known state of the GPU.
64 */
65void
66iris_lost_context_state(struct iris_batch *batch)
67{
68   struct iris_context *ice = batch->ice;
69
70   if (batch->name == IRIS_BATCH_RENDER) {
71      batch->screen->vtbl.init_render_context(batch);
72   } else if (batch->name == IRIS_BATCH_COMPUTE) {
73      batch->screen->vtbl.init_compute_context(batch);
74   } else if (batch->name == IRIS_BATCH_BLITTER) {
75      /* No state to set up */
76   } else {
77      unreachable("unhandled batch reset");
78   }
79
80   ice->state.dirty = ~0ull;
81   ice->state.stage_dirty = ~0ull;
82   ice->state.current_hash_scale = 0;
83   memset(&ice->shaders.urb, 0, sizeof(ice->shaders.urb));
84   memset(ice->state.last_block, 0, sizeof(ice->state.last_block));
85   memset(ice->state.last_grid, 0, sizeof(ice->state.last_grid));
86   ice->state.last_grid_dim = 0;
87   batch->last_binder_address = ~0ull;
88   batch->last_aux_map_state = 0;
89   batch->screen->vtbl.lost_genx_state(ice, batch);
90}
91
92static enum pipe_reset_status
93iris_get_device_reset_status(struct pipe_context *ctx)
94{
95   struct iris_context *ice = (struct iris_context *)ctx;
96
97   enum pipe_reset_status worst_reset = PIPE_NO_RESET;
98
99   /* Check the reset status of each batch's hardware context, and take the
100    * worst status (if one was guilty, proclaim guilt).
101    */
102   iris_foreach_batch(ice, batch) {
103      /* This will also recreate the hardware contexts as necessary, so any
104       * future queries will show no resets.  We only want to report once.
105       */
106      enum pipe_reset_status batch_reset =
107         iris_batch_check_for_reset(batch);
108
109      if (batch_reset == PIPE_NO_RESET)
110         continue;
111
112      if (worst_reset == PIPE_NO_RESET) {
113         worst_reset = batch_reset;
114      } else {
115         /* GUILTY < INNOCENT < UNKNOWN */
116         worst_reset = MIN2(worst_reset, batch_reset);
117      }
118   }
119
120   if (worst_reset != PIPE_NO_RESET && ice->reset.reset)
121      ice->reset.reset(ice->reset.data, worst_reset);
122
123   return worst_reset;
124}
125
126static void
127iris_set_device_reset_callback(struct pipe_context *ctx,
128                               const struct pipe_device_reset_callback *cb)
129{
130   struct iris_context *ice = (struct iris_context *)ctx;
131
132   if (cb)
133      ice->reset = *cb;
134   else
135      memset(&ice->reset, 0, sizeof(ice->reset));
136}
137
138static void
139iris_get_sample_position(struct pipe_context *ctx,
140                         unsigned sample_count,
141                         unsigned sample_index,
142                         float *out_value)
143{
144   union {
145      struct {
146         float x[16];
147         float y[16];
148      } a;
149      struct {
150         float  _0XOffset,  _1XOffset,  _2XOffset,  _3XOffset,
151                _4XOffset,  _5XOffset,  _6XOffset,  _7XOffset,
152                _8XOffset,  _9XOffset, _10XOffset, _11XOffset,
153               _12XOffset, _13XOffset, _14XOffset, _15XOffset;
154         float  _0YOffset,  _1YOffset,  _2YOffset,  _3YOffset,
155                _4YOffset,  _5YOffset,  _6YOffset,  _7YOffset,
156                _8YOffset,  _9YOffset, _10YOffset, _11YOffset,
157               _12YOffset, _13YOffset, _14YOffset, _15YOffset;
158      } v;
159   } u;
160   switch (sample_count) {
161   case 1:  INTEL_SAMPLE_POS_1X(u.v._);  break;
162   case 2:  INTEL_SAMPLE_POS_2X(u.v._);  break;
163   case 4:  INTEL_SAMPLE_POS_4X(u.v._);  break;
164   case 8:  INTEL_SAMPLE_POS_8X(u.v._);  break;
165   case 16: INTEL_SAMPLE_POS_16X(u.v._); break;
166   default: unreachable("invalid sample count");
167   }
168
169   out_value[0] = u.a.x[sample_index];
170   out_value[1] = u.a.y[sample_index];
171}
172
173static bool
174create_dirty_dmabuf_set(struct iris_context *ice)
175{
176   assert(ice->dirty_dmabufs == NULL);
177
178   ice->dirty_dmabufs = _mesa_pointer_set_create(ice);
179   return ice->dirty_dmabufs != NULL;
180}
181
182void
183iris_mark_dirty_dmabuf(struct iris_context *ice,
184                       struct pipe_resource *res)
185{
186   if (!_mesa_set_search(ice->dirty_dmabufs, res)) {
187      _mesa_set_add(ice->dirty_dmabufs, res);
188      pipe_reference(NULL, &res->reference);
189   }
190}
191
192static void
193clear_dirty_dmabuf_set(struct iris_context *ice)
194{
195   set_foreach(ice->dirty_dmabufs, entry) {
196      struct pipe_resource *res = (struct pipe_resource *)entry->key;
197      if (pipe_reference(&res->reference, NULL))
198         res->screen->resource_destroy(res->screen, res);
199   }
200
201   _mesa_set_clear(ice->dirty_dmabufs, NULL);
202}
203
204void
205iris_flush_dirty_dmabufs(struct iris_context *ice)
206{
207   set_foreach(ice->dirty_dmabufs, entry) {
208      struct pipe_resource *res = (struct pipe_resource *)entry->key;
209      ice->ctx.flush_resource(&ice->ctx, res);
210   }
211
212   clear_dirty_dmabuf_set(ice);
213}
214
215/**
216 * Destroy a context, freeing any associated memory.
217 */
218void
219iris_destroy_context(struct pipe_context *ctx)
220{
221   struct iris_context *ice = (struct iris_context *)ctx;
222   struct iris_screen *screen = (struct iris_screen *)ctx->screen;
223
224   if (ctx->stream_uploader)
225      u_upload_destroy(ctx->stream_uploader);
226   if (ctx->const_uploader)
227      u_upload_destroy(ctx->const_uploader);
228
229   clear_dirty_dmabuf_set(ice);
230
231   screen->vtbl.destroy_state(ice);
232
233   for (unsigned i = 0; i < ARRAY_SIZE(ice->shaders.scratch_surfs); i++)
234      pipe_resource_reference(&ice->shaders.scratch_surfs[i].res, NULL);
235
236   for (unsigned i = 0; i < ARRAY_SIZE(ice->shaders.scratch_bos); i++) {
237      for (unsigned j = 0; j < ARRAY_SIZE(ice->shaders.scratch_bos[i]); j++)
238         iris_bo_unreference(ice->shaders.scratch_bos[i][j]);
239   }
240
241   iris_destroy_program_cache(ice);
242   if (screen->measure.config)
243      iris_destroy_ctx_measure(ice);
244
245   u_upload_destroy(ice->state.surface_uploader);
246   u_upload_destroy(ice->state.bindless_uploader);
247   u_upload_destroy(ice->state.dynamic_uploader);
248   u_upload_destroy(ice->query_buffer_uploader);
249
250   iris_destroy_batches(ice);
251   iris_destroy_binder(&ice->state.binder);
252
253   iris_utrace_fini(ice);
254
255   slab_destroy_child(&ice->transfer_pool);
256   slab_destroy_child(&ice->transfer_pool_unsync);
257
258   ralloc_free(ice);
259}
260
261#define genX_call(devinfo, func, ...)             \
262   switch ((devinfo)->verx10) {                   \
263   case 125:                                      \
264      gfx125_##func(__VA_ARGS__);                 \
265      break;                                      \
266   case 120:                                      \
267      gfx12_##func(__VA_ARGS__);                  \
268      break;                                      \
269   case 110:                                      \
270      gfx11_##func(__VA_ARGS__);                  \
271      break;                                      \
272   case 90:                                       \
273      gfx9_##func(__VA_ARGS__);                   \
274      break;                                      \
275   case 80:                                       \
276      gfx8_##func(__VA_ARGS__);                   \
277      break;                                      \
278   default:                                       \
279      unreachable("Unknown hardware generation"); \
280   }
281
282/**
283 * Create a context.
284 *
285 * This is where each context begins.
286 */
287struct pipe_context *
288iris_create_context(struct pipe_screen *pscreen, void *priv, unsigned flags)
289{
290   struct iris_screen *screen = (struct iris_screen*)pscreen;
291   const struct intel_device_info *devinfo = &screen->devinfo;
292   struct iris_context *ice = rzalloc(NULL, struct iris_context);
293
294   if (!ice)
295      return NULL;
296
297   struct pipe_context *ctx = &ice->ctx;
298
299   ctx->screen = pscreen;
300   ctx->priv = priv;
301
302   ctx->stream_uploader = u_upload_create_default(ctx);
303   if (!ctx->stream_uploader) {
304      free(ctx);
305      return NULL;
306   }
307   ctx->const_uploader = u_upload_create(ctx, 1024 * 1024,
308                                         PIPE_BIND_CONSTANT_BUFFER,
309                                         PIPE_USAGE_IMMUTABLE,
310                                         IRIS_RESOURCE_FLAG_DEVICE_MEM);
311   if (!ctx->const_uploader) {
312      u_upload_destroy(ctx->stream_uploader);
313      free(ctx);
314      return NULL;
315   }
316
317   if (!create_dirty_dmabuf_set(ice)) {
318      ralloc_free(ice);
319      return NULL;
320   }
321
322   ctx->destroy = iris_destroy_context;
323   ctx->set_debug_callback = iris_set_debug_callback;
324   ctx->set_device_reset_callback = iris_set_device_reset_callback;
325   ctx->get_device_reset_status = iris_get_device_reset_status;
326   ctx->get_sample_position = iris_get_sample_position;
327
328   iris_init_context_fence_functions(ctx);
329   iris_init_blit_functions(ctx);
330   iris_init_clear_functions(ctx);
331   iris_init_program_functions(ctx);
332   iris_init_resource_functions(ctx);
333   iris_init_flush_functions(ctx);
334   iris_init_perfquery_functions(ctx);
335
336   iris_init_program_cache(ice);
337   iris_init_binder(ice);
338
339   slab_create_child(&ice->transfer_pool, &screen->transfer_pool);
340   slab_create_child(&ice->transfer_pool_unsync, &screen->transfer_pool);
341
342   ice->state.surface_uploader =
343      u_upload_create(ctx, 64 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
344                      IRIS_RESOURCE_FLAG_SURFACE_MEMZONE |
345                      IRIS_RESOURCE_FLAG_DEVICE_MEM);
346   ice->state.bindless_uploader =
347      u_upload_create(ctx, 64 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
348                      IRIS_RESOURCE_FLAG_BINDLESS_MEMZONE |
349                      IRIS_RESOURCE_FLAG_DEVICE_MEM);
350   ice->state.dynamic_uploader =
351      u_upload_create(ctx, 64 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
352                      IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE |
353                      IRIS_RESOURCE_FLAG_DEVICE_MEM);
354
355   ice->query_buffer_uploader =
356      u_upload_create(ctx, 16 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_STAGING,
357                      0);
358
359   genX_call(devinfo, init_state, ice);
360   genX_call(devinfo, init_blorp, ice);
361   genX_call(devinfo, init_query, ice);
362
363   int priority = 0;
364   if (flags & PIPE_CONTEXT_HIGH_PRIORITY)
365      priority = INTEL_CONTEXT_HIGH_PRIORITY;
366   if (flags & PIPE_CONTEXT_LOW_PRIORITY)
367      priority = INTEL_CONTEXT_LOW_PRIORITY;
368
369   if (INTEL_DEBUG(DEBUG_BATCH))
370      ice->state.sizes = _mesa_hash_table_u64_create(ice);
371
372   /* Do this before initializing the batches */
373   iris_utrace_init(ice);
374
375   iris_init_batches(ice, priority);
376
377   screen->vtbl.init_render_context(&ice->batches[IRIS_BATCH_RENDER]);
378   screen->vtbl.init_compute_context(&ice->batches[IRIS_BATCH_COMPUTE]);
379
380   if (!(flags & PIPE_CONTEXT_PREFER_THREADED))
381      return ctx;
382
383   /* Clover doesn't support u_threaded_context */
384   if (flags & PIPE_CONTEXT_COMPUTE_ONLY)
385      return ctx;
386
387   return threaded_context_create(ctx, &screen->transfer_pool,
388                                  iris_replace_buffer_storage,
389                                  NULL, /* TODO: asynchronous flushes? */
390                                  &ice->thrctx);
391}
392