1/*
2 * Copyright (c) 2012-2015 Etnaviv Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
13 * of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 *
23 * Authors:
24 *    Wladimir J. van der Laan <laanwj@gmail.com>
25 */
26
27#include "etnaviv_transfer.h"
28#include "etnaviv_clear_blit.h"
29#include "etnaviv_context.h"
30#include "etnaviv_debug.h"
31#include "etnaviv_etc2.h"
32#include "etnaviv_screen.h"
33
34#include "pipe/p_defines.h"
35#include "pipe/p_format.h"
36#include "pipe/p_screen.h"
37#include "pipe/p_state.h"
38#include "util/format/u_format.h"
39#include "util/u_inlines.h"
40#include "util/u_memory.h"
41#include "util/u_surface.h"
42#include "util/u_transfer.h"
43
44#include "hw/common_3d.xml.h"
45
46#include "drm-uapi/drm_fourcc.h"
47
48/* Compute offset into a 1D/2D/3D buffer of a certain box.
49 * This box must be aligned to the block width and height of the
50 * underlying format. */
51static inline size_t
52etna_compute_offset(enum pipe_format format, const struct pipe_box *box,
53                    size_t stride, size_t layer_stride)
54{
55   return box->z * layer_stride +
56          box->y / util_format_get_blockheight(format) * stride +
57          box->x / util_format_get_blockwidth(format) *
58             util_format_get_blocksize(format);
59}
60
61static void etna_patch_data(void *buffer, const struct pipe_transfer *ptrans)
62{
63   struct pipe_resource *prsc = ptrans->resource;
64   struct etna_resource *rsc = etna_resource(prsc);
65   struct etna_resource_level *level = &rsc->levels[ptrans->level];
66
67   if (likely(!etna_etc2_needs_patching(prsc)))
68      return;
69
70   if (level->patched)
71      return;
72
73   /* do have the offsets of blocks to patch? */
74   if (!level->patch_offsets) {
75      level->patch_offsets = CALLOC_STRUCT(util_dynarray);
76
77      etna_etc2_calculate_blocks(buffer, ptrans->stride,
78                                         ptrans->box.width, ptrans->box.height,
79                                         prsc->format, level->patch_offsets);
80   }
81
82   etna_etc2_patch(buffer, level->patch_offsets);
83
84   level->patched = true;
85}
86
87static void etna_unpatch_data(void *buffer, const struct pipe_transfer *ptrans)
88{
89   struct pipe_resource *prsc = ptrans->resource;
90   struct etna_resource *rsc = etna_resource(prsc);
91   struct etna_resource_level *level = &rsc->levels[ptrans->level];
92
93   if (!level->patched)
94      return;
95
96   etna_etc2_patch(buffer, level->patch_offsets);
97
98   level->patched = false;
99}
100
101static void
102etna_transfer_unmap(struct pipe_context *pctx, struct pipe_transfer *ptrans)
103{
104   struct etna_context *ctx = etna_context(pctx);
105   struct etna_transfer *trans = etna_transfer(ptrans);
106   struct etna_resource *rsc = etna_resource(ptrans->resource);
107
108   /* XXX
109    * When writing to a resource that is already in use, replace the resource
110    * with a completely new buffer
111    * and free the old one using a fenced free.
112    * The most tricky case to implement will be: tiled or supertiled surface,
113    * partial write, target not aligned to 4/64. */
114   assert(ptrans->level <= rsc->base.last_level);
115
116   if (rsc->texture && !etna_resource_newer(rsc, etna_resource(rsc->texture)))
117      rsc = etna_resource(rsc->texture); /* switch to using the texture resource */
118
119   /*
120    * Temporary resources are always pulled into the CPU domain, must push them
121    * back into GPU domain before the RS execs the blit to the base resource.
122    */
123   if (trans->rsc)
124      etna_bo_cpu_fini(etna_resource(trans->rsc)->bo);
125
126   if (ptrans->usage & PIPE_MAP_WRITE) {
127      if (trans->rsc) {
128         /* We have a temporary resource due to either tile status or
129          * tiling format. Write back the updated buffer contents.
130          * FIXME: we need to invalidate the tile status. */
131         etna_copy_resource_box(pctx, ptrans->resource, trans->rsc, ptrans->level, &ptrans->box);
132      } else if (trans->staging) {
133         /* map buffer object */
134         struct etna_resource_level *res_level = &rsc->levels[ptrans->level];
135
136         if (rsc->layout == ETNA_LAYOUT_TILED) {
137            for (unsigned z = 0; z < ptrans->box.depth; z++) {
138               etna_texture_tile(
139                  trans->mapped + (ptrans->box.z + z) * res_level->layer_stride,
140                  trans->staging + z * ptrans->layer_stride,
141                  ptrans->box.x, ptrans->box.y,
142                  res_level->stride, ptrans->box.width, ptrans->box.height,
143                  ptrans->stride, util_format_get_blocksize(rsc->base.format));
144            }
145         } else if (rsc->layout == ETNA_LAYOUT_LINEAR) {
146            util_copy_box(trans->mapped, rsc->base.format, res_level->stride,
147                          res_level->layer_stride, ptrans->box.x,
148                          ptrans->box.y, ptrans->box.z, ptrans->box.width,
149                          ptrans->box.height, ptrans->box.depth,
150                          trans->staging, ptrans->stride,
151                          ptrans->layer_stride, 0, 0, 0 /* src x,y,z */);
152         } else {
153            BUG("unsupported tiling %i", rsc->layout);
154         }
155
156         FREE(trans->staging);
157      }
158
159      rsc->seqno++;
160
161      if (rsc->base.bind & PIPE_BIND_SAMPLER_VIEW) {
162         ctx->dirty |= ETNA_DIRTY_TEXTURE_CACHES;
163      }
164   }
165
166   /* We need to have the patched data ready for the GPU. */
167   etna_patch_data(trans->mapped, ptrans);
168
169   /*
170    * Transfers without a temporary are only pulled into the CPU domain if they
171    * are not mapped unsynchronized. If they are, must push them back into GPU
172    * domain after CPU access is finished.
173    */
174   if (!trans->rsc && !(ptrans->usage & PIPE_MAP_UNSYNCHRONIZED))
175      etna_bo_cpu_fini(rsc->bo);
176
177   if ((ptrans->resource->target == PIPE_BUFFER) &&
178       (ptrans->usage & PIPE_MAP_WRITE)) {
179      util_range_add(&rsc->base,
180                     &rsc->valid_buffer_range,
181                     ptrans->box.x,
182                     ptrans->box.x + ptrans->box.width);
183      }
184
185   pipe_resource_reference(&trans->rsc, NULL);
186   pipe_resource_reference(&ptrans->resource, NULL);
187   slab_free(&ctx->transfer_pool, trans);
188}
189
190static void *
191etna_transfer_map(struct pipe_context *pctx, struct pipe_resource *prsc,
192                  unsigned level,
193                  unsigned usage,
194                  const struct pipe_box *box,
195                  struct pipe_transfer **out_transfer)
196{
197   struct etna_context *ctx = etna_context(pctx);
198   struct etna_screen *screen = ctx->screen;
199   struct etna_resource *rsc = etna_resource(prsc);
200   struct etna_transfer *trans;
201   struct pipe_transfer *ptrans;
202   enum pipe_format format = prsc->format;
203
204   trans = slab_zalloc(&ctx->transfer_pool);
205   if (!trans)
206      return NULL;
207
208   /*
209    * Upgrade to UNSYNCHRONIZED if target is PIPE_BUFFER and range is uninitialized.
210    */
211   if ((usage & PIPE_MAP_WRITE) &&
212       (prsc->target == PIPE_BUFFER) &&
213       !util_ranges_intersect(&rsc->valid_buffer_range,
214                              box->x,
215                              box->x + box->width)) {
216      usage |= PIPE_MAP_UNSYNCHRONIZED;
217   }
218
219   /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
220    * being mapped. If we add buffer reallocation to avoid CPU/GPU sync this
221    * check needs to be extended to coherent mappings and shared resources.
222    */
223   if ((usage & PIPE_MAP_DISCARD_RANGE) &&
224       !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
225       prsc->last_level == 0 &&
226       prsc->width0 == box->width &&
227       prsc->height0 == box->height &&
228       prsc->depth0 == box->depth &&
229       prsc->array_size == 1) {
230      usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
231   }
232
233   ptrans = &trans->base;
234   pipe_resource_reference(&ptrans->resource, prsc);
235   ptrans->level = level;
236   ptrans->usage = usage;
237   ptrans->box = *box;
238
239   assert(level <= prsc->last_level);
240
241   /* This one is a little tricky: if we have a separate render resource, which
242    * is newer than the base resource we want the transfer to target this one,
243    * to get the most up-to-date content, but only if we don't have a texture
244    * target of the same age, as transfering in/out of the texture target is
245    * generally preferred for the reasons listed below */
246   if (rsc->render && etna_resource_newer(etna_resource(rsc->render), rsc) &&
247       (!rsc->texture || etna_resource_newer(etna_resource(rsc->render),
248                                             etna_resource(rsc->texture)))) {
249      rsc = etna_resource(rsc->render);
250   }
251
252   if (rsc->texture && !etna_resource_newer(rsc, etna_resource(rsc->texture))) {
253      /* We have a texture resource which is the same age or newer than the
254       * render resource. Use the texture resource, which avoids bouncing
255       * pixels between the two resources, and we can de-tile it in s/w. */
256      rsc = etna_resource(rsc->texture);
257   } else if (rsc->ts_bo ||
258              (rsc->layout != ETNA_LAYOUT_LINEAR &&
259               etna_resource_hw_tileable(screen->specs.use_blt, prsc) &&
260               /* HALIGN 4 resources are incompatible with the resolve engine,
261                * so fall back to using software to detile this resource. */
262               rsc->halign != TEXTURE_HALIGN_FOUR)) {
263      /* If the surface has tile status, we need to resolve it first.
264       * The strategy we implement here is to use the RS to copy the
265       * depth buffer, filling in the "holes" where the tile status
266       * indicates that it's clear. We also do this for tiled
267       * resources, but only if the RS can blit them. */
268      if (usage & PIPE_MAP_DIRECTLY) {
269         slab_free(&ctx->transfer_pool, trans);
270         BUG("unsupported map flags %#x with tile status/tiled layout", usage);
271         return NULL;
272      }
273
274      if (prsc->depth0 > 1 && rsc->ts_bo) {
275         slab_free(&ctx->transfer_pool, trans);
276         BUG("resource has depth >1 with tile status");
277         return NULL;
278      }
279
280      struct pipe_resource templ = *prsc;
281      templ.nr_samples = 0;
282      templ.bind = PIPE_BIND_RENDER_TARGET;
283
284      trans->rsc = etna_resource_alloc(pctx->screen, ETNA_LAYOUT_LINEAR,
285                                       DRM_FORMAT_MOD_LINEAR, &templ);
286      if (!trans->rsc) {
287         slab_free(&ctx->transfer_pool, trans);
288         return NULL;
289      }
290
291      if (!screen->specs.use_blt) {
292         /* Need to align the transfer region to satisfy RS restrictions, as we
293          * really want to hit the RS blit path here.
294          */
295         unsigned w_align, h_align;
296
297         if (rsc->layout & ETNA_LAYOUT_BIT_SUPER) {
298            w_align = 64;
299            h_align = 64 * ctx->screen->specs.pixel_pipes;
300         } else {
301            w_align = ETNA_RS_WIDTH_MASK + 1;
302            h_align = ETNA_RS_HEIGHT_MASK + 1;
303         }
304
305         ptrans->box.width += ptrans->box.x & (w_align - 1);
306         ptrans->box.x = ptrans->box.x & ~(w_align - 1);
307         ptrans->box.width = align(ptrans->box.width, (ETNA_RS_WIDTH_MASK + 1));
308         ptrans->box.height += ptrans->box.y & (h_align - 1);
309         ptrans->box.y = ptrans->box.y & ~(h_align - 1);
310         ptrans->box.height = align(ptrans->box.height, ETNA_RS_HEIGHT_MASK + 1);
311      }
312
313      if (!(usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE))
314         etna_copy_resource_box(pctx, trans->rsc, &rsc->base, level, &ptrans->box);
315
316      /* Switch to using the temporary resource instead */
317      rsc = etna_resource(trans->rsc);
318   }
319
320   struct etna_resource_level *res_level = &rsc->levels[level];
321
322   /* XXX we don't handle PIPE_MAP_FLUSH_EXPLICIT; this flag can be ignored
323    * when mapping in-place,
324    * but when not in place we need to fire off the copy operation in
325    * transfer_flush_region (currently
326    * a no-op) instead of unmap. Need to handle this to support
327    * ARB_map_buffer_range extension at least.
328    */
329   /* XXX we don't take care of current operations on the resource; which can
330      be, at some point in the pipeline
331      which is not yet executed:
332
333      - bound as surface
334      - bound through vertex buffer
335      - bound through index buffer
336      - bound in sampler view
337      - used in clear_render_target / clear_depth_stencil operation
338      - used in blit
339      - used in resource_copy_region
340
341      How do other drivers record this information over course of the rendering
342      pipeline?
343      Is it necessary at all? Only in case we want to provide a fast path and
344      map the resource directly
345      (and for PIPE_MAP_DIRECTLY) and we don't want to force a sync.
346      We also need to know whether the resource is in use to determine if a sync
347      is needed (or just do it
348      always, but that comes at the expense of performance).
349
350      A conservative approximation without too much overhead would be to mark
351      all resources that have
352      been bound at some point as busy. A drawback would be that accessing
353      resources that have
354      been bound but are no longer in use for a while still carry a performance
355      penalty. On the other hand,
356      the program could be using PIPE_MAP_DISCARD_WHOLE_RESOURCE or
357      PIPE_MAP_UNSYNCHRONIZED to
358      avoid this in the first place...
359
360      A) We use an in-pipe copy engine, and queue the copy operation after unmap
361      so that the copy
362         will be performed when all current commands have been executed.
363         Using the RS is possible, not sure if always efficient. This can also
364      do any kind of tiling for us.
365         Only possible when PIPE_MAP_DISCARD_RANGE is set.
366      B) We discard the entire resource (or at least, the mipmap level) and
367      allocate new memory for it.
368         Only possible when mapping the entire resource or
369      PIPE_MAP_DISCARD_WHOLE_RESOURCE is set.
370    */
371
372   /*
373    * Pull resources into the CPU domain. Only skipped for unsynchronized
374    * transfers without a temporary resource.
375    */
376   if (trans->rsc || !(usage & PIPE_MAP_UNSYNCHRONIZED)) {
377      enum etna_resource_status status = etna_resource_status(ctx, rsc);
378      uint32_t prep_flags = 0;
379
380      /*
381       * Always flush if we have the temporary resource and have a copy to this
382       * outstanding. Otherwise infer flush requirement from resource access and
383       * current GPU usage (reads must wait for GPU writes, writes must have
384       * exclusive access to the buffer).
385       */
386      if ((trans->rsc && (status & ETNA_PENDING_WRITE)) ||
387          (!trans->rsc &&
388           (((usage & PIPE_MAP_READ) && (status & ETNA_PENDING_WRITE)) ||
389           ((usage & PIPE_MAP_WRITE) && status)))) {
390         pctx->flush(pctx, NULL, 0);
391      }
392
393      if (usage & PIPE_MAP_READ)
394         prep_flags |= DRM_ETNA_PREP_READ;
395      if (usage & PIPE_MAP_WRITE)
396         prep_flags |= DRM_ETNA_PREP_WRITE;
397
398      /*
399       * The ETC2 patching operates in-place on the resource, so the resource will
400       * get written even on read-only transfers. This blocks the GPU to sample
401       * from this resource.
402       */
403      if ((usage & PIPE_MAP_READ) && etna_etc2_needs_patching(prsc))
404         prep_flags |= DRM_ETNA_PREP_WRITE;
405
406      if (etna_bo_cpu_prep(rsc->bo, prep_flags))
407         goto fail_prep;
408   }
409
410   /* map buffer object */
411   trans->mapped = etna_bo_map(rsc->bo);
412   if (!trans->mapped)
413      goto fail;
414
415   *out_transfer = ptrans;
416
417   if (rsc->layout == ETNA_LAYOUT_LINEAR) {
418      ptrans->stride = res_level->stride;
419      ptrans->layer_stride = res_level->layer_stride;
420
421      trans->mapped += res_level->offset +
422             etna_compute_offset(prsc->format, box, res_level->stride,
423                                 res_level->layer_stride);
424
425      /* We need to have the unpatched data ready for the gfx stack. */
426      if (usage & PIPE_MAP_READ)
427         etna_unpatch_data(trans->mapped, ptrans);
428
429      return trans->mapped;
430   } else {
431      unsigned divSizeX = util_format_get_blockwidth(format);
432      unsigned divSizeY = util_format_get_blockheight(format);
433
434      /* No direct mappings of tiled, since we need to manually
435       * tile/untile.
436       */
437      if (usage & PIPE_MAP_DIRECTLY)
438         goto fail;
439
440      trans->mapped += res_level->offset;
441      ptrans->stride = align(box->width, divSizeX) * util_format_get_blocksize(format); /* row stride in bytes */
442      ptrans->layer_stride = align(box->height, divSizeY) * ptrans->stride;
443      size_t size = ptrans->layer_stride * box->depth;
444
445      trans->staging = MALLOC(size);
446      if (!trans->staging)
447         goto fail;
448
449      if (usage & PIPE_MAP_READ) {
450         if (rsc->layout == ETNA_LAYOUT_TILED) {
451            for (unsigned z = 0; z < ptrans->box.depth; z++) {
452               etna_texture_untile(trans->staging + z * ptrans->layer_stride,
453                                   trans->mapped + (ptrans->box.z + z) * res_level->layer_stride,
454                                   ptrans->box.x, ptrans->box.y, res_level->stride,
455                                   ptrans->box.width, ptrans->box.height, ptrans->stride,
456                                   util_format_get_blocksize(rsc->base.format));
457            }
458         } else if (rsc->layout == ETNA_LAYOUT_LINEAR) {
459            util_copy_box(trans->staging, rsc->base.format, ptrans->stride,
460                          ptrans->layer_stride, 0, 0, 0, /* dst x,y,z */
461                          ptrans->box.width, ptrans->box.height,
462                          ptrans->box.depth, trans->mapped, res_level->stride,
463                          res_level->layer_stride, ptrans->box.x,
464                          ptrans->box.y, ptrans->box.z);
465         } else {
466            /* TODO supertiling */
467            BUG("unsupported tiling %i for reading", rsc->layout);
468         }
469      }
470
471      return trans->staging;
472   }
473
474fail:
475   etna_bo_cpu_fini(rsc->bo);
476fail_prep:
477   etna_transfer_unmap(pctx, ptrans);
478   return NULL;
479}
480
481static void
482etna_transfer_flush_region(struct pipe_context *pctx,
483                           struct pipe_transfer *ptrans,
484                           const struct pipe_box *box)
485{
486   struct etna_resource *rsc = etna_resource(ptrans->resource);
487
488   if (ptrans->resource->target == PIPE_BUFFER)
489      util_range_add(&rsc->base,
490                     &rsc->valid_buffer_range,
491                     ptrans->box.x + box->x,
492                     ptrans->box.x + box->x + box->width);
493}
494
495void
496etna_transfer_init(struct pipe_context *pctx)
497{
498   pctx->buffer_map = etna_transfer_map;
499   pctx->texture_map = etna_transfer_map;
500   pctx->transfer_flush_region = etna_transfer_flush_region;
501   pctx->buffer_unmap = etna_transfer_unmap;
502   pctx->texture_unmap = etna_transfer_unmap;
503   pctx->buffer_subdata = u_default_buffer_subdata;
504   pctx->texture_subdata = u_default_texture_subdata;
505}
506