1/*
2 * Copyright (c) 2017-2019 Lima Project
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sub license,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the
12 * next paragraph) shall be included in all copies or substantial portions
13 * of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 *
23 */
24
25#include "util/u_memory.h"
26#include "util/u_blitter.h"
27#include "util/format/u_format.h"
28#include "util/u_inlines.h"
29#include "util/u_math.h"
30#include "util/u_debug.h"
31#include "util/u_transfer.h"
32#include "util/u_surface.h"
33#include "util/u_transfer_helper.h"
34#include "util/hash_table.h"
35#include "util/ralloc.h"
36#include "util/u_drm.h"
37#include "renderonly/renderonly.h"
38
39#include "frontend/drm_driver.h"
40
41#include "drm-uapi/drm_fourcc.h"
42#include "drm-uapi/lima_drm.h"
43
44#include "lima_screen.h"
45#include "lima_context.h"
46#include "lima_resource.h"
47#include "lima_bo.h"
48#include "lima_util.h"
49#include "lima_blit.h"
50
51#include "pan_minmax_cache.h"
52#include "pan_tiling.h"
53
54static struct pipe_resource *
55lima_resource_create_scanout(struct pipe_screen *pscreen,
56                             const struct pipe_resource *templat,
57                             unsigned width, unsigned height)
58{
59   struct lima_screen *screen = lima_screen(pscreen);
60   struct renderonly_scanout *scanout;
61   struct winsys_handle handle;
62   struct pipe_resource *pres;
63
64   struct pipe_resource scanout_templat = *templat;
65   scanout_templat.width0 = width;
66   scanout_templat.height0 = height;
67   scanout_templat.screen = pscreen;
68
69   scanout = renderonly_scanout_for_resource(&scanout_templat,
70                                             screen->ro, &handle);
71   if (!scanout)
72      return NULL;
73
74   assert(handle.type == WINSYS_HANDLE_TYPE_FD);
75   pres = pscreen->resource_from_handle(pscreen, templat, &handle,
76                                        PIPE_HANDLE_USAGE_FRAMEBUFFER_WRITE);
77
78   close(handle.handle);
79   if (!pres) {
80      renderonly_scanout_destroy(scanout, screen->ro);
81      return NULL;
82   }
83
84   struct lima_resource *res = lima_resource(pres);
85   res->scanout = scanout;
86
87   return pres;
88}
89
90static uint32_t
91setup_miptree(struct lima_resource *res,
92              unsigned width0, unsigned height0,
93              bool align_to_tile)
94{
95   struct pipe_resource *pres = &res->base;
96   unsigned level;
97   unsigned width = width0;
98   unsigned height = height0;
99   unsigned depth = pres->depth0;
100   unsigned nr_samples = MAX2(pres->nr_samples, 1);
101   uint32_t size = 0;
102
103   for (level = 0; level <= pres->last_level; level++) {
104      uint32_t actual_level_size;
105      uint32_t stride;
106      unsigned aligned_width;
107      unsigned aligned_height;
108
109      if (align_to_tile) {
110         aligned_width = align(width, 16);
111         aligned_height = align(height, 16);
112      } else {
113         aligned_width = width;
114         aligned_height = height;
115      }
116
117      stride = util_format_get_stride(pres->format, aligned_width);
118      actual_level_size = stride *
119         util_format_get_nblocksy(pres->format, aligned_height) *
120         pres->array_size * depth;
121
122      res->levels[level].width = aligned_width;
123      res->levels[level].stride = stride;
124      res->levels[level].offset = size;
125      res->levels[level].layer_stride = util_format_get_stride(pres->format, align(width, 16)) * align(height, 16);
126
127      if (util_format_is_compressed(pres->format))
128         res->levels[level].layer_stride /= 4;
129
130      size += align(actual_level_size, 64);
131
132      width = u_minify(width, 1);
133      height = u_minify(height, 1);
134      depth = u_minify(depth, 1);
135   }
136
137   if (nr_samples > 1)
138      res->mrt_pitch = size;
139
140   size *= nr_samples;
141
142   return size;
143}
144
145static struct pipe_resource *
146lima_resource_create_bo(struct pipe_screen *pscreen,
147                        const struct pipe_resource *templat,
148                        unsigned width, unsigned height,
149                        bool align_to_tile)
150{
151   struct lima_screen *screen = lima_screen(pscreen);
152   struct lima_resource *res;
153   struct pipe_resource *pres;
154
155   res = CALLOC_STRUCT(lima_resource);
156   if (!res)
157      return NULL;
158
159   res->base = *templat;
160   res->base.screen = pscreen;
161   pipe_reference_init(&res->base.reference, 1);
162
163   pres = &res->base;
164
165   uint32_t size = setup_miptree(res, width, height, align_to_tile);
166   size = align(size, LIMA_PAGE_SIZE);
167
168   res->bo = lima_bo_create(screen, size, 0);
169   if (!res->bo) {
170      FREE(res);
171      return NULL;
172   }
173
174   return pres;
175}
176
177static struct pipe_resource *
178_lima_resource_create_with_modifiers(struct pipe_screen *pscreen,
179                                     const struct pipe_resource *templat,
180                                     const uint64_t *modifiers,
181                                     int count)
182{
183   struct lima_screen *screen = lima_screen(pscreen);
184   bool should_tile = lima_debug & LIMA_DEBUG_NO_TILING ? false : true;
185   unsigned width, height;
186   bool has_user_modifiers = true;
187   bool align_to_tile = false;
188
189   if (count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID)
190      has_user_modifiers = false;
191
192   /* VBOs/PBOs are untiled (and 1 height). */
193   if (templat->target == PIPE_BUFFER)
194      should_tile = false;
195
196   if (templat->bind & (PIPE_BIND_LINEAR | PIPE_BIND_SCANOUT))
197      should_tile = false;
198
199   /* If there's no user modifiers and buffer is shared we use linear */
200   if (!has_user_modifiers && (templat->bind & PIPE_BIND_SHARED))
201      should_tile = false;
202
203   if (has_user_modifiers &&
204      !drm_find_modifier(DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED,
205                         modifiers, count))
206      should_tile = false;
207
208   /* Don't align index, vertex or constant buffers */
209   if (templat->bind & (PIPE_BIND_INDEX_BUFFER |
210                        PIPE_BIND_VERTEX_BUFFER |
211                        PIPE_BIND_CONSTANT_BUFFER)) {
212      width = templat->width0;
213      height = templat->height0;
214   } else {
215      width = align(templat->width0, 16);
216      height = align(templat->height0, 16);
217      align_to_tile = true;
218   }
219
220   struct pipe_resource *pres;
221   if (screen->ro && (templat->bind & PIPE_BIND_SCANOUT))
222      pres = lima_resource_create_scanout(pscreen, templat, width, height);
223   else
224      pres = lima_resource_create_bo(pscreen, templat, width, height, align_to_tile);
225
226   if (pres) {
227      struct lima_resource *res = lima_resource(pres);
228      res->tiled = should_tile;
229
230      if (templat->bind & PIPE_BIND_INDEX_BUFFER)
231         res->index_cache = CALLOC_STRUCT(panfrost_minmax_cache);
232
233      debug_printf("%s: pres=%p width=%u height=%u depth=%u target=%d "
234                   "bind=%x usage=%d tile=%d last_level=%d\n", __func__,
235                   pres, pres->width0, pres->height0, pres->depth0,
236                   pres->target, pres->bind, pres->usage, should_tile, templat->last_level);
237   }
238   return pres;
239}
240
241static struct pipe_resource *
242lima_resource_create(struct pipe_screen *pscreen,
243                     const struct pipe_resource *templat)
244{
245   const uint64_t mod = DRM_FORMAT_MOD_INVALID;
246
247   return _lima_resource_create_with_modifiers(pscreen, templat, &mod, 1);
248}
249
250static struct pipe_resource *
251lima_resource_create_with_modifiers(struct pipe_screen *pscreen,
252                                    const struct pipe_resource *templat,
253                                    const uint64_t *modifiers,
254                                    int count)
255{
256   struct pipe_resource tmpl = *templat;
257
258   /* gbm_bo_create_with_modifiers & gbm_surface_create_with_modifiers
259    * don't have usage parameter, but buffer created by these functions
260    * may be used for scanout. So we assume buffer created by this
261    * function always enable scanout if linear modifier is permitted.
262    */
263   if (drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count))
264      tmpl.bind |= PIPE_BIND_SCANOUT;
265
266   return _lima_resource_create_with_modifiers(pscreen, &tmpl, modifiers, count);
267}
268
269static void
270lima_resource_destroy(struct pipe_screen *pscreen, struct pipe_resource *pres)
271{
272   struct lima_screen *screen = lima_screen(pscreen);
273   struct lima_resource *res = lima_resource(pres);
274
275   if (res->bo)
276      lima_bo_unreference(res->bo);
277
278   if (res->scanout)
279      renderonly_scanout_destroy(res->scanout, screen->ro);
280
281   if (res->damage.region)
282      FREE(res->damage.region);
283
284   if (res->index_cache)
285      FREE(res->index_cache);
286
287   FREE(res);
288}
289
290static struct pipe_resource *
291lima_resource_from_handle(struct pipe_screen *pscreen,
292        const struct pipe_resource *templat,
293        struct winsys_handle *handle, unsigned usage)
294{
295   if (templat->bind & (PIPE_BIND_SAMPLER_VIEW |
296                        PIPE_BIND_RENDER_TARGET |
297                        PIPE_BIND_DEPTH_STENCIL)) {
298      /* sampler hardware need offset alignment 64, while render hardware
299       * need offset alignment 8, but due to render target may be reloaded
300       * which uses the sampler, set alignment requrement to 64 for all
301       */
302      if (handle->offset & 0x3f) {
303         debug_error("import buffer offset not properly aligned\n");
304         return NULL;
305      }
306   }
307
308   struct lima_resource *res = CALLOC_STRUCT(lima_resource);
309   if (!res)
310      return NULL;
311
312   struct pipe_resource *pres = &res->base;
313   *pres = *templat;
314   pres->screen = pscreen;
315   pipe_reference_init(&pres->reference, 1);
316   res->levels[0].offset = handle->offset;
317   res->levels[0].stride = handle->stride;
318
319   struct lima_screen *screen = lima_screen(pscreen);
320   res->bo = lima_bo_import(screen, handle);
321   if (!res->bo) {
322      FREE(res);
323      return NULL;
324   }
325
326   res->modifier_constant = true;
327
328   switch (handle->modifier) {
329   case DRM_FORMAT_MOD_LINEAR:
330      res->tiled = false;
331      break;
332   case DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED:
333      res->tiled = true;
334      break;
335   case DRM_FORMAT_MOD_INVALID:
336      /* Modifier wasn't specified and it's shared buffer. We create these
337       * as linear, so disable tiling.
338       */
339      res->tiled = false;
340      break;
341   default:
342      fprintf(stderr, "Attempted to import unsupported modifier 0x%llx\n",
343                  (long long)handle->modifier);
344      goto err_out;
345   }
346
347   /* check alignment for the buffer */
348   if (res->tiled ||
349       (pres->bind & (PIPE_BIND_RENDER_TARGET | PIPE_BIND_DEPTH_STENCIL))) {
350      unsigned width, height, stride, size;
351
352      width = align(pres->width0, 16);
353      height = align(pres->height0, 16);
354      stride = util_format_get_stride(pres->format, width);
355      size = util_format_get_2d_size(pres->format, stride, height);
356
357      if (res->tiled && res->levels[0].stride != stride) {
358         fprintf(stderr, "tiled imported buffer has mismatching stride: %d (BO) != %d (expected)",
359                     res->levels[0].stride, stride);
360         goto err_out;
361      }
362
363      if (!res->tiled && (res->levels[0].stride % 8)) {
364         fprintf(stderr, "linear imported buffer stride is not aligned to 8 bytes: %d\n",
365                 res->levels[0].stride);
366      }
367
368      if (!res->tiled && res->levels[0].stride < stride) {
369         fprintf(stderr, "linear imported buffer stride is smaller than minimal: %d (BO) < %d (min)",
370                 res->levels[0].stride, stride);
371         goto err_out;
372      }
373
374      if ((res->bo->size - res->levels[0].offset) < size) {
375         fprintf(stderr, "imported bo size is smaller than expected: %d (BO) < %d (expected)\n",
376                 (res->bo->size - res->levels[0].offset), size);
377         goto err_out;
378      }
379
380      res->levels[0].width = width;
381   }
382   else
383      res->levels[0].width = pres->width0;
384
385   if (screen->ro) {
386      /* Make sure that renderonly has a handle to our buffer in the
387       * display's fd, so that a later renderonly_get_handle()
388       * returns correct handles or GEM names.
389       */
390      res->scanout =
391         renderonly_create_gpu_import_for_resource(pres,
392                                                   screen->ro,
393                                                   NULL);
394      /* ignore failiure to allow importing non-displayable buffer */
395   }
396
397   return pres;
398
399err_out:
400   lima_resource_destroy(pscreen, pres);
401   return NULL;
402}
403
404static bool
405lima_resource_get_handle(struct pipe_screen *pscreen,
406                         struct pipe_context *pctx,
407                         struct pipe_resource *pres,
408                         struct winsys_handle *handle, unsigned usage)
409{
410   struct lima_screen *screen = lima_screen(pscreen);
411   struct lima_resource *res = lima_resource(pres);
412
413   if (res->tiled)
414      handle->modifier = DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED;
415   else
416      handle->modifier = DRM_FORMAT_MOD_LINEAR;
417
418   res->modifier_constant = true;
419
420   if (handle->type == WINSYS_HANDLE_TYPE_KMS && screen->ro)
421      return renderonly_get_handle(res->scanout, handle);
422
423   if (!lima_bo_export(res->bo, handle))
424      return false;
425
426   handle->offset = res->levels[0].offset;
427   handle->stride = res->levels[0].stride;
428   return true;
429}
430
431static bool
432lima_resource_get_param(struct pipe_screen *pscreen,
433                        struct pipe_context *pctx,
434                        struct pipe_resource *pres,
435                        unsigned plane, unsigned layer, unsigned level,
436                        enum pipe_resource_param param,
437                        unsigned usage, uint64_t *value)
438{
439   struct lima_resource *res = lima_resource(pres);
440
441   switch (param) {
442   case PIPE_RESOURCE_PARAM_STRIDE:
443      *value = res->levels[level].stride;
444      return true;
445   case PIPE_RESOURCE_PARAM_OFFSET:
446      *value = res->levels[level].offset;
447      return true;
448   case PIPE_RESOURCE_PARAM_MODIFIER:
449      if (res->tiled)
450         *value = DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED;
451      else
452         *value = DRM_FORMAT_MOD_LINEAR;
453
454      return true;
455   default:
456      return false;
457   }
458}
459
460static void
461get_scissor_from_box(struct pipe_scissor_state *s,
462                     const struct pipe_box *b, int h)
463{
464   int y = h - (b->y + b->height);
465   /* region in tile unit */
466   s->minx = b->x >> 4;
467   s->miny = y >> 4;
468   s->maxx = (b->x + b->width + 0xf) >> 4;
469   s->maxy = (y + b->height + 0xf) >> 4;
470}
471
472static void
473get_damage_bound_box(struct pipe_resource *pres,
474                     const struct pipe_box *rects,
475                     unsigned int nrects,
476                     struct pipe_scissor_state *bound)
477{
478   struct pipe_box b = rects[0];
479
480   for (int i = 1; i < nrects; i++)
481      u_box_union_2d(&b, &b, rects + i);
482
483   int ret = u_box_clip_2d(&b, &b, pres->width0, pres->height0);
484   if (ret < 0)
485      memset(bound, 0, sizeof(*bound));
486   else
487      get_scissor_from_box(bound, &b, pres->height0);
488}
489
490static void
491lima_resource_set_damage_region(struct pipe_screen *pscreen,
492                                struct pipe_resource *pres,
493                                unsigned int nrects,
494                                const struct pipe_box *rects)
495{
496   struct lima_resource *res = lima_resource(pres);
497   struct lima_damage_region *damage = &res->damage;
498   int i;
499
500   if (damage->region) {
501      FREE(damage->region);
502      damage->region = NULL;
503      damage->num_region = 0;
504   }
505
506   if (!nrects)
507      return;
508
509   /* check full damage
510    *
511    * TODO: currently only check if there is any single damage
512    * region that can cover the full render target; there may
513    * be some accurate way, but a single window size damage
514    * region is most of the case from weston
515    */
516   for (i = 0; i < nrects; i++) {
517      if (rects[i].x <= 0 && rects[i].y <= 0 &&
518          rects[i].x + rects[i].width >= pres->width0 &&
519          rects[i].y + rects[i].height >= pres->height0)
520         return;
521   }
522
523   struct pipe_scissor_state *bound = &damage->bound;
524   get_damage_bound_box(pres, rects, nrects, bound);
525
526   damage->region = CALLOC(nrects, sizeof(*damage->region));
527   if (!damage->region)
528      return;
529
530   for (i = 0; i < nrects; i++)
531      get_scissor_from_box(damage->region + i, rects + i,
532                           pres->height0);
533
534   /* is region aligned to tiles? */
535   damage->aligned = true;
536   for (i = 0; i < nrects; i++) {
537      if (rects[i].x & 0xf || rects[i].y & 0xf ||
538          rects[i].width & 0xf || rects[i].height & 0xf) {
539         damage->aligned = false;
540         break;
541      }
542   }
543
544   damage->num_region = nrects;
545}
546
547static struct pipe_surface *
548lima_surface_create(struct pipe_context *pctx,
549                    struct pipe_resource *pres,
550                    const struct pipe_surface *surf_tmpl)
551{
552   struct lima_surface *surf = CALLOC_STRUCT(lima_surface);
553
554   if (!surf)
555      return NULL;
556
557   assert(surf_tmpl->u.tex.first_layer == surf_tmpl->u.tex.last_layer);
558
559   struct pipe_surface *psurf = &surf->base;
560   unsigned level = surf_tmpl->u.tex.level;
561
562   pipe_reference_init(&psurf->reference, 1);
563   pipe_resource_reference(&psurf->texture, pres);
564
565   psurf->context = pctx;
566   psurf->format = surf_tmpl->format;
567   psurf->width = u_minify(pres->width0, level);
568   psurf->height = u_minify(pres->height0, level);
569   psurf->nr_samples = surf_tmpl->nr_samples;
570   psurf->u.tex.level = level;
571   psurf->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
572   psurf->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
573
574   surf->tiled_w = align(psurf->width, 16) >> 4;
575   surf->tiled_h = align(psurf->height, 16) >> 4;
576
577   surf->reload = 0;
578   if (util_format_has_stencil(util_format_description(psurf->format)))
579      surf->reload |= PIPE_CLEAR_STENCIL;
580   if (util_format_has_depth(util_format_description(psurf->format)))
581      surf->reload |= PIPE_CLEAR_DEPTH;
582   if (!util_format_is_depth_or_stencil(psurf->format))
583      surf->reload |= PIPE_CLEAR_COLOR0;
584
585   return &surf->base;
586}
587
588static void
589lima_surface_destroy(struct pipe_context *pctx, struct pipe_surface *psurf)
590{
591   struct lima_surface *surf = lima_surface(psurf);
592
593   pipe_resource_reference(&psurf->texture, NULL);
594   FREE(surf);
595}
596
597static void *
598lima_transfer_map(struct pipe_context *pctx,
599                  struct pipe_resource *pres,
600                  unsigned level,
601                  unsigned usage,
602                  const struct pipe_box *box,
603                  struct pipe_transfer **pptrans)
604{
605   struct lima_screen *screen = lima_screen(pres->screen);
606   struct lima_context *ctx = lima_context(pctx);
607   struct lima_resource *res = lima_resource(pres);
608   struct lima_bo *bo = res->bo;
609   struct lima_transfer *trans;
610   struct pipe_transfer *ptrans;
611
612   /* No direct mappings of tiled, since we need to manually
613    * tile/untile.
614    */
615   if (res->tiled && (usage & PIPE_MAP_DIRECTLY))
616      return NULL;
617
618   /* bo might be in use in a previous stream draw. Allocate a new
619    * one for the resource to avoid overwriting data in use. */
620   if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
621      struct lima_bo *new_bo;
622      assert(res->bo && res->bo->size);
623
624      new_bo = lima_bo_create(screen, res->bo->size, res->bo->flags);
625      if (!new_bo)
626         return NULL;
627
628      lima_bo_unreference(res->bo);
629      res->bo = new_bo;
630
631      if (pres->bind & PIPE_BIND_VERTEX_BUFFER)
632         ctx->dirty |= LIMA_CONTEXT_DIRTY_VERTEX_BUFF;
633
634      bo = res->bo;
635   }
636   else if (!(usage & PIPE_MAP_UNSYNCHRONIZED) &&
637            (usage & PIPE_MAP_READ_WRITE)) {
638      /* use once buffers are made sure to not read/write overlapped
639       * range, so no need to sync */
640      lima_flush_job_accessing_bo(ctx, bo, usage & PIPE_MAP_WRITE);
641
642      unsigned op = usage & PIPE_MAP_WRITE ?
643         LIMA_GEM_WAIT_WRITE : LIMA_GEM_WAIT_READ;
644      lima_bo_wait(bo, op, PIPE_TIMEOUT_INFINITE);
645   }
646
647   if (!lima_bo_map(bo))
648      return NULL;
649
650   trans = slab_zalloc(&ctx->transfer_pool);
651   if (!trans)
652      return NULL;
653
654   ptrans = &trans->base;
655
656   pipe_resource_reference(&ptrans->resource, pres);
657   ptrans->level = level;
658   ptrans->usage = usage;
659   ptrans->box = *box;
660
661   *pptrans = ptrans;
662
663   if (res->tiled) {
664      ptrans->stride = util_format_get_stride(pres->format, ptrans->box.width);
665      ptrans->layer_stride = ptrans->stride * ptrans->box.height;
666
667      trans->staging = malloc(ptrans->stride * ptrans->box.height * ptrans->box.depth);
668
669      if (usage & PIPE_MAP_READ) {
670         unsigned line_stride = res->levels[level].stride;
671         unsigned row_height = util_format_is_compressed(pres->format) ? 4 : 16;
672         unsigned row_stride = line_stride * row_height;
673
674         unsigned i;
675         for (i = 0; i < ptrans->box.depth; i++)
676            panfrost_load_tiled_image(
677               trans->staging + i * ptrans->stride * ptrans->box.height,
678               bo->map + res->levels[level].offset + (i + box->z) * res->levels[level].layer_stride,
679               ptrans->box.x, ptrans->box.y,
680               ptrans->box.width, ptrans->box.height,
681               ptrans->stride,
682               row_stride,
683               pres->format);
684      }
685
686      return trans->staging;
687   } else {
688      unsigned dpw = PIPE_MAP_DIRECTLY | PIPE_MAP_WRITE |
689                     PIPE_MAP_PERSISTENT;
690      if ((usage & dpw) == dpw && res->index_cache)
691         return NULL;
692
693      ptrans->stride = res->levels[level].stride;
694      ptrans->layer_stride = res->levels[level].layer_stride;
695
696      if ((usage & PIPE_MAP_WRITE) && (usage & PIPE_MAP_DIRECTLY))
697         panfrost_minmax_cache_invalidate(res->index_cache, ptrans);
698
699      return bo->map + res->levels[level].offset +
700         box->z * res->levels[level].layer_stride +
701         box->y / util_format_get_blockheight(pres->format) * ptrans->stride +
702         box->x / util_format_get_blockwidth(pres->format) *
703         util_format_get_blocksize(pres->format);
704   }
705}
706
707static bool
708lima_should_convert_linear(struct lima_resource *res,
709                           struct pipe_transfer *ptrans)
710{
711   if (res->modifier_constant)
712          return false;
713
714   /* Overwriting the entire resource indicates streaming, for which
715    * linear layout is most efficient due to the lack of expensive
716    * conversion.
717    *
718    * For now we just switch to linear after a number of complete
719    * overwrites to keep things simple, but we could do better.
720    */
721
722   unsigned depth = res->base.target == PIPE_TEXTURE_3D ?
723                    res->base.depth0 : res->base.array_size;
724   bool entire_overwrite =
725          res->base.last_level == 0 &&
726          ptrans->box.width == res->base.width0 &&
727          ptrans->box.height == res->base.height0 &&
728          ptrans->box.depth == depth &&
729          ptrans->box.x == 0 &&
730          ptrans->box.y == 0 &&
731          ptrans->box.z == 0;
732
733   if (entire_overwrite)
734          ++res->full_updates;
735
736   return res->full_updates >= LAYOUT_CONVERT_THRESHOLD;
737}
738
739static void
740lima_transfer_flush_region(struct pipe_context *pctx,
741                           struct pipe_transfer *ptrans,
742                           const struct pipe_box *box)
743{
744   struct lima_context *ctx = lima_context(pctx);
745   struct lima_resource *res = lima_resource(ptrans->resource);
746   struct lima_transfer *trans = lima_transfer(ptrans);
747   struct lima_bo *bo = res->bo;
748   struct pipe_resource *pres;
749
750   if (trans->staging) {
751      pres = &res->base;
752      if (trans->base.usage & PIPE_MAP_WRITE) {
753         unsigned i;
754         if (lima_should_convert_linear(res, ptrans)) {
755            /* It's safe to re-use the same BO since tiled BO always has
756             * aligned dimensions */
757            for (i = 0; i < trans->base.box.depth; i++) {
758               util_copy_rect(bo->map + res->levels[0].offset +
759                                 (i + trans->base.box.z) * res->levels[0].stride,
760                              res->base.format,
761                              res->levels[0].stride,
762                              0, 0,
763                              ptrans->box.width,
764                              ptrans->box.height,
765                              trans->staging + i * ptrans->stride * ptrans->box.height,
766                              ptrans->stride,
767                              0, 0);
768            }
769            res->tiled = false;
770            res->modifier_constant = true;
771            /* Update texture descriptor */
772            ctx->dirty |= LIMA_CONTEXT_DIRTY_TEXTURES;
773         } else {
774            unsigned line_stride = res->levels[ptrans->level].stride;
775            unsigned row_height = util_format_is_compressed(pres->format) ? 4 : 16;
776            unsigned row_stride = line_stride * row_height;
777
778            for (i = 0; i < trans->base.box.depth; i++)
779               panfrost_store_tiled_image(
780                  bo->map + res->levels[trans->base.level].offset + (i + trans->base.box.z) * res->levels[trans->base.level].layer_stride,
781                  trans->staging + i * ptrans->stride * ptrans->box.height,
782                  ptrans->box.x, ptrans->box.y,
783                  ptrans->box.width, ptrans->box.height,
784                  row_stride,
785                  ptrans->stride,
786                  pres->format);
787         }
788      }
789   }
790}
791
792static void
793lima_transfer_unmap(struct pipe_context *pctx,
794                    struct pipe_transfer *ptrans)
795{
796   struct lima_context *ctx = lima_context(pctx);
797   struct lima_transfer *trans = lima_transfer(ptrans);
798   struct lima_resource *res = lima_resource(ptrans->resource);
799
800   struct pipe_box box;
801   u_box_2d(0, 0, ptrans->box.width, ptrans->box.height, &box);
802   lima_transfer_flush_region(pctx, ptrans, &box);
803   if (trans->staging)
804      free(trans->staging);
805   panfrost_minmax_cache_invalidate(res->index_cache, ptrans);
806
807   pipe_resource_reference(&ptrans->resource, NULL);
808   slab_free(&ctx->transfer_pool, trans);
809}
810
811static void
812lima_util_blitter_save_states(struct lima_context *ctx)
813{
814   util_blitter_save_blend(ctx->blitter, (void *)ctx->blend);
815   util_blitter_save_depth_stencil_alpha(ctx->blitter, (void *)ctx->zsa);
816   util_blitter_save_stencil_ref(ctx->blitter, &ctx->stencil_ref);
817   util_blitter_save_rasterizer(ctx->blitter, (void *)ctx->rasterizer);
818   util_blitter_save_fragment_shader(ctx->blitter, ctx->uncomp_fs);
819   util_blitter_save_vertex_shader(ctx->blitter, ctx->uncomp_vs);
820   util_blitter_save_viewport(ctx->blitter,
821                              &ctx->viewport.transform);
822   util_blitter_save_scissor(ctx->blitter, &ctx->scissor);
823   util_blitter_save_vertex_elements(ctx->blitter,
824                                     ctx->vertex_elements);
825   util_blitter_save_vertex_buffer_slot(ctx->blitter,
826                                        ctx->vertex_buffers.vb);
827
828   util_blitter_save_framebuffer(ctx->blitter, &ctx->framebuffer.base);
829
830   util_blitter_save_fragment_sampler_states(ctx->blitter,
831                                             ctx->tex_stateobj.num_samplers,
832                                             (void**)ctx->tex_stateobj.samplers);
833   util_blitter_save_fragment_sampler_views(ctx->blitter,
834                                            ctx->tex_stateobj.num_textures,
835                                            ctx->tex_stateobj.textures);
836}
837
838static void
839lima_blit(struct pipe_context *pctx, const struct pipe_blit_info *blit_info)
840{
841   struct lima_context *ctx = lima_context(pctx);
842   struct pipe_blit_info info = *blit_info;
843
844   if (lima_do_blit(pctx, blit_info)) {
845       return;
846   }
847
848   if (util_try_blit_via_copy_region(pctx, &info, false)) {
849      return; /* done */
850   }
851
852   if (info.mask & PIPE_MASK_S) {
853      debug_printf("lima: cannot blit stencil, skipping\n");
854      info.mask &= ~PIPE_MASK_S;
855   }
856
857   if (!util_blitter_is_blit_supported(ctx->blitter, &info)) {
858      debug_printf("lima: blit unsupported %s -> %s\n",
859                   util_format_short_name(info.src.resource->format),
860                   util_format_short_name(info.dst.resource->format));
861      return;
862   }
863
864   lima_util_blitter_save_states(ctx);
865
866   util_blitter_blit(ctx->blitter, &info);
867}
868
869static void
870lima_flush_resource(struct pipe_context *pctx, struct pipe_resource *resource)
871{
872
873}
874
875static void
876lima_texture_subdata(struct pipe_context *pctx,
877                     struct pipe_resource *prsc,
878                     unsigned level,
879                     unsigned usage,
880                     const struct pipe_box *box,
881                     const void *data,
882                     unsigned stride,
883                     unsigned layer_stride)
884{
885   struct lima_context *ctx = lima_context(pctx);
886   struct lima_resource *res = lima_resource(prsc);
887
888   if (!res->tiled) {
889      u_default_texture_subdata(pctx, prsc, level, usage, box,
890                                data, stride, layer_stride);
891      return;
892   }
893
894   assert(!(usage & PIPE_MAP_READ));
895
896   struct lima_transfer t = {
897      .base = {
898         .resource = prsc,
899         .usage = PIPE_MAP_WRITE,
900         .level = level,
901         .box = *box,
902         .stride = stride,
903         .layer_stride = layer_stride,
904      },
905      .staging = (void *)data,
906   };
907
908   lima_flush_job_accessing_bo(ctx, res->bo, true);
909   lima_bo_wait(res->bo, LIMA_GEM_WAIT_WRITE, PIPE_TIMEOUT_INFINITE);
910   if (!lima_bo_map(res->bo))
911      return;
912
913   struct pipe_box tbox;
914   u_box_2d(0, 0, t.base.box.width, t.base.box.height, &tbox);
915   lima_transfer_flush_region(pctx, &t.base, &tbox);
916}
917
918static const struct u_transfer_vtbl transfer_vtbl = {
919   .resource_create       = lima_resource_create,
920   .resource_destroy      = lima_resource_destroy,
921   .transfer_map          = lima_transfer_map,
922   .transfer_unmap        = lima_transfer_unmap,
923   .transfer_flush_region = lima_transfer_flush_region,
924};
925
926void
927lima_resource_screen_init(struct lima_screen *screen)
928{
929   screen->base.resource_create = lima_resource_create;
930   screen->base.resource_create_with_modifiers = lima_resource_create_with_modifiers;
931   screen->base.resource_from_handle = lima_resource_from_handle;
932   screen->base.resource_destroy = lima_resource_destroy;
933   screen->base.resource_get_handle = lima_resource_get_handle;
934   screen->base.resource_get_param = lima_resource_get_param;
935   screen->base.set_damage_region = lima_resource_set_damage_region;
936   screen->base.transfer_helper = u_transfer_helper_create(&transfer_vtbl,
937                                                           false, false,
938                                                           false, true,
939                                                           false);
940}
941
942void
943lima_resource_context_init(struct lima_context *ctx)
944{
945   ctx->base.create_surface = lima_surface_create;
946   ctx->base.surface_destroy = lima_surface_destroy;
947
948   ctx->base.buffer_subdata = u_default_buffer_subdata;
949   ctx->base.texture_subdata = lima_texture_subdata;
950   /* TODO: optimize resource_copy_region to do copy directly
951    * between 2 tiled or tiled and linear resources instead of
952    * using staging buffer.
953    */
954   ctx->base.resource_copy_region = util_resource_copy_region;
955
956   ctx->base.blit = lima_blit;
957
958   ctx->base.buffer_map = u_transfer_helper_transfer_map;
959   ctx->base.texture_map = u_transfer_helper_transfer_map;
960   ctx->base.transfer_flush_region = u_transfer_helper_transfer_flush_region;
961   ctx->base.buffer_unmap = u_transfer_helper_transfer_unmap;
962   ctx->base.texture_unmap = u_transfer_helper_transfer_unmap;
963
964   ctx->base.flush_resource = lima_flush_resource;
965}
966