1/*
2 * Copyright © 2014-2017 Broadcom
3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24
25#include "pipe/p_defines.h"
26#include "util/u_memory.h"
27#include "util/format/u_format.h"
28#include "util/u_inlines.h"
29#include "util/u_surface.h"
30#include "util/u_transfer_helper.h"
31#include "util/u_upload_mgr.h"
32#include "util/format/u_format_zs.h"
33#include "util/u_drm.h"
34
35#include "drm-uapi/drm_fourcc.h"
36#include "v3d_screen.h"
37#include "v3d_context.h"
38#include "v3d_resource.h"
39#include "broadcom/cle/v3d_packet_v33_pack.h"
40
41static void
42v3d_debug_resource_layout(struct v3d_resource *rsc, const char *caller)
43{
44        if (!(unlikely(V3D_DEBUG & V3D_DEBUG_SURFACE)))
45                return;
46
47        struct pipe_resource *prsc = &rsc->base;
48
49        if (prsc->target == PIPE_BUFFER) {
50                fprintf(stderr,
51                        "rsc %s %p (format %s), %dx%d buffer @0x%08x-0x%08x\n",
52                        caller, rsc,
53                        util_format_short_name(prsc->format),
54                        prsc->width0, prsc->height0,
55                        rsc->bo->offset,
56                        rsc->bo->offset + rsc->bo->size - 1);
57                return;
58        }
59
60        static const char *const tiling_descriptions[] = {
61                [V3D_TILING_RASTER] = "R",
62                [V3D_TILING_LINEARTILE] = "LT",
63                [V3D_TILING_UBLINEAR_1_COLUMN] = "UB1",
64                [V3D_TILING_UBLINEAR_2_COLUMN] = "UB2",
65                [V3D_TILING_UIF_NO_XOR] = "UIF",
66                [V3D_TILING_UIF_XOR] = "UIF^",
67        };
68
69        for (int i = 0; i <= prsc->last_level; i++) {
70                struct v3d_resource_slice *slice = &rsc->slices[i];
71
72                int level_width = slice->stride / rsc->cpp;
73                int level_height = slice->padded_height;
74                int level_depth =
75                        u_minify(util_next_power_of_two(prsc->depth0), i);
76
77                fprintf(stderr,
78                        "rsc %s %p (format %s), %dx%d: "
79                        "level %d (%s) %dx%dx%d -> %dx%dx%d, stride %d@0x%08x\n",
80                        caller, rsc,
81                        util_format_short_name(prsc->format),
82                        prsc->width0, prsc->height0,
83                        i, tiling_descriptions[slice->tiling],
84                        u_minify(prsc->width0, i),
85                        u_minify(prsc->height0, i),
86                        u_minify(prsc->depth0, i),
87                        level_width,
88                        level_height,
89                        level_depth,
90                        slice->stride,
91                        rsc->bo->offset + slice->offset);
92        }
93}
94
95static bool
96v3d_resource_bo_alloc(struct v3d_resource *rsc)
97{
98        struct pipe_resource *prsc = &rsc->base;
99        struct pipe_screen *pscreen = prsc->screen;
100        struct v3d_bo *bo;
101
102        bo = v3d_bo_alloc(v3d_screen(pscreen), rsc->size, "resource");
103        if (bo) {
104                v3d_bo_unreference(&rsc->bo);
105                rsc->bo = bo;
106                rsc->serial_id++;
107                v3d_debug_resource_layout(rsc, "alloc");
108                return true;
109        } else {
110                return false;
111        }
112}
113
114static void
115v3d_resource_transfer_unmap(struct pipe_context *pctx,
116                            struct pipe_transfer *ptrans)
117{
118        struct v3d_context *v3d = v3d_context(pctx);
119        struct v3d_transfer *trans = v3d_transfer(ptrans);
120
121        if (trans->map) {
122                struct v3d_resource *rsc = v3d_resource(ptrans->resource);
123                struct v3d_resource_slice *slice = &rsc->slices[ptrans->level];
124
125                if (ptrans->usage & PIPE_MAP_WRITE) {
126                        for (int z = 0; z < ptrans->box.depth; z++) {
127                                void *dst = rsc->bo->map +
128                                        v3d_layer_offset(&rsc->base,
129                                                         ptrans->level,
130                                                         ptrans->box.z + z);
131                                v3d_store_tiled_image(dst,
132                                                      slice->stride,
133                                                      (trans->map +
134                                                       ptrans->stride *
135                                                       ptrans->box.height * z),
136                                                      ptrans->stride,
137                                                      slice->tiling, rsc->cpp,
138                                                      slice->padded_height,
139                                                      &ptrans->box);
140                        }
141                }
142                free(trans->map);
143        }
144
145        pipe_resource_reference(&ptrans->resource, NULL);
146        slab_free(&v3d->transfer_pool, ptrans);
147}
148
149static void
150rebind_sampler_views(struct v3d_context *v3d,
151                     struct v3d_resource *rsc)
152{
153        for (int st = 0; st < PIPE_SHADER_TYPES; st++) {
154                struct v3d_texture_stateobj *tex = v3d->tex + st;
155
156                for (unsigned i = 0; i < tex->num_textures; i++) {
157                        struct pipe_sampler_view *psview = tex->textures[i];
158
159                        if (psview->texture != &rsc->base)
160                                continue;
161
162                        struct v3d_sampler_view *sview =
163                                v3d_sampler_view(psview);
164
165                        v3d_create_texture_shader_state_bo(v3d, sview);
166
167                        v3d_flag_dirty_sampler_state(v3d, st);
168                }
169        }
170}
171
172static void
173v3d_map_usage_prep(struct pipe_context *pctx,
174                   struct pipe_resource *prsc,
175                   unsigned usage)
176{
177        struct v3d_context *v3d = v3d_context(pctx);
178        struct v3d_resource *rsc = v3d_resource(prsc);
179
180        if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
181                if (v3d_resource_bo_alloc(rsc)) {
182                        /* If it might be bound as one of our vertex buffers
183                         * or UBOs, make sure we re-emit vertex buffer state
184                         * or uniforms.
185                         */
186                        if (prsc->bind & PIPE_BIND_VERTEX_BUFFER)
187                                v3d->dirty |= V3D_DIRTY_VTXBUF;
188                        if (prsc->bind & PIPE_BIND_CONSTANT_BUFFER)
189                                v3d->dirty |= V3D_DIRTY_CONSTBUF;
190                        /* Since we are changing the texture BO we need to
191                         * update any bound samplers to point to the new
192                         * BO. Notice we can have samplers that are not
193                         * currently bound to the state that won't be
194                         * updated. These will be fixed when they are bound in
195                         * v3d_set_sampler_views.
196                         */
197                        if (prsc->bind & PIPE_BIND_SAMPLER_VIEW)
198                                rebind_sampler_views(v3d, rsc);
199                } else {
200                        /* If we failed to reallocate, flush users so that we
201                         * don't violate any syncing requirements.
202                         */
203                        v3d_flush_jobs_reading_resource(v3d, prsc,
204                                                        V3D_FLUSH_DEFAULT,
205                                                        false);
206                }
207        } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
208                /* If we're writing and the buffer is being used by the CL, we
209                 * have to flush the CL first.  If we're only reading, we need
210                 * to flush if the CL has written our buffer.
211                 */
212                if (usage & PIPE_MAP_WRITE) {
213                        v3d_flush_jobs_reading_resource(v3d, prsc,
214                                                        V3D_FLUSH_ALWAYS,
215                                                        false);
216                } else {
217                        v3d_flush_jobs_writing_resource(v3d, prsc,
218                                                        V3D_FLUSH_ALWAYS,
219                                                        false);
220                }
221        }
222
223        if (usage & PIPE_MAP_WRITE) {
224                rsc->writes++;
225                rsc->initialized_buffers = ~0;
226        }
227}
228
229static void *
230v3d_resource_transfer_map(struct pipe_context *pctx,
231                          struct pipe_resource *prsc,
232                          unsigned level, unsigned usage,
233                          const struct pipe_box *box,
234                          struct pipe_transfer **pptrans)
235{
236        struct v3d_context *v3d = v3d_context(pctx);
237        struct v3d_resource *rsc = v3d_resource(prsc);
238        struct v3d_transfer *trans;
239        struct pipe_transfer *ptrans;
240        enum pipe_format format = prsc->format;
241        char *buf;
242
243        /* MSAA maps should have been handled by u_transfer_helper. */
244        assert(prsc->nr_samples <= 1);
245
246        /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
247         * being mapped.
248         */
249        if ((usage & PIPE_MAP_DISCARD_RANGE) &&
250            !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
251            !(prsc->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
252            prsc->last_level == 0 &&
253            prsc->width0 == box->width &&
254            prsc->height0 == box->height &&
255            prsc->depth0 == box->depth &&
256            prsc->array_size == 1 &&
257            rsc->bo->private) {
258                usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
259        }
260
261        v3d_map_usage_prep(pctx, prsc, usage);
262
263        trans = slab_zalloc(&v3d->transfer_pool);
264        if (!trans)
265                return NULL;
266
267        /* XXX: Handle DONTBLOCK, DISCARD_RANGE, PERSISTENT, COHERENT. */
268
269        ptrans = &trans->base;
270
271        pipe_resource_reference(&ptrans->resource, prsc);
272        ptrans->level = level;
273        ptrans->usage = usage;
274        ptrans->box = *box;
275
276        /* Note that the current kernel implementation is synchronous, so no
277         * need to do syncing stuff here yet.
278         */
279
280        if (usage & PIPE_MAP_UNSYNCHRONIZED)
281                buf = v3d_bo_map_unsynchronized(rsc->bo);
282        else
283                buf = v3d_bo_map(rsc->bo);
284        if (!buf) {
285                fprintf(stderr, "Failed to map bo\n");
286                goto fail;
287        }
288
289        *pptrans = ptrans;
290
291        /* Our load/store routines work on entire compressed blocks. */
292        u_box_pixels_to_blocks(&ptrans->box, &ptrans->box, format);
293
294        struct v3d_resource_slice *slice = &rsc->slices[level];
295        if (rsc->tiled) {
296                /* No direct mappings of tiled, since we need to manually
297                 * tile/untile.
298                 */
299                if (usage & PIPE_MAP_DIRECTLY)
300                        return NULL;
301
302                ptrans->stride = ptrans->box.width * rsc->cpp;
303                ptrans->layer_stride = ptrans->stride * ptrans->box.height;
304
305                trans->map = malloc(ptrans->layer_stride * ptrans->box.depth);
306
307                if (usage & PIPE_MAP_READ) {
308                        for (int z = 0; z < ptrans->box.depth; z++) {
309                                void *src = rsc->bo->map +
310                                        v3d_layer_offset(&rsc->base,
311                                                         ptrans->level,
312                                                         ptrans->box.z + z);
313                                v3d_load_tiled_image((trans->map +
314                                                      ptrans->stride *
315                                                      ptrans->box.height * z),
316                                                     ptrans->stride,
317                                                     src,
318                                                     slice->stride,
319                                                     slice->tiling, rsc->cpp,
320                                                     slice->padded_height,
321                                             &ptrans->box);
322                        }
323                }
324                return trans->map;
325        } else {
326                ptrans->stride = slice->stride;
327                ptrans->layer_stride = rsc->cube_map_stride;
328
329                return buf + slice->offset +
330                        ptrans->box.y * ptrans->stride +
331                        ptrans->box.x * rsc->cpp +
332                        ptrans->box.z * rsc->cube_map_stride;
333        }
334
335
336fail:
337        v3d_resource_transfer_unmap(pctx, ptrans);
338        return NULL;
339}
340
341static void
342v3d_texture_subdata(struct pipe_context *pctx,
343                    struct pipe_resource *prsc,
344                    unsigned level,
345                    unsigned usage,
346                    const struct pipe_box *box,
347                    const void *data,
348                    unsigned stride,
349                    unsigned layer_stride)
350{
351        struct v3d_resource *rsc = v3d_resource(prsc);
352        struct v3d_resource_slice *slice = &rsc->slices[level];
353
354        /* For a direct mapping, we can just take the u_transfer path. */
355        if (!rsc->tiled) {
356                return u_default_texture_subdata(pctx, prsc, level, usage, box,
357                                                 data, stride, layer_stride);
358        }
359
360        /* Otherwise, map and store the texture data directly into the tiled
361         * texture.  Note that gallium's texture_subdata may be called with
362         * obvious usage flags missing!
363         */
364        v3d_map_usage_prep(pctx, prsc, usage | (PIPE_MAP_WRITE |
365                                                PIPE_MAP_DISCARD_RANGE));
366
367        void *buf;
368        if (usage & PIPE_MAP_UNSYNCHRONIZED)
369                buf = v3d_bo_map_unsynchronized(rsc->bo);
370        else
371                buf = v3d_bo_map(rsc->bo);
372
373        for (int i = 0; i < box->depth; i++) {
374                v3d_store_tiled_image(buf +
375                                      v3d_layer_offset(&rsc->base,
376                                                       level,
377                                                       box->z + i),
378                                      slice->stride,
379                                      (void *)data + layer_stride * i,
380                                      stride,
381                                      slice->tiling, rsc->cpp, slice->padded_height,
382                                      box);
383        }
384}
385
386static void
387v3d_resource_destroy(struct pipe_screen *pscreen,
388                     struct pipe_resource *prsc)
389{
390        struct v3d_screen *screen = v3d_screen(pscreen);
391        struct v3d_resource *rsc = v3d_resource(prsc);
392
393        if (rsc->scanout)
394                renderonly_scanout_destroy(rsc->scanout, screen->ro);
395
396        v3d_bo_unreference(&rsc->bo);
397        free(rsc);
398}
399
400static uint64_t
401v3d_resource_modifier(struct v3d_resource *rsc)
402{
403        if (rsc->tiled) {
404                /* A shared tiled buffer should always be allocated as UIF,
405                 * not UBLINEAR or LT.
406                 */
407                assert(rsc->slices[0].tiling == V3D_TILING_UIF_XOR ||
408                       rsc->slices[0].tiling == V3D_TILING_UIF_NO_XOR);
409                return DRM_FORMAT_MOD_BROADCOM_UIF;
410        } else {
411                return DRM_FORMAT_MOD_LINEAR;
412        }
413}
414
415static bool
416v3d_resource_get_handle(struct pipe_screen *pscreen,
417                        struct pipe_context *pctx,
418                        struct pipe_resource *prsc,
419                        struct winsys_handle *whandle,
420                        unsigned usage)
421{
422        struct v3d_screen *screen = v3d_screen(pscreen);
423        struct v3d_resource *rsc = v3d_resource(prsc);
424        struct v3d_bo *bo = rsc->bo;
425
426        whandle->stride = rsc->slices[0].stride;
427        whandle->offset = 0;
428        whandle->modifier = v3d_resource_modifier(rsc);
429
430        /* If we're passing some reference to our BO out to some other part of
431         * the system, then we can't do any optimizations about only us being
432         * the ones seeing it (like BO caching).
433         */
434        bo->private = false;
435
436        switch (whandle->type) {
437        case WINSYS_HANDLE_TYPE_SHARED:
438                return v3d_bo_flink(bo, &whandle->handle);
439        case WINSYS_HANDLE_TYPE_KMS:
440                if (screen->ro) {
441                        if (renderonly_get_handle(rsc->scanout, whandle)) {
442                                whandle->stride = rsc->slices[0].stride;
443                                return true;
444                        }
445                        return false;
446                }
447                whandle->handle = bo->handle;
448                return true;
449        case WINSYS_HANDLE_TYPE_FD:
450                whandle->handle = v3d_bo_get_dmabuf(bo);
451                return whandle->handle != -1;
452        }
453
454        return false;
455}
456
457static bool
458v3d_resource_get_param(struct pipe_screen *pscreen,
459                       struct pipe_context *pctx, struct pipe_resource *prsc,
460                       unsigned plane, unsigned layer, unsigned level,
461                       enum pipe_resource_param param,
462                       unsigned usage, uint64_t *value)
463{
464        struct v3d_resource *rsc = v3d_resource(prsc);
465
466        switch (param) {
467        case PIPE_RESOURCE_PARAM_STRIDE:
468                *value = rsc->slices[level].stride;
469                return true;
470        case PIPE_RESOURCE_PARAM_OFFSET:
471                *value = 0;
472                return true;
473        case PIPE_RESOURCE_PARAM_MODIFIER:
474                *value = v3d_resource_modifier(rsc);
475                return true;
476        default:
477                return false;
478        }
479}
480
481#define PAGE_UB_ROWS (V3D_UIFCFG_PAGE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
482#define PAGE_UB_ROWS_TIMES_1_5 ((PAGE_UB_ROWS * 3) >> 1)
483#define PAGE_CACHE_UB_ROWS (V3D_PAGE_CACHE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
484#define PAGE_CACHE_MINUS_1_5_UB_ROWS (PAGE_CACHE_UB_ROWS - PAGE_UB_ROWS_TIMES_1_5)
485
486/**
487 * Computes the HW's UIFblock padding for a given height/cpp.
488 *
489 * The goal of the padding is to keep pages of the same color (bank number) at
490 * least half a page away from each other vertically when crossing between
491 * between columns of UIF blocks.
492 */
493static uint32_t
494v3d_get_ub_pad(struct v3d_resource *rsc, uint32_t height)
495{
496        uint32_t utile_h = v3d_utile_height(rsc->cpp);
497        uint32_t uif_block_h = utile_h * 2;
498        uint32_t height_ub = height / uif_block_h;
499
500        uint32_t height_offset_in_pc = height_ub % PAGE_CACHE_UB_ROWS;
501
502        /* For the perfectly-aligned-for-UIF-XOR case, don't add any pad. */
503        if (height_offset_in_pc == 0)
504                return 0;
505
506        /* Try padding up to where we're offset by at least half a page. */
507        if (height_offset_in_pc < PAGE_UB_ROWS_TIMES_1_5) {
508                /* If we fit entirely in the page cache, don't pad. */
509                if (height_ub < PAGE_CACHE_UB_ROWS)
510                        return 0;
511                else
512                        return PAGE_UB_ROWS_TIMES_1_5 - height_offset_in_pc;
513        }
514
515        /* If we're close to being aligned to page cache size, then round up
516         * and rely on XOR.
517         */
518        if (height_offset_in_pc > PAGE_CACHE_MINUS_1_5_UB_ROWS)
519                return PAGE_CACHE_UB_ROWS - height_offset_in_pc;
520
521        /* Otherwise, we're far enough away (top and bottom) to not need any
522         * padding.
523         */
524        return 0;
525}
526
527static void
528v3d_setup_slices(struct v3d_resource *rsc, uint32_t winsys_stride,
529                 bool uif_top)
530{
531        struct pipe_resource *prsc = &rsc->base;
532        uint32_t width = prsc->width0;
533        uint32_t height = prsc->height0;
534        uint32_t depth = prsc->depth0;
535        /* Note that power-of-two padding is based on level 1.  These are not
536         * equivalent to just util_next_power_of_two(dimension), because at a
537         * level 0 dimension of 9, the level 1 power-of-two padded value is 4,
538         * not 8.
539         */
540        uint32_t pot_width = 2 * util_next_power_of_two(u_minify(width, 1));
541        uint32_t pot_height = 2 * util_next_power_of_two(u_minify(height, 1));
542        uint32_t pot_depth = 2 * util_next_power_of_two(u_minify(depth, 1));
543        uint32_t offset = 0;
544        uint32_t utile_w = v3d_utile_width(rsc->cpp);
545        uint32_t utile_h = v3d_utile_height(rsc->cpp);
546        uint32_t uif_block_w = utile_w * 2;
547        uint32_t uif_block_h = utile_h * 2;
548        uint32_t block_width = util_format_get_blockwidth(prsc->format);
549        uint32_t block_height = util_format_get_blockheight(prsc->format);
550        bool msaa = prsc->nr_samples > 1;
551
552        /* MSAA textures/renderbuffers are always laid out as single-level
553         * UIF.
554         */
555        uif_top |= msaa;
556
557        /* Check some easy mistakes to make in a resource_create() call that
558         * will break our setup.
559         */
560        assert(prsc->array_size != 0);
561        assert(prsc->depth0 != 0);
562
563        for (int i = prsc->last_level; i >= 0; i--) {
564                struct v3d_resource_slice *slice = &rsc->slices[i];
565
566                uint32_t level_width, level_height, level_depth;
567                if (i < 2) {
568                        level_width = u_minify(width, i);
569                        level_height = u_minify(height, i);
570                } else {
571                        level_width = u_minify(pot_width, i);
572                        level_height = u_minify(pot_height, i);
573                }
574                if (i < 1)
575                        level_depth = u_minify(depth, i);
576                else
577                        level_depth = u_minify(pot_depth, i);
578
579                if (msaa) {
580                        level_width *= 2;
581                        level_height *= 2;
582                }
583
584                level_width = DIV_ROUND_UP(level_width, block_width);
585                level_height = DIV_ROUND_UP(level_height, block_height);
586
587                if (!rsc->tiled) {
588                        slice->tiling = V3D_TILING_RASTER;
589                        if (prsc->target == PIPE_TEXTURE_1D)
590                                level_width = align(level_width, 64 / rsc->cpp);
591                } else {
592                        if ((i != 0 || !uif_top) &&
593                            (level_width <= utile_w ||
594                             level_height <= utile_h)) {
595                                slice->tiling = V3D_TILING_LINEARTILE;
596                                level_width = align(level_width, utile_w);
597                                level_height = align(level_height, utile_h);
598                        } else if ((i != 0 || !uif_top) &&
599                                   level_width <= uif_block_w) {
600                                slice->tiling = V3D_TILING_UBLINEAR_1_COLUMN;
601                                level_width = align(level_width, uif_block_w);
602                                level_height = align(level_height, uif_block_h);
603                        } else if ((i != 0 || !uif_top) &&
604                                   level_width <= 2 * uif_block_w) {
605                                slice->tiling = V3D_TILING_UBLINEAR_2_COLUMN;
606                                level_width = align(level_width, 2 * uif_block_w);
607                                level_height = align(level_height, uif_block_h);
608                        } else {
609                                /* We align the width to a 4-block column of
610                                 * UIF blocks, but we only align height to UIF
611                                 * blocks.
612                                 */
613                                level_width = align(level_width,
614                                                    4 * uif_block_w);
615                                level_height = align(level_height,
616                                                     uif_block_h);
617
618                                slice->ub_pad = v3d_get_ub_pad(rsc,
619                                                               level_height);
620                                level_height += slice->ub_pad * uif_block_h;
621
622                                /* If the padding set us to to be aligned to
623                                 * the page cache size, then the HW will use
624                                 * the XOR bit on odd columns to get us
625                                 * perfectly misaligned
626                                 */
627                                if ((level_height / uif_block_h) %
628                                    (V3D_PAGE_CACHE_SIZE /
629                                     V3D_UIFBLOCK_ROW_SIZE) == 0) {
630                                        slice->tiling = V3D_TILING_UIF_XOR;
631                                } else {
632                                        slice->tiling = V3D_TILING_UIF_NO_XOR;
633                                }
634                        }
635                }
636
637                slice->offset = offset;
638                if (winsys_stride)
639                        slice->stride = winsys_stride;
640                else
641                        slice->stride = level_width * rsc->cpp;
642                slice->padded_height = level_height;
643                slice->size = level_height * slice->stride;
644
645                uint32_t slice_total_size = slice->size * level_depth;
646
647                /* The HW aligns level 1's base to a page if any of level 1 or
648                 * below could be UIF XOR.  The lower levels then inherit the
649                 * alignment for as long as necessary, thanks to being power of
650                 * two aligned.
651                 */
652                if (i == 1 &&
653                    level_width > 4 * uif_block_w &&
654                    level_height > PAGE_CACHE_MINUS_1_5_UB_ROWS * uif_block_h) {
655                        slice_total_size = align(slice_total_size,
656                                                 V3D_UIFCFG_PAGE_SIZE);
657                }
658
659                offset += slice_total_size;
660
661        }
662        rsc->size = offset;
663
664        /* UIF/UBLINEAR levels need to be aligned to UIF-blocks, and LT only
665         * needs to be aligned to utile boundaries.  Since tiles are laid out
666         * from small to big in memory, we need to align the later UIF slices
667         * to UIF blocks, if they were preceded by non-UIF-block-aligned LT
668         * slices.
669         *
670         * We additionally align to 4k, which improves UIF XOR performance.
671         */
672        uint32_t page_align_offset = (align(rsc->slices[0].offset, 4096) -
673                                      rsc->slices[0].offset);
674        if (page_align_offset) {
675                rsc->size += page_align_offset;
676                for (int i = 0; i <= prsc->last_level; i++)
677                        rsc->slices[i].offset += page_align_offset;
678        }
679
680        /* Arrays and cube textures have a stride which is the distance from
681         * one full mipmap tree to the next (64b aligned).  For 3D textures,
682         * we need to program the stride between slices of miplevel 0.
683         */
684        if (prsc->target != PIPE_TEXTURE_3D) {
685                rsc->cube_map_stride = align(rsc->slices[0].offset +
686                                             rsc->slices[0].size, 64);
687                rsc->size += rsc->cube_map_stride * (prsc->array_size - 1);
688        } else {
689                rsc->cube_map_stride = rsc->slices[0].size;
690        }
691}
692
693uint32_t
694v3d_layer_offset(struct pipe_resource *prsc, uint32_t level, uint32_t layer)
695{
696        struct v3d_resource *rsc = v3d_resource(prsc);
697        struct v3d_resource_slice *slice = &rsc->slices[level];
698
699        if (prsc->target == PIPE_TEXTURE_3D)
700                return slice->offset + layer * slice->size;
701        else
702                return slice->offset + layer * rsc->cube_map_stride;
703}
704
705static struct v3d_resource *
706v3d_resource_setup(struct pipe_screen *pscreen,
707                   const struct pipe_resource *tmpl)
708{
709        struct v3d_screen *screen = v3d_screen(pscreen);
710        struct v3d_resource *rsc = CALLOC_STRUCT(v3d_resource);
711        if (!rsc)
712                return NULL;
713        struct pipe_resource *prsc = &rsc->base;
714
715        *prsc = *tmpl;
716
717        pipe_reference_init(&prsc->reference, 1);
718        prsc->screen = pscreen;
719
720        if (prsc->nr_samples <= 1 ||
721            screen->devinfo.ver >= 40 ||
722            util_format_is_depth_or_stencil(prsc->format)) {
723                rsc->cpp = util_format_get_blocksize(prsc->format);
724                if (screen->devinfo.ver < 40 && prsc->nr_samples > 1)
725                        rsc->cpp *= prsc->nr_samples;
726        } else {
727                assert(v3d_rt_format_supported(&screen->devinfo, prsc->format));
728                uint32_t output_image_format =
729                        v3d_get_rt_format(&screen->devinfo, prsc->format);
730                uint32_t internal_type;
731                uint32_t internal_bpp;
732                v3d_get_internal_type_bpp_for_output_format(&screen->devinfo,
733                                                            output_image_format,
734                                                            &internal_type,
735                                                            &internal_bpp);
736                switch (internal_bpp) {
737                case V3D_INTERNAL_BPP_32:
738                        rsc->cpp = 4;
739                        break;
740                case V3D_INTERNAL_BPP_64:
741                        rsc->cpp = 8;
742                        break;
743                case V3D_INTERNAL_BPP_128:
744                        rsc->cpp = 16;
745                        break;
746                }
747        }
748
749        rsc->serial_id++;
750
751        assert(rsc->cpp);
752
753        return rsc;
754}
755
756static struct pipe_resource *
757v3d_resource_create_with_modifiers(struct pipe_screen *pscreen,
758                                   const struct pipe_resource *tmpl,
759                                   const uint64_t *modifiers,
760                                   int count)
761{
762        struct v3d_screen *screen = v3d_screen(pscreen);
763
764        bool linear_ok = drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count);
765        struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
766        struct pipe_resource *prsc = &rsc->base;
767        /* Use a tiled layout if we can, for better 3D performance. */
768        bool should_tile = true;
769
770        assert(tmpl->target != PIPE_BUFFER ||
771               (tmpl->format == PIPE_FORMAT_NONE ||
772                util_format_get_blocksize(tmpl->format) == 1));
773
774        /* VBOs/PBOs/Texture Buffer Objects are untiled (and 1 height). */
775        if (tmpl->target == PIPE_BUFFER)
776                should_tile = false;
777
778        /* Cursors are always linear, and the user can request linear as well.
779         */
780        if (tmpl->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR))
781                should_tile = false;
782
783        /* 1D and 1D_ARRAY textures are always raster-order. */
784        if (tmpl->target == PIPE_TEXTURE_1D ||
785            tmpl->target == PIPE_TEXTURE_1D_ARRAY)
786                should_tile = false;
787
788        /* Scanout BOs for simulator need to be linear for interaction with
789         * i965.
790         */
791        if (using_v3d_simulator &&
792            tmpl->bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT))
793                should_tile = false;
794
795        /* If using the old-school SCANOUT flag, we don't know what the screen
796         * might support other than linear. Just force linear.
797         */
798        if (tmpl->bind & PIPE_BIND_SCANOUT)
799                should_tile = false;
800
801        /* No user-specified modifier; determine our own. */
802        if (count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID) {
803                linear_ok = true;
804                rsc->tiled = should_tile;
805        } else if (should_tile &&
806                   drm_find_modifier(DRM_FORMAT_MOD_BROADCOM_UIF,
807                                 modifiers, count)) {
808                rsc->tiled = true;
809        } else if (linear_ok) {
810                rsc->tiled = false;
811        } else {
812                fprintf(stderr, "Unsupported modifier requested\n");
813                goto fail;
814        }
815
816        rsc->internal_format = prsc->format;
817
818        v3d_setup_slices(rsc, 0, tmpl->bind & PIPE_BIND_SHARED);
819
820        if (screen->ro && (tmpl->bind & PIPE_BIND_SCANOUT)) {
821                struct winsys_handle handle;
822                struct pipe_resource scanout_tmpl = {
823                        .target = prsc->target,
824                        .format = PIPE_FORMAT_RGBA8888_UNORM,
825                        .width0 = 1024, /* one page */
826                        .height0 = align(rsc->size, 4096) / 4096,
827                        .depth0 = 1,
828                        .array_size = 1,
829                };
830
831                rsc->scanout =
832                        renderonly_scanout_for_resource(&scanout_tmpl,
833                                                        screen->ro,
834                                                        &handle);
835
836                if (!rsc->scanout) {
837                        fprintf(stderr, "Failed to create scanout resource\n");
838                        goto fail;
839                }
840                assert(handle.type == WINSYS_HANDLE_TYPE_FD);
841                rsc->bo = v3d_bo_open_dmabuf(screen, handle.handle);
842                close(handle.handle);
843
844                if (!rsc->bo)
845                        goto fail;
846
847                v3d_debug_resource_layout(rsc, "renderonly");
848
849                return prsc;
850        } else {
851                if (!v3d_resource_bo_alloc(rsc))
852                        goto fail;
853        }
854
855        return prsc;
856fail:
857        v3d_resource_destroy(pscreen, prsc);
858        return NULL;
859}
860
861struct pipe_resource *
862v3d_resource_create(struct pipe_screen *pscreen,
863                    const struct pipe_resource *tmpl)
864{
865        const uint64_t mod = DRM_FORMAT_MOD_INVALID;
866        return v3d_resource_create_with_modifiers(pscreen, tmpl, &mod, 1);
867}
868
869static struct pipe_resource *
870v3d_resource_from_handle(struct pipe_screen *pscreen,
871                         const struct pipe_resource *tmpl,
872                         struct winsys_handle *whandle,
873                         unsigned usage)
874{
875        struct v3d_screen *screen = v3d_screen(pscreen);
876        struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
877        struct pipe_resource *prsc = &rsc->base;
878        struct v3d_resource_slice *slice = &rsc->slices[0];
879
880        if (!rsc)
881                return NULL;
882
883        switch (whandle->modifier) {
884        case DRM_FORMAT_MOD_LINEAR:
885                rsc->tiled = false;
886                break;
887        case DRM_FORMAT_MOD_BROADCOM_UIF:
888                rsc->tiled = true;
889                break;
890        case DRM_FORMAT_MOD_INVALID:
891                rsc->tiled = screen->ro == NULL;
892                break;
893        default:
894                switch(fourcc_mod_broadcom_mod(whandle->modifier)) {
895                case DRM_FORMAT_MOD_BROADCOM_SAND128:
896                        rsc->tiled = false;
897                        rsc->sand_col128_stride =
898                                fourcc_mod_broadcom_param(whandle->modifier);
899                        break;
900                default:
901                        fprintf(stderr,
902                                "Attempt to import unsupported modifier 0x%llx\n",
903                                (long long)whandle->modifier);
904                        goto fail;
905                }
906        }
907
908        switch (whandle->type) {
909        case WINSYS_HANDLE_TYPE_SHARED:
910                rsc->bo = v3d_bo_open_name(screen, whandle->handle);
911                break;
912        case WINSYS_HANDLE_TYPE_FD:
913                rsc->bo = v3d_bo_open_dmabuf(screen, whandle->handle);
914                break;
915        default:
916                fprintf(stderr,
917                        "Attempt to import unsupported handle type %d\n",
918                        whandle->type);
919                goto fail;
920        }
921
922        if (!rsc->bo)
923                goto fail;
924
925        rsc->internal_format = prsc->format;
926
927        v3d_setup_slices(rsc, whandle->stride, true);
928        v3d_debug_resource_layout(rsc, "import");
929
930        if (whandle->offset != 0) {
931                if (rsc->tiled) {
932                        fprintf(stderr,
933                                "Attempt to import unsupported winsys offset %u\n",
934                                whandle->offset);
935                        goto fail;
936                }
937                rsc->slices[0].offset += whandle->offset;
938
939                if (rsc->slices[0].offset + rsc->slices[0].size >
940                    rsc->bo->size) {
941                        fprintf(stderr, "Attempt to import "
942                                "with overflowing offset (%d + %d > %d)\n",
943                                whandle->offset,
944                                rsc->slices[0].size,
945                                rsc->bo->size);
946                         goto fail;
947                 }
948        }
949
950        if (screen->ro) {
951                /* Make sure that renderonly has a handle to our buffer in the
952                 * display's fd, so that a later renderonly_get_handle()
953                 * returns correct handles or GEM names.
954                 */
955                rsc->scanout =
956                        renderonly_create_gpu_import_for_resource(prsc,
957                                                                  screen->ro,
958                                                                  NULL);
959        }
960
961        if (rsc->tiled && whandle->stride != slice->stride) {
962                static bool warned = false;
963                if (!warned) {
964                        warned = true;
965                        fprintf(stderr,
966                                "Attempting to import %dx%d %s with "
967                                "unsupported stride %d instead of %d\n",
968                                prsc->width0, prsc->height0,
969                                util_format_short_name(prsc->format),
970                                whandle->stride,
971                                slice->stride);
972                }
973                goto fail;
974        } else if (!rsc->tiled) {
975                slice->stride = whandle->stride;
976        }
977
978        return prsc;
979
980fail:
981        v3d_resource_destroy(pscreen, prsc);
982        return NULL;
983}
984
985void
986v3d_update_shadow_texture(struct pipe_context *pctx,
987                          struct pipe_sampler_view *pview)
988{
989        struct v3d_context *v3d = v3d_context(pctx);
990        struct v3d_sampler_view *view = v3d_sampler_view(pview);
991        struct v3d_resource *shadow = v3d_resource(view->texture);
992        struct v3d_resource *orig = v3d_resource(pview->texture);
993
994        assert(view->texture != pview->texture);
995
996        if (shadow->writes == orig->writes && orig->bo->private)
997                return;
998
999        perf_debug("Updating %dx%d@%d shadow for linear texture\n",
1000                   orig->base.width0, orig->base.height0,
1001                   pview->u.tex.first_level);
1002
1003        for (int i = 0; i <= shadow->base.last_level; i++) {
1004                unsigned width = u_minify(shadow->base.width0, i);
1005                unsigned height = u_minify(shadow->base.height0, i);
1006                struct pipe_blit_info info = {
1007                        .dst = {
1008                                .resource = &shadow->base,
1009                                .level = i,
1010                                .box = {
1011                                        .x = 0,
1012                                        .y = 0,
1013                                        .z = 0,
1014                                        .width = width,
1015                                        .height = height,
1016                                        .depth = 1,
1017                                },
1018                                .format = shadow->base.format,
1019                        },
1020                        .src = {
1021                                .resource = &orig->base,
1022                                .level = pview->u.tex.first_level + i,
1023                                .box = {
1024                                        .x = 0,
1025                                        .y = 0,
1026                                        .z = 0,
1027                                        .width = width,
1028                                        .height = height,
1029                                        .depth = 1,
1030                                },
1031                                .format = orig->base.format,
1032                        },
1033                        .mask = util_format_get_mask(orig->base.format),
1034                };
1035                pctx->blit(pctx, &info);
1036        }
1037
1038        shadow->writes = orig->writes;
1039}
1040
1041static struct pipe_surface *
1042v3d_create_surface(struct pipe_context *pctx,
1043                   struct pipe_resource *ptex,
1044                   const struct pipe_surface *surf_tmpl)
1045{
1046        struct v3d_context *v3d = v3d_context(pctx);
1047        struct v3d_screen *screen = v3d->screen;
1048        struct v3d_surface *surface = CALLOC_STRUCT(v3d_surface);
1049        struct v3d_resource *rsc = v3d_resource(ptex);
1050
1051        if (!surface)
1052                return NULL;
1053
1054        struct pipe_surface *psurf = &surface->base;
1055        unsigned level = surf_tmpl->u.tex.level;
1056        struct v3d_resource_slice *slice = &rsc->slices[level];
1057
1058        pipe_reference_init(&psurf->reference, 1);
1059        pipe_resource_reference(&psurf->texture, ptex);
1060
1061        psurf->context = pctx;
1062        psurf->format = surf_tmpl->format;
1063        psurf->width = u_minify(ptex->width0, level);
1064        psurf->height = u_minify(ptex->height0, level);
1065        psurf->u.tex.level = level;
1066        psurf->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
1067        psurf->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
1068
1069        surface->offset = v3d_layer_offset(ptex, level,
1070                                           psurf->u.tex.first_layer);
1071        surface->tiling = slice->tiling;
1072
1073        surface->format = v3d_get_rt_format(&screen->devinfo, psurf->format);
1074
1075        const struct util_format_description *desc =
1076                util_format_description(psurf->format);
1077
1078        surface->swap_rb = (desc->swizzle[0] == PIPE_SWIZZLE_Z &&
1079                            psurf->format != PIPE_FORMAT_B5G6R5_UNORM);
1080
1081        if (util_format_is_depth_or_stencil(psurf->format)) {
1082                switch (psurf->format) {
1083                case PIPE_FORMAT_Z16_UNORM:
1084                        surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_16;
1085                        break;
1086                case PIPE_FORMAT_Z32_FLOAT:
1087                case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1088                        surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_32F;
1089                        break;
1090                default:
1091                        surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_24;
1092                }
1093        } else {
1094                uint32_t bpp, type;
1095                v3d_get_internal_type_bpp_for_output_format(&screen->devinfo,
1096                                                            surface->format,
1097                                                            &type, &bpp);
1098                surface->internal_type = type;
1099                surface->internal_bpp = bpp;
1100        }
1101
1102        if (surface->tiling == V3D_TILING_UIF_NO_XOR ||
1103            surface->tiling == V3D_TILING_UIF_XOR) {
1104                surface->padded_height_of_output_image_in_uif_blocks =
1105                        (slice->padded_height /
1106                         (2 * v3d_utile_height(rsc->cpp)));
1107        }
1108
1109        if (rsc->separate_stencil) {
1110                surface->separate_stencil =
1111                        v3d_create_surface(pctx, &rsc->separate_stencil->base,
1112                                           surf_tmpl);
1113        }
1114
1115        return &surface->base;
1116}
1117
1118static void
1119v3d_surface_destroy(struct pipe_context *pctx, struct pipe_surface *psurf)
1120{
1121        struct v3d_surface *surf = v3d_surface(psurf);
1122
1123        if (surf->separate_stencil)
1124                pipe_surface_reference(&surf->separate_stencil, NULL);
1125
1126        pipe_resource_reference(&psurf->texture, NULL);
1127        FREE(psurf);
1128}
1129
1130static void
1131v3d_flush_resource(struct pipe_context *pctx, struct pipe_resource *resource)
1132{
1133        /* All calls to flush_resource are followed by a flush of the context,
1134         * so there's nothing to do.
1135         */
1136}
1137
1138static enum pipe_format
1139v3d_resource_get_internal_format(struct pipe_resource *prsc)
1140{
1141        return v3d_resource(prsc)->internal_format;
1142}
1143
1144static void
1145v3d_resource_set_stencil(struct pipe_resource *prsc,
1146                         struct pipe_resource *stencil)
1147{
1148        v3d_resource(prsc)->separate_stencil = v3d_resource(stencil);
1149}
1150
1151static struct pipe_resource *
1152v3d_resource_get_stencil(struct pipe_resource *prsc)
1153{
1154        struct v3d_resource *rsc = v3d_resource(prsc);
1155
1156        return &rsc->separate_stencil->base;
1157}
1158
1159static const struct u_transfer_vtbl transfer_vtbl = {
1160        .resource_create          = v3d_resource_create,
1161        .resource_destroy         = v3d_resource_destroy,
1162        .transfer_map             = v3d_resource_transfer_map,
1163        .transfer_unmap           = v3d_resource_transfer_unmap,
1164        .transfer_flush_region    = u_default_transfer_flush_region,
1165        .get_internal_format      = v3d_resource_get_internal_format,
1166        .set_stencil              = v3d_resource_set_stencil,
1167        .get_stencil              = v3d_resource_get_stencil,
1168};
1169
1170void
1171v3d_resource_screen_init(struct pipe_screen *pscreen)
1172{
1173        pscreen->resource_create_with_modifiers =
1174                v3d_resource_create_with_modifiers;
1175        pscreen->resource_create = u_transfer_helper_resource_create;
1176        pscreen->resource_from_handle = v3d_resource_from_handle;
1177        pscreen->resource_get_handle = v3d_resource_get_handle;
1178        pscreen->resource_get_param = v3d_resource_get_param;
1179        pscreen->resource_destroy = u_transfer_helper_resource_destroy;
1180        pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl,
1181                                                            true, false,
1182                                                            true, true,
1183                                                            false);
1184}
1185
1186void
1187v3d_resource_context_init(struct pipe_context *pctx)
1188{
1189        pctx->buffer_map = u_transfer_helper_transfer_map;
1190        pctx->texture_map = u_transfer_helper_transfer_map;
1191        pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
1192        pctx->buffer_unmap = u_transfer_helper_transfer_unmap;
1193        pctx->texture_unmap = u_transfer_helper_transfer_unmap;
1194        pctx->buffer_subdata = u_default_buffer_subdata;
1195        pctx->texture_subdata = v3d_texture_subdata;
1196        pctx->create_surface = v3d_create_surface;
1197        pctx->surface_destroy = v3d_surface_destroy;
1198        pctx->resource_copy_region = util_resource_copy_region;
1199        pctx->blit = v3d_blit;
1200        pctx->generate_mipmap = v3d_generate_mipmap;
1201        pctx->flush_resource = v3d_flush_resource;
1202}
1203