1/* 2 * Copyright © 2014-2017 Broadcom 3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org> 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 * IN THE SOFTWARE. 23 */ 24 25#include "pipe/p_defines.h" 26#include "util/u_memory.h" 27#include "util/format/u_format.h" 28#include "util/u_inlines.h" 29#include "util/u_surface.h" 30#include "util/u_transfer_helper.h" 31#include "util/u_upload_mgr.h" 32#include "util/format/u_format_zs.h" 33#include "util/u_drm.h" 34 35#include "drm-uapi/drm_fourcc.h" 36#include "v3d_screen.h" 37#include "v3d_context.h" 38#include "v3d_resource.h" 39#include "broadcom/cle/v3d_packet_v33_pack.h" 40 41static void 42v3d_debug_resource_layout(struct v3d_resource *rsc, const char *caller) 43{ 44 if (!(unlikely(V3D_DEBUG & V3D_DEBUG_SURFACE))) 45 return; 46 47 struct pipe_resource *prsc = &rsc->base; 48 49 if (prsc->target == PIPE_BUFFER) { 50 fprintf(stderr, 51 "rsc %s %p (format %s), %dx%d buffer @0x%08x-0x%08x\n", 52 caller, rsc, 53 util_format_short_name(prsc->format), 54 prsc->width0, prsc->height0, 55 rsc->bo->offset, 56 rsc->bo->offset + rsc->bo->size - 1); 57 return; 58 } 59 60 static const char *const tiling_descriptions[] = { 61 [V3D_TILING_RASTER] = "R", 62 [V3D_TILING_LINEARTILE] = "LT", 63 [V3D_TILING_UBLINEAR_1_COLUMN] = "UB1", 64 [V3D_TILING_UBLINEAR_2_COLUMN] = "UB2", 65 [V3D_TILING_UIF_NO_XOR] = "UIF", 66 [V3D_TILING_UIF_XOR] = "UIF^", 67 }; 68 69 for (int i = 0; i <= prsc->last_level; i++) { 70 struct v3d_resource_slice *slice = &rsc->slices[i]; 71 72 int level_width = slice->stride / rsc->cpp; 73 int level_height = slice->padded_height; 74 int level_depth = 75 u_minify(util_next_power_of_two(prsc->depth0), i); 76 77 fprintf(stderr, 78 "rsc %s %p (format %s), %dx%d: " 79 "level %d (%s) %dx%dx%d -> %dx%dx%d, stride %d@0x%08x\n", 80 caller, rsc, 81 util_format_short_name(prsc->format), 82 prsc->width0, prsc->height0, 83 i, tiling_descriptions[slice->tiling], 84 u_minify(prsc->width0, i), 85 u_minify(prsc->height0, i), 86 u_minify(prsc->depth0, i), 87 level_width, 88 level_height, 89 level_depth, 90 slice->stride, 91 rsc->bo->offset + slice->offset); 92 } 93} 94 95static bool 96v3d_resource_bo_alloc(struct v3d_resource *rsc) 97{ 98 struct pipe_resource *prsc = &rsc->base; 99 struct pipe_screen *pscreen = prsc->screen; 100 struct v3d_bo *bo; 101 102 bo = v3d_bo_alloc(v3d_screen(pscreen), rsc->size, "resource"); 103 if (bo) { 104 v3d_bo_unreference(&rsc->bo); 105 rsc->bo = bo; 106 rsc->serial_id++; 107 v3d_debug_resource_layout(rsc, "alloc"); 108 return true; 109 } else { 110 return false; 111 } 112} 113 114static void 115v3d_resource_transfer_unmap(struct pipe_context *pctx, 116 struct pipe_transfer *ptrans) 117{ 118 struct v3d_context *v3d = v3d_context(pctx); 119 struct v3d_transfer *trans = v3d_transfer(ptrans); 120 121 if (trans->map) { 122 struct v3d_resource *rsc = v3d_resource(ptrans->resource); 123 struct v3d_resource_slice *slice = &rsc->slices[ptrans->level]; 124 125 if (ptrans->usage & PIPE_MAP_WRITE) { 126 for (int z = 0; z < ptrans->box.depth; z++) { 127 void *dst = rsc->bo->map + 128 v3d_layer_offset(&rsc->base, 129 ptrans->level, 130 ptrans->box.z + z); 131 v3d_store_tiled_image(dst, 132 slice->stride, 133 (trans->map + 134 ptrans->stride * 135 ptrans->box.height * z), 136 ptrans->stride, 137 slice->tiling, rsc->cpp, 138 slice->padded_height, 139 &ptrans->box); 140 } 141 } 142 free(trans->map); 143 } 144 145 pipe_resource_reference(&ptrans->resource, NULL); 146 slab_free(&v3d->transfer_pool, ptrans); 147} 148 149static void 150rebind_sampler_views(struct v3d_context *v3d, 151 struct v3d_resource *rsc) 152{ 153 for (int st = 0; st < PIPE_SHADER_TYPES; st++) { 154 struct v3d_texture_stateobj *tex = v3d->tex + st; 155 156 for (unsigned i = 0; i < tex->num_textures; i++) { 157 struct pipe_sampler_view *psview = tex->textures[i]; 158 159 if (psview->texture != &rsc->base) 160 continue; 161 162 struct v3d_sampler_view *sview = 163 v3d_sampler_view(psview); 164 165 v3d_create_texture_shader_state_bo(v3d, sview); 166 167 v3d_flag_dirty_sampler_state(v3d, st); 168 } 169 } 170} 171 172static void 173v3d_map_usage_prep(struct pipe_context *pctx, 174 struct pipe_resource *prsc, 175 unsigned usage) 176{ 177 struct v3d_context *v3d = v3d_context(pctx); 178 struct v3d_resource *rsc = v3d_resource(prsc); 179 180 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) { 181 if (v3d_resource_bo_alloc(rsc)) { 182 /* If it might be bound as one of our vertex buffers 183 * or UBOs, make sure we re-emit vertex buffer state 184 * or uniforms. 185 */ 186 if (prsc->bind & PIPE_BIND_VERTEX_BUFFER) 187 v3d->dirty |= V3D_DIRTY_VTXBUF; 188 if (prsc->bind & PIPE_BIND_CONSTANT_BUFFER) 189 v3d->dirty |= V3D_DIRTY_CONSTBUF; 190 /* Since we are changing the texture BO we need to 191 * update any bound samplers to point to the new 192 * BO. Notice we can have samplers that are not 193 * currently bound to the state that won't be 194 * updated. These will be fixed when they are bound in 195 * v3d_set_sampler_views. 196 */ 197 if (prsc->bind & PIPE_BIND_SAMPLER_VIEW) 198 rebind_sampler_views(v3d, rsc); 199 } else { 200 /* If we failed to reallocate, flush users so that we 201 * don't violate any syncing requirements. 202 */ 203 v3d_flush_jobs_reading_resource(v3d, prsc, 204 V3D_FLUSH_DEFAULT, 205 false); 206 } 207 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) { 208 /* If we're writing and the buffer is being used by the CL, we 209 * have to flush the CL first. If we're only reading, we need 210 * to flush if the CL has written our buffer. 211 */ 212 if (usage & PIPE_MAP_WRITE) { 213 v3d_flush_jobs_reading_resource(v3d, prsc, 214 V3D_FLUSH_ALWAYS, 215 false); 216 } else { 217 v3d_flush_jobs_writing_resource(v3d, prsc, 218 V3D_FLUSH_ALWAYS, 219 false); 220 } 221 } 222 223 if (usage & PIPE_MAP_WRITE) { 224 rsc->writes++; 225 rsc->initialized_buffers = ~0; 226 } 227} 228 229static void * 230v3d_resource_transfer_map(struct pipe_context *pctx, 231 struct pipe_resource *prsc, 232 unsigned level, unsigned usage, 233 const struct pipe_box *box, 234 struct pipe_transfer **pptrans) 235{ 236 struct v3d_context *v3d = v3d_context(pctx); 237 struct v3d_resource *rsc = v3d_resource(prsc); 238 struct v3d_transfer *trans; 239 struct pipe_transfer *ptrans; 240 enum pipe_format format = prsc->format; 241 char *buf; 242 243 /* MSAA maps should have been handled by u_transfer_helper. */ 244 assert(prsc->nr_samples <= 1); 245 246 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is 247 * being mapped. 248 */ 249 if ((usage & PIPE_MAP_DISCARD_RANGE) && 250 !(usage & PIPE_MAP_UNSYNCHRONIZED) && 251 !(prsc->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) && 252 prsc->last_level == 0 && 253 prsc->width0 == box->width && 254 prsc->height0 == box->height && 255 prsc->depth0 == box->depth && 256 prsc->array_size == 1 && 257 rsc->bo->private) { 258 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE; 259 } 260 261 v3d_map_usage_prep(pctx, prsc, usage); 262 263 trans = slab_zalloc(&v3d->transfer_pool); 264 if (!trans) 265 return NULL; 266 267 /* XXX: Handle DONTBLOCK, DISCARD_RANGE, PERSISTENT, COHERENT. */ 268 269 ptrans = &trans->base; 270 271 pipe_resource_reference(&ptrans->resource, prsc); 272 ptrans->level = level; 273 ptrans->usage = usage; 274 ptrans->box = *box; 275 276 /* Note that the current kernel implementation is synchronous, so no 277 * need to do syncing stuff here yet. 278 */ 279 280 if (usage & PIPE_MAP_UNSYNCHRONIZED) 281 buf = v3d_bo_map_unsynchronized(rsc->bo); 282 else 283 buf = v3d_bo_map(rsc->bo); 284 if (!buf) { 285 fprintf(stderr, "Failed to map bo\n"); 286 goto fail; 287 } 288 289 *pptrans = ptrans; 290 291 /* Our load/store routines work on entire compressed blocks. */ 292 u_box_pixels_to_blocks(&ptrans->box, &ptrans->box, format); 293 294 struct v3d_resource_slice *slice = &rsc->slices[level]; 295 if (rsc->tiled) { 296 /* No direct mappings of tiled, since we need to manually 297 * tile/untile. 298 */ 299 if (usage & PIPE_MAP_DIRECTLY) 300 return NULL; 301 302 ptrans->stride = ptrans->box.width * rsc->cpp; 303 ptrans->layer_stride = ptrans->stride * ptrans->box.height; 304 305 trans->map = malloc(ptrans->layer_stride * ptrans->box.depth); 306 307 if (usage & PIPE_MAP_READ) { 308 for (int z = 0; z < ptrans->box.depth; z++) { 309 void *src = rsc->bo->map + 310 v3d_layer_offset(&rsc->base, 311 ptrans->level, 312 ptrans->box.z + z); 313 v3d_load_tiled_image((trans->map + 314 ptrans->stride * 315 ptrans->box.height * z), 316 ptrans->stride, 317 src, 318 slice->stride, 319 slice->tiling, rsc->cpp, 320 slice->padded_height, 321 &ptrans->box); 322 } 323 } 324 return trans->map; 325 } else { 326 ptrans->stride = slice->stride; 327 ptrans->layer_stride = rsc->cube_map_stride; 328 329 return buf + slice->offset + 330 ptrans->box.y * ptrans->stride + 331 ptrans->box.x * rsc->cpp + 332 ptrans->box.z * rsc->cube_map_stride; 333 } 334 335 336fail: 337 v3d_resource_transfer_unmap(pctx, ptrans); 338 return NULL; 339} 340 341static void 342v3d_texture_subdata(struct pipe_context *pctx, 343 struct pipe_resource *prsc, 344 unsigned level, 345 unsigned usage, 346 const struct pipe_box *box, 347 const void *data, 348 unsigned stride, 349 unsigned layer_stride) 350{ 351 struct v3d_resource *rsc = v3d_resource(prsc); 352 struct v3d_resource_slice *slice = &rsc->slices[level]; 353 354 /* For a direct mapping, we can just take the u_transfer path. */ 355 if (!rsc->tiled) { 356 return u_default_texture_subdata(pctx, prsc, level, usage, box, 357 data, stride, layer_stride); 358 } 359 360 /* Otherwise, map and store the texture data directly into the tiled 361 * texture. Note that gallium's texture_subdata may be called with 362 * obvious usage flags missing! 363 */ 364 v3d_map_usage_prep(pctx, prsc, usage | (PIPE_MAP_WRITE | 365 PIPE_MAP_DISCARD_RANGE)); 366 367 void *buf; 368 if (usage & PIPE_MAP_UNSYNCHRONIZED) 369 buf = v3d_bo_map_unsynchronized(rsc->bo); 370 else 371 buf = v3d_bo_map(rsc->bo); 372 373 for (int i = 0; i < box->depth; i++) { 374 v3d_store_tiled_image(buf + 375 v3d_layer_offset(&rsc->base, 376 level, 377 box->z + i), 378 slice->stride, 379 (void *)data + layer_stride * i, 380 stride, 381 slice->tiling, rsc->cpp, slice->padded_height, 382 box); 383 } 384} 385 386static void 387v3d_resource_destroy(struct pipe_screen *pscreen, 388 struct pipe_resource *prsc) 389{ 390 struct v3d_screen *screen = v3d_screen(pscreen); 391 struct v3d_resource *rsc = v3d_resource(prsc); 392 393 if (rsc->scanout) 394 renderonly_scanout_destroy(rsc->scanout, screen->ro); 395 396 v3d_bo_unreference(&rsc->bo); 397 free(rsc); 398} 399 400static uint64_t 401v3d_resource_modifier(struct v3d_resource *rsc) 402{ 403 if (rsc->tiled) { 404 /* A shared tiled buffer should always be allocated as UIF, 405 * not UBLINEAR or LT. 406 */ 407 assert(rsc->slices[0].tiling == V3D_TILING_UIF_XOR || 408 rsc->slices[0].tiling == V3D_TILING_UIF_NO_XOR); 409 return DRM_FORMAT_MOD_BROADCOM_UIF; 410 } else { 411 return DRM_FORMAT_MOD_LINEAR; 412 } 413} 414 415static bool 416v3d_resource_get_handle(struct pipe_screen *pscreen, 417 struct pipe_context *pctx, 418 struct pipe_resource *prsc, 419 struct winsys_handle *whandle, 420 unsigned usage) 421{ 422 struct v3d_screen *screen = v3d_screen(pscreen); 423 struct v3d_resource *rsc = v3d_resource(prsc); 424 struct v3d_bo *bo = rsc->bo; 425 426 whandle->stride = rsc->slices[0].stride; 427 whandle->offset = 0; 428 whandle->modifier = v3d_resource_modifier(rsc); 429 430 /* If we're passing some reference to our BO out to some other part of 431 * the system, then we can't do any optimizations about only us being 432 * the ones seeing it (like BO caching). 433 */ 434 bo->private = false; 435 436 switch (whandle->type) { 437 case WINSYS_HANDLE_TYPE_SHARED: 438 return v3d_bo_flink(bo, &whandle->handle); 439 case WINSYS_HANDLE_TYPE_KMS: 440 if (screen->ro) { 441 if (renderonly_get_handle(rsc->scanout, whandle)) { 442 whandle->stride = rsc->slices[0].stride; 443 return true; 444 } 445 return false; 446 } 447 whandle->handle = bo->handle; 448 return true; 449 case WINSYS_HANDLE_TYPE_FD: 450 whandle->handle = v3d_bo_get_dmabuf(bo); 451 return whandle->handle != -1; 452 } 453 454 return false; 455} 456 457static bool 458v3d_resource_get_param(struct pipe_screen *pscreen, 459 struct pipe_context *pctx, struct pipe_resource *prsc, 460 unsigned plane, unsigned layer, unsigned level, 461 enum pipe_resource_param param, 462 unsigned usage, uint64_t *value) 463{ 464 struct v3d_resource *rsc = v3d_resource(prsc); 465 466 switch (param) { 467 case PIPE_RESOURCE_PARAM_STRIDE: 468 *value = rsc->slices[level].stride; 469 return true; 470 case PIPE_RESOURCE_PARAM_OFFSET: 471 *value = 0; 472 return true; 473 case PIPE_RESOURCE_PARAM_MODIFIER: 474 *value = v3d_resource_modifier(rsc); 475 return true; 476 default: 477 return false; 478 } 479} 480 481#define PAGE_UB_ROWS (V3D_UIFCFG_PAGE_SIZE / V3D_UIFBLOCK_ROW_SIZE) 482#define PAGE_UB_ROWS_TIMES_1_5 ((PAGE_UB_ROWS * 3) >> 1) 483#define PAGE_CACHE_UB_ROWS (V3D_PAGE_CACHE_SIZE / V3D_UIFBLOCK_ROW_SIZE) 484#define PAGE_CACHE_MINUS_1_5_UB_ROWS (PAGE_CACHE_UB_ROWS - PAGE_UB_ROWS_TIMES_1_5) 485 486/** 487 * Computes the HW's UIFblock padding for a given height/cpp. 488 * 489 * The goal of the padding is to keep pages of the same color (bank number) at 490 * least half a page away from each other vertically when crossing between 491 * between columns of UIF blocks. 492 */ 493static uint32_t 494v3d_get_ub_pad(struct v3d_resource *rsc, uint32_t height) 495{ 496 uint32_t utile_h = v3d_utile_height(rsc->cpp); 497 uint32_t uif_block_h = utile_h * 2; 498 uint32_t height_ub = height / uif_block_h; 499 500 uint32_t height_offset_in_pc = height_ub % PAGE_CACHE_UB_ROWS; 501 502 /* For the perfectly-aligned-for-UIF-XOR case, don't add any pad. */ 503 if (height_offset_in_pc == 0) 504 return 0; 505 506 /* Try padding up to where we're offset by at least half a page. */ 507 if (height_offset_in_pc < PAGE_UB_ROWS_TIMES_1_5) { 508 /* If we fit entirely in the page cache, don't pad. */ 509 if (height_ub < PAGE_CACHE_UB_ROWS) 510 return 0; 511 else 512 return PAGE_UB_ROWS_TIMES_1_5 - height_offset_in_pc; 513 } 514 515 /* If we're close to being aligned to page cache size, then round up 516 * and rely on XOR. 517 */ 518 if (height_offset_in_pc > PAGE_CACHE_MINUS_1_5_UB_ROWS) 519 return PAGE_CACHE_UB_ROWS - height_offset_in_pc; 520 521 /* Otherwise, we're far enough away (top and bottom) to not need any 522 * padding. 523 */ 524 return 0; 525} 526 527static void 528v3d_setup_slices(struct v3d_resource *rsc, uint32_t winsys_stride, 529 bool uif_top) 530{ 531 struct pipe_resource *prsc = &rsc->base; 532 uint32_t width = prsc->width0; 533 uint32_t height = prsc->height0; 534 uint32_t depth = prsc->depth0; 535 /* Note that power-of-two padding is based on level 1. These are not 536 * equivalent to just util_next_power_of_two(dimension), because at a 537 * level 0 dimension of 9, the level 1 power-of-two padded value is 4, 538 * not 8. 539 */ 540 uint32_t pot_width = 2 * util_next_power_of_two(u_minify(width, 1)); 541 uint32_t pot_height = 2 * util_next_power_of_two(u_minify(height, 1)); 542 uint32_t pot_depth = 2 * util_next_power_of_two(u_minify(depth, 1)); 543 uint32_t offset = 0; 544 uint32_t utile_w = v3d_utile_width(rsc->cpp); 545 uint32_t utile_h = v3d_utile_height(rsc->cpp); 546 uint32_t uif_block_w = utile_w * 2; 547 uint32_t uif_block_h = utile_h * 2; 548 uint32_t block_width = util_format_get_blockwidth(prsc->format); 549 uint32_t block_height = util_format_get_blockheight(prsc->format); 550 bool msaa = prsc->nr_samples > 1; 551 552 /* MSAA textures/renderbuffers are always laid out as single-level 553 * UIF. 554 */ 555 uif_top |= msaa; 556 557 /* Check some easy mistakes to make in a resource_create() call that 558 * will break our setup. 559 */ 560 assert(prsc->array_size != 0); 561 assert(prsc->depth0 != 0); 562 563 for (int i = prsc->last_level; i >= 0; i--) { 564 struct v3d_resource_slice *slice = &rsc->slices[i]; 565 566 uint32_t level_width, level_height, level_depth; 567 if (i < 2) { 568 level_width = u_minify(width, i); 569 level_height = u_minify(height, i); 570 } else { 571 level_width = u_minify(pot_width, i); 572 level_height = u_minify(pot_height, i); 573 } 574 if (i < 1) 575 level_depth = u_minify(depth, i); 576 else 577 level_depth = u_minify(pot_depth, i); 578 579 if (msaa) { 580 level_width *= 2; 581 level_height *= 2; 582 } 583 584 level_width = DIV_ROUND_UP(level_width, block_width); 585 level_height = DIV_ROUND_UP(level_height, block_height); 586 587 if (!rsc->tiled) { 588 slice->tiling = V3D_TILING_RASTER; 589 if (prsc->target == PIPE_TEXTURE_1D) 590 level_width = align(level_width, 64 / rsc->cpp); 591 } else { 592 if ((i != 0 || !uif_top) && 593 (level_width <= utile_w || 594 level_height <= utile_h)) { 595 slice->tiling = V3D_TILING_LINEARTILE; 596 level_width = align(level_width, utile_w); 597 level_height = align(level_height, utile_h); 598 } else if ((i != 0 || !uif_top) && 599 level_width <= uif_block_w) { 600 slice->tiling = V3D_TILING_UBLINEAR_1_COLUMN; 601 level_width = align(level_width, uif_block_w); 602 level_height = align(level_height, uif_block_h); 603 } else if ((i != 0 || !uif_top) && 604 level_width <= 2 * uif_block_w) { 605 slice->tiling = V3D_TILING_UBLINEAR_2_COLUMN; 606 level_width = align(level_width, 2 * uif_block_w); 607 level_height = align(level_height, uif_block_h); 608 } else { 609 /* We align the width to a 4-block column of 610 * UIF blocks, but we only align height to UIF 611 * blocks. 612 */ 613 level_width = align(level_width, 614 4 * uif_block_w); 615 level_height = align(level_height, 616 uif_block_h); 617 618 slice->ub_pad = v3d_get_ub_pad(rsc, 619 level_height); 620 level_height += slice->ub_pad * uif_block_h; 621 622 /* If the padding set us to to be aligned to 623 * the page cache size, then the HW will use 624 * the XOR bit on odd columns to get us 625 * perfectly misaligned 626 */ 627 if ((level_height / uif_block_h) % 628 (V3D_PAGE_CACHE_SIZE / 629 V3D_UIFBLOCK_ROW_SIZE) == 0) { 630 slice->tiling = V3D_TILING_UIF_XOR; 631 } else { 632 slice->tiling = V3D_TILING_UIF_NO_XOR; 633 } 634 } 635 } 636 637 slice->offset = offset; 638 if (winsys_stride) 639 slice->stride = winsys_stride; 640 else 641 slice->stride = level_width * rsc->cpp; 642 slice->padded_height = level_height; 643 slice->size = level_height * slice->stride; 644 645 uint32_t slice_total_size = slice->size * level_depth; 646 647 /* The HW aligns level 1's base to a page if any of level 1 or 648 * below could be UIF XOR. The lower levels then inherit the 649 * alignment for as long as necessary, thanks to being power of 650 * two aligned. 651 */ 652 if (i == 1 && 653 level_width > 4 * uif_block_w && 654 level_height > PAGE_CACHE_MINUS_1_5_UB_ROWS * uif_block_h) { 655 slice_total_size = align(slice_total_size, 656 V3D_UIFCFG_PAGE_SIZE); 657 } 658 659 offset += slice_total_size; 660 661 } 662 rsc->size = offset; 663 664 /* UIF/UBLINEAR levels need to be aligned to UIF-blocks, and LT only 665 * needs to be aligned to utile boundaries. Since tiles are laid out 666 * from small to big in memory, we need to align the later UIF slices 667 * to UIF blocks, if they were preceded by non-UIF-block-aligned LT 668 * slices. 669 * 670 * We additionally align to 4k, which improves UIF XOR performance. 671 */ 672 uint32_t page_align_offset = (align(rsc->slices[0].offset, 4096) - 673 rsc->slices[0].offset); 674 if (page_align_offset) { 675 rsc->size += page_align_offset; 676 for (int i = 0; i <= prsc->last_level; i++) 677 rsc->slices[i].offset += page_align_offset; 678 } 679 680 /* Arrays and cube textures have a stride which is the distance from 681 * one full mipmap tree to the next (64b aligned). For 3D textures, 682 * we need to program the stride between slices of miplevel 0. 683 */ 684 if (prsc->target != PIPE_TEXTURE_3D) { 685 rsc->cube_map_stride = align(rsc->slices[0].offset + 686 rsc->slices[0].size, 64); 687 rsc->size += rsc->cube_map_stride * (prsc->array_size - 1); 688 } else { 689 rsc->cube_map_stride = rsc->slices[0].size; 690 } 691} 692 693uint32_t 694v3d_layer_offset(struct pipe_resource *prsc, uint32_t level, uint32_t layer) 695{ 696 struct v3d_resource *rsc = v3d_resource(prsc); 697 struct v3d_resource_slice *slice = &rsc->slices[level]; 698 699 if (prsc->target == PIPE_TEXTURE_3D) 700 return slice->offset + layer * slice->size; 701 else 702 return slice->offset + layer * rsc->cube_map_stride; 703} 704 705static struct v3d_resource * 706v3d_resource_setup(struct pipe_screen *pscreen, 707 const struct pipe_resource *tmpl) 708{ 709 struct v3d_screen *screen = v3d_screen(pscreen); 710 struct v3d_resource *rsc = CALLOC_STRUCT(v3d_resource); 711 if (!rsc) 712 return NULL; 713 struct pipe_resource *prsc = &rsc->base; 714 715 *prsc = *tmpl; 716 717 pipe_reference_init(&prsc->reference, 1); 718 prsc->screen = pscreen; 719 720 if (prsc->nr_samples <= 1 || 721 screen->devinfo.ver >= 40 || 722 util_format_is_depth_or_stencil(prsc->format)) { 723 rsc->cpp = util_format_get_blocksize(prsc->format); 724 if (screen->devinfo.ver < 40 && prsc->nr_samples > 1) 725 rsc->cpp *= prsc->nr_samples; 726 } else { 727 assert(v3d_rt_format_supported(&screen->devinfo, prsc->format)); 728 uint32_t output_image_format = 729 v3d_get_rt_format(&screen->devinfo, prsc->format); 730 uint32_t internal_type; 731 uint32_t internal_bpp; 732 v3d_get_internal_type_bpp_for_output_format(&screen->devinfo, 733 output_image_format, 734 &internal_type, 735 &internal_bpp); 736 switch (internal_bpp) { 737 case V3D_INTERNAL_BPP_32: 738 rsc->cpp = 4; 739 break; 740 case V3D_INTERNAL_BPP_64: 741 rsc->cpp = 8; 742 break; 743 case V3D_INTERNAL_BPP_128: 744 rsc->cpp = 16; 745 break; 746 } 747 } 748 749 rsc->serial_id++; 750 751 assert(rsc->cpp); 752 753 return rsc; 754} 755 756static struct pipe_resource * 757v3d_resource_create_with_modifiers(struct pipe_screen *pscreen, 758 const struct pipe_resource *tmpl, 759 const uint64_t *modifiers, 760 int count) 761{ 762 struct v3d_screen *screen = v3d_screen(pscreen); 763 764 bool linear_ok = drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count); 765 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl); 766 struct pipe_resource *prsc = &rsc->base; 767 /* Use a tiled layout if we can, for better 3D performance. */ 768 bool should_tile = true; 769 770 assert(tmpl->target != PIPE_BUFFER || 771 (tmpl->format == PIPE_FORMAT_NONE || 772 util_format_get_blocksize(tmpl->format) == 1)); 773 774 /* VBOs/PBOs/Texture Buffer Objects are untiled (and 1 height). */ 775 if (tmpl->target == PIPE_BUFFER) 776 should_tile = false; 777 778 /* Cursors are always linear, and the user can request linear as well. 779 */ 780 if (tmpl->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR)) 781 should_tile = false; 782 783 /* 1D and 1D_ARRAY textures are always raster-order. */ 784 if (tmpl->target == PIPE_TEXTURE_1D || 785 tmpl->target == PIPE_TEXTURE_1D_ARRAY) 786 should_tile = false; 787 788 /* Scanout BOs for simulator need to be linear for interaction with 789 * i965. 790 */ 791 if (using_v3d_simulator && 792 tmpl->bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT)) 793 should_tile = false; 794 795 /* If using the old-school SCANOUT flag, we don't know what the screen 796 * might support other than linear. Just force linear. 797 */ 798 if (tmpl->bind & PIPE_BIND_SCANOUT) 799 should_tile = false; 800 801 /* No user-specified modifier; determine our own. */ 802 if (count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID) { 803 linear_ok = true; 804 rsc->tiled = should_tile; 805 } else if (should_tile && 806 drm_find_modifier(DRM_FORMAT_MOD_BROADCOM_UIF, 807 modifiers, count)) { 808 rsc->tiled = true; 809 } else if (linear_ok) { 810 rsc->tiled = false; 811 } else { 812 fprintf(stderr, "Unsupported modifier requested\n"); 813 goto fail; 814 } 815 816 rsc->internal_format = prsc->format; 817 818 v3d_setup_slices(rsc, 0, tmpl->bind & PIPE_BIND_SHARED); 819 820 if (screen->ro && (tmpl->bind & PIPE_BIND_SCANOUT)) { 821 struct winsys_handle handle; 822 struct pipe_resource scanout_tmpl = { 823 .target = prsc->target, 824 .format = PIPE_FORMAT_RGBA8888_UNORM, 825 .width0 = 1024, /* one page */ 826 .height0 = align(rsc->size, 4096) / 4096, 827 .depth0 = 1, 828 .array_size = 1, 829 }; 830 831 rsc->scanout = 832 renderonly_scanout_for_resource(&scanout_tmpl, 833 screen->ro, 834 &handle); 835 836 if (!rsc->scanout) { 837 fprintf(stderr, "Failed to create scanout resource\n"); 838 goto fail; 839 } 840 assert(handle.type == WINSYS_HANDLE_TYPE_FD); 841 rsc->bo = v3d_bo_open_dmabuf(screen, handle.handle); 842 close(handle.handle); 843 844 if (!rsc->bo) 845 goto fail; 846 847 v3d_debug_resource_layout(rsc, "renderonly"); 848 849 return prsc; 850 } else { 851 if (!v3d_resource_bo_alloc(rsc)) 852 goto fail; 853 } 854 855 return prsc; 856fail: 857 v3d_resource_destroy(pscreen, prsc); 858 return NULL; 859} 860 861struct pipe_resource * 862v3d_resource_create(struct pipe_screen *pscreen, 863 const struct pipe_resource *tmpl) 864{ 865 const uint64_t mod = DRM_FORMAT_MOD_INVALID; 866 return v3d_resource_create_with_modifiers(pscreen, tmpl, &mod, 1); 867} 868 869static struct pipe_resource * 870v3d_resource_from_handle(struct pipe_screen *pscreen, 871 const struct pipe_resource *tmpl, 872 struct winsys_handle *whandle, 873 unsigned usage) 874{ 875 struct v3d_screen *screen = v3d_screen(pscreen); 876 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl); 877 struct pipe_resource *prsc = &rsc->base; 878 struct v3d_resource_slice *slice = &rsc->slices[0]; 879 880 if (!rsc) 881 return NULL; 882 883 switch (whandle->modifier) { 884 case DRM_FORMAT_MOD_LINEAR: 885 rsc->tiled = false; 886 break; 887 case DRM_FORMAT_MOD_BROADCOM_UIF: 888 rsc->tiled = true; 889 break; 890 case DRM_FORMAT_MOD_INVALID: 891 rsc->tiled = screen->ro == NULL; 892 break; 893 default: 894 switch(fourcc_mod_broadcom_mod(whandle->modifier)) { 895 case DRM_FORMAT_MOD_BROADCOM_SAND128: 896 rsc->tiled = false; 897 rsc->sand_col128_stride = 898 fourcc_mod_broadcom_param(whandle->modifier); 899 break; 900 default: 901 fprintf(stderr, 902 "Attempt to import unsupported modifier 0x%llx\n", 903 (long long)whandle->modifier); 904 goto fail; 905 } 906 } 907 908 switch (whandle->type) { 909 case WINSYS_HANDLE_TYPE_SHARED: 910 rsc->bo = v3d_bo_open_name(screen, whandle->handle); 911 break; 912 case WINSYS_HANDLE_TYPE_FD: 913 rsc->bo = v3d_bo_open_dmabuf(screen, whandle->handle); 914 break; 915 default: 916 fprintf(stderr, 917 "Attempt to import unsupported handle type %d\n", 918 whandle->type); 919 goto fail; 920 } 921 922 if (!rsc->bo) 923 goto fail; 924 925 rsc->internal_format = prsc->format; 926 927 v3d_setup_slices(rsc, whandle->stride, true); 928 v3d_debug_resource_layout(rsc, "import"); 929 930 if (whandle->offset != 0) { 931 if (rsc->tiled) { 932 fprintf(stderr, 933 "Attempt to import unsupported winsys offset %u\n", 934 whandle->offset); 935 goto fail; 936 } 937 rsc->slices[0].offset += whandle->offset; 938 939 if (rsc->slices[0].offset + rsc->slices[0].size > 940 rsc->bo->size) { 941 fprintf(stderr, "Attempt to import " 942 "with overflowing offset (%d + %d > %d)\n", 943 whandle->offset, 944 rsc->slices[0].size, 945 rsc->bo->size); 946 goto fail; 947 } 948 } 949 950 if (screen->ro) { 951 /* Make sure that renderonly has a handle to our buffer in the 952 * display's fd, so that a later renderonly_get_handle() 953 * returns correct handles or GEM names. 954 */ 955 rsc->scanout = 956 renderonly_create_gpu_import_for_resource(prsc, 957 screen->ro, 958 NULL); 959 } 960 961 if (rsc->tiled && whandle->stride != slice->stride) { 962 static bool warned = false; 963 if (!warned) { 964 warned = true; 965 fprintf(stderr, 966 "Attempting to import %dx%d %s with " 967 "unsupported stride %d instead of %d\n", 968 prsc->width0, prsc->height0, 969 util_format_short_name(prsc->format), 970 whandle->stride, 971 slice->stride); 972 } 973 goto fail; 974 } else if (!rsc->tiled) { 975 slice->stride = whandle->stride; 976 } 977 978 return prsc; 979 980fail: 981 v3d_resource_destroy(pscreen, prsc); 982 return NULL; 983} 984 985void 986v3d_update_shadow_texture(struct pipe_context *pctx, 987 struct pipe_sampler_view *pview) 988{ 989 struct v3d_context *v3d = v3d_context(pctx); 990 struct v3d_sampler_view *view = v3d_sampler_view(pview); 991 struct v3d_resource *shadow = v3d_resource(view->texture); 992 struct v3d_resource *orig = v3d_resource(pview->texture); 993 994 assert(view->texture != pview->texture); 995 996 if (shadow->writes == orig->writes && orig->bo->private) 997 return; 998 999 perf_debug("Updating %dx%d@%d shadow for linear texture\n", 1000 orig->base.width0, orig->base.height0, 1001 pview->u.tex.first_level); 1002 1003 for (int i = 0; i <= shadow->base.last_level; i++) { 1004 unsigned width = u_minify(shadow->base.width0, i); 1005 unsigned height = u_minify(shadow->base.height0, i); 1006 struct pipe_blit_info info = { 1007 .dst = { 1008 .resource = &shadow->base, 1009 .level = i, 1010 .box = { 1011 .x = 0, 1012 .y = 0, 1013 .z = 0, 1014 .width = width, 1015 .height = height, 1016 .depth = 1, 1017 }, 1018 .format = shadow->base.format, 1019 }, 1020 .src = { 1021 .resource = &orig->base, 1022 .level = pview->u.tex.first_level + i, 1023 .box = { 1024 .x = 0, 1025 .y = 0, 1026 .z = 0, 1027 .width = width, 1028 .height = height, 1029 .depth = 1, 1030 }, 1031 .format = orig->base.format, 1032 }, 1033 .mask = util_format_get_mask(orig->base.format), 1034 }; 1035 pctx->blit(pctx, &info); 1036 } 1037 1038 shadow->writes = orig->writes; 1039} 1040 1041static struct pipe_surface * 1042v3d_create_surface(struct pipe_context *pctx, 1043 struct pipe_resource *ptex, 1044 const struct pipe_surface *surf_tmpl) 1045{ 1046 struct v3d_context *v3d = v3d_context(pctx); 1047 struct v3d_screen *screen = v3d->screen; 1048 struct v3d_surface *surface = CALLOC_STRUCT(v3d_surface); 1049 struct v3d_resource *rsc = v3d_resource(ptex); 1050 1051 if (!surface) 1052 return NULL; 1053 1054 struct pipe_surface *psurf = &surface->base; 1055 unsigned level = surf_tmpl->u.tex.level; 1056 struct v3d_resource_slice *slice = &rsc->slices[level]; 1057 1058 pipe_reference_init(&psurf->reference, 1); 1059 pipe_resource_reference(&psurf->texture, ptex); 1060 1061 psurf->context = pctx; 1062 psurf->format = surf_tmpl->format; 1063 psurf->width = u_minify(ptex->width0, level); 1064 psurf->height = u_minify(ptex->height0, level); 1065 psurf->u.tex.level = level; 1066 psurf->u.tex.first_layer = surf_tmpl->u.tex.first_layer; 1067 psurf->u.tex.last_layer = surf_tmpl->u.tex.last_layer; 1068 1069 surface->offset = v3d_layer_offset(ptex, level, 1070 psurf->u.tex.first_layer); 1071 surface->tiling = slice->tiling; 1072 1073 surface->format = v3d_get_rt_format(&screen->devinfo, psurf->format); 1074 1075 const struct util_format_description *desc = 1076 util_format_description(psurf->format); 1077 1078 surface->swap_rb = (desc->swizzle[0] == PIPE_SWIZZLE_Z && 1079 psurf->format != PIPE_FORMAT_B5G6R5_UNORM); 1080 1081 if (util_format_is_depth_or_stencil(psurf->format)) { 1082 switch (psurf->format) { 1083 case PIPE_FORMAT_Z16_UNORM: 1084 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_16; 1085 break; 1086 case PIPE_FORMAT_Z32_FLOAT: 1087 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: 1088 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_32F; 1089 break; 1090 default: 1091 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_24; 1092 } 1093 } else { 1094 uint32_t bpp, type; 1095 v3d_get_internal_type_bpp_for_output_format(&screen->devinfo, 1096 surface->format, 1097 &type, &bpp); 1098 surface->internal_type = type; 1099 surface->internal_bpp = bpp; 1100 } 1101 1102 if (surface->tiling == V3D_TILING_UIF_NO_XOR || 1103 surface->tiling == V3D_TILING_UIF_XOR) { 1104 surface->padded_height_of_output_image_in_uif_blocks = 1105 (slice->padded_height / 1106 (2 * v3d_utile_height(rsc->cpp))); 1107 } 1108 1109 if (rsc->separate_stencil) { 1110 surface->separate_stencil = 1111 v3d_create_surface(pctx, &rsc->separate_stencil->base, 1112 surf_tmpl); 1113 } 1114 1115 return &surface->base; 1116} 1117 1118static void 1119v3d_surface_destroy(struct pipe_context *pctx, struct pipe_surface *psurf) 1120{ 1121 struct v3d_surface *surf = v3d_surface(psurf); 1122 1123 if (surf->separate_stencil) 1124 pipe_surface_reference(&surf->separate_stencil, NULL); 1125 1126 pipe_resource_reference(&psurf->texture, NULL); 1127 FREE(psurf); 1128} 1129 1130static void 1131v3d_flush_resource(struct pipe_context *pctx, struct pipe_resource *resource) 1132{ 1133 /* All calls to flush_resource are followed by a flush of the context, 1134 * so there's nothing to do. 1135 */ 1136} 1137 1138static enum pipe_format 1139v3d_resource_get_internal_format(struct pipe_resource *prsc) 1140{ 1141 return v3d_resource(prsc)->internal_format; 1142} 1143 1144static void 1145v3d_resource_set_stencil(struct pipe_resource *prsc, 1146 struct pipe_resource *stencil) 1147{ 1148 v3d_resource(prsc)->separate_stencil = v3d_resource(stencil); 1149} 1150 1151static struct pipe_resource * 1152v3d_resource_get_stencil(struct pipe_resource *prsc) 1153{ 1154 struct v3d_resource *rsc = v3d_resource(prsc); 1155 1156 return &rsc->separate_stencil->base; 1157} 1158 1159static const struct u_transfer_vtbl transfer_vtbl = { 1160 .resource_create = v3d_resource_create, 1161 .resource_destroy = v3d_resource_destroy, 1162 .transfer_map = v3d_resource_transfer_map, 1163 .transfer_unmap = v3d_resource_transfer_unmap, 1164 .transfer_flush_region = u_default_transfer_flush_region, 1165 .get_internal_format = v3d_resource_get_internal_format, 1166 .set_stencil = v3d_resource_set_stencil, 1167 .get_stencil = v3d_resource_get_stencil, 1168}; 1169 1170void 1171v3d_resource_screen_init(struct pipe_screen *pscreen) 1172{ 1173 pscreen->resource_create_with_modifiers = 1174 v3d_resource_create_with_modifiers; 1175 pscreen->resource_create = u_transfer_helper_resource_create; 1176 pscreen->resource_from_handle = v3d_resource_from_handle; 1177 pscreen->resource_get_handle = v3d_resource_get_handle; 1178 pscreen->resource_get_param = v3d_resource_get_param; 1179 pscreen->resource_destroy = u_transfer_helper_resource_destroy; 1180 pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl, 1181 true, false, 1182 true, true, 1183 false); 1184} 1185 1186void 1187v3d_resource_context_init(struct pipe_context *pctx) 1188{ 1189 pctx->buffer_map = u_transfer_helper_transfer_map; 1190 pctx->texture_map = u_transfer_helper_transfer_map; 1191 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region; 1192 pctx->buffer_unmap = u_transfer_helper_transfer_unmap; 1193 pctx->texture_unmap = u_transfer_helper_transfer_unmap; 1194 pctx->buffer_subdata = u_default_buffer_subdata; 1195 pctx->texture_subdata = v3d_texture_subdata; 1196 pctx->create_surface = v3d_create_surface; 1197 pctx->surface_destroy = v3d_surface_destroy; 1198 pctx->resource_copy_region = util_resource_copy_region; 1199 pctx->blit = v3d_blit; 1200 pctx->generate_mipmap = v3d_generate_mipmap; 1201 pctx->flush_resource = v3d_flush_resource; 1202} 1203