1/* 2 * Copyright (C) 2008 VMware, Inc. 3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org> 4 * Copyright (C) 2014-2017 Broadcom 5 * Copyright (C) 2018-2019 Alyssa Rosenzweig 6 * Copyright (C) 2019 Collabora, Ltd. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the "Software"), 10 * to deal in the Software without restriction, including without limitation 11 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 12 * and/or sell copies of the Software, and to permit persons to whom the 13 * Software is furnished to do so, subject to the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the next 16 * paragraph) shall be included in all copies or substantial portions of the 17 * Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 22 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 23 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 24 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 25 * SOFTWARE. 26 * 27 * Authors (Collabora): 28 * Tomeu Vizoso <tomeu.vizoso@collabora.com> 29 * Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com> 30 * 31 */ 32 33#include <xf86drm.h> 34#include <fcntl.h> 35#include "drm-uapi/drm_fourcc.h" 36 37#include "frontend/winsys_handle.h" 38#include "util/format/u_format.h" 39#include "util/u_memory.h" 40#include "util/u_surface.h" 41#include "util/u_transfer.h" 42#include "util/u_transfer_helper.h" 43#include "util/u_gen_mipmap.h" 44#include "util/u_drm.h" 45 46#include "pan_bo.h" 47#include "pan_context.h" 48#include "pan_screen.h" 49#include "pan_resource.h" 50#include "pan_util.h" 51#include "pan_tiling.h" 52#include "decode.h" 53 54static bool 55panfrost_should_checksum(const struct panfrost_device *dev, const struct panfrost_resource *pres); 56 57static struct pipe_resource * 58panfrost_resource_from_handle(struct pipe_screen *pscreen, 59 const struct pipe_resource *templat, 60 struct winsys_handle *whandle, 61 unsigned usage) 62{ 63 struct panfrost_device *dev = pan_device(pscreen); 64 struct panfrost_resource *rsc; 65 struct pipe_resource *prsc; 66 67 assert(whandle->type == WINSYS_HANDLE_TYPE_FD); 68 69 rsc = CALLOC_STRUCT(panfrost_resource); 70 if (!rsc) 71 return NULL; 72 73 prsc = &rsc->base; 74 75 *prsc = *templat; 76 77 pipe_reference_init(&prsc->reference, 1); 78 prsc->screen = pscreen; 79 80 uint64_t mod = whandle->modifier == DRM_FORMAT_MOD_INVALID ? 81 DRM_FORMAT_MOD_LINEAR : whandle->modifier; 82 enum mali_texture_dimension dim = 83 panfrost_translate_texture_dimension(templat->target); 84 enum pan_image_crc_mode crc_mode = 85 panfrost_should_checksum(dev, rsc) ? 86 PAN_IMAGE_CRC_OOB : PAN_IMAGE_CRC_NONE; 87 struct pan_image_explicit_layout explicit_layout = { 88 .offset = whandle->offset, 89 .row_stride = panfrost_from_legacy_stride(whandle->stride, templat->format, mod) 90 }; 91 92 rsc->image.layout = (struct pan_image_layout) { 93 .modifier = mod, 94 .format = templat->format, 95 .dim = dim, 96 .width = prsc->width0, 97 .height = prsc->height0, 98 .depth = prsc->depth0, 99 .array_size = prsc->array_size, 100 .nr_samples = MAX2(prsc->nr_samples, 1), 101 .nr_slices = 1, 102 .crc_mode = crc_mode 103 }; 104 105 bool valid = pan_image_layout_init(&rsc->image.layout, &explicit_layout); 106 107 if (!valid) { 108 FREE(rsc); 109 return NULL; 110 } 111 112 rsc->image.data.bo = panfrost_bo_import(dev, whandle->handle); 113 /* Sometimes an import can fail e.g. on an invalid buffer fd, out of 114 * memory space to mmap it etc. 115 */ 116 if (!rsc->image.data.bo) { 117 FREE(rsc); 118 return NULL; 119 } 120 if (rsc->image.layout.crc_mode == PAN_IMAGE_CRC_OOB) 121 rsc->image.crc.bo = panfrost_bo_create(dev, rsc->image.layout.crc_size, 0, "CRC data"); 122 123 rsc->modifier_constant = true; 124 125 BITSET_SET(rsc->valid.data, 0); 126 panfrost_resource_set_damage_region(pscreen, &rsc->base, 0, NULL); 127 128 if (dev->ro) { 129 rsc->scanout = 130 renderonly_create_gpu_import_for_resource(prsc, dev->ro, NULL); 131 /* failure is expected in some cases.. */ 132 } 133 134 return prsc; 135} 136 137static bool 138panfrost_resource_get_handle(struct pipe_screen *pscreen, 139 struct pipe_context *ctx, 140 struct pipe_resource *pt, 141 struct winsys_handle *handle, 142 unsigned usage) 143{ 144 struct panfrost_device *dev = pan_device(pscreen); 145 struct panfrost_resource *rsrc; 146 struct renderonly_scanout *scanout; 147 struct pipe_resource *cur = pt; 148 149 /* Even though panfrost doesn't support multi-planar formats, we 150 * can get here through GBM, which does. Walk the list of planes 151 * to find the right one. 152 */ 153 for (int i = 0; i < handle->plane; i++) { 154 cur = cur->next; 155 if (!cur) 156 return false; 157 } 158 rsrc = pan_resource(cur); 159 scanout = rsrc->scanout; 160 161 handle->modifier = rsrc->image.layout.modifier; 162 rsrc->modifier_constant = true; 163 164 if (handle->type == WINSYS_HANDLE_TYPE_KMS && dev->ro) { 165 return renderonly_get_handle(scanout, handle); 166 } else if (handle->type == WINSYS_HANDLE_TYPE_KMS) { 167 handle->handle = rsrc->image.data.bo->gem_handle; 168 } else if (handle->type == WINSYS_HANDLE_TYPE_FD) { 169 int fd = panfrost_bo_export(rsrc->image.data.bo); 170 171 if (fd < 0) 172 return false; 173 174 handle->handle = fd; 175 } else { 176 /* Other handle types not supported */ 177 return false; 178 } 179 180 handle->stride = panfrost_get_legacy_stride(&rsrc->image.layout, 0); 181 handle->offset = rsrc->image.layout.slices[0].offset; 182 return true; 183} 184 185static bool 186panfrost_resource_get_param(struct pipe_screen *pscreen, 187 struct pipe_context *pctx, struct pipe_resource *prsc, 188 unsigned plane, unsigned layer, unsigned level, 189 enum pipe_resource_param param, 190 unsigned usage, uint64_t *value) 191{ 192 struct panfrost_resource *rsrc = (struct panfrost_resource *) prsc; 193 struct pipe_resource *cur; 194 unsigned count; 195 196 switch (param) { 197 case PIPE_RESOURCE_PARAM_STRIDE: 198 *value = panfrost_get_legacy_stride(&rsrc->image.layout, level); 199 return true; 200 case PIPE_RESOURCE_PARAM_OFFSET: 201 *value = rsrc->image.layout.slices[level].offset; 202 return true; 203 case PIPE_RESOURCE_PARAM_MODIFIER: 204 *value = rsrc->image.layout.modifier; 205 return true; 206 case PIPE_RESOURCE_PARAM_NPLANES: 207 /* Panfrost doesn't directly support multi-planar formats, 208 * but we should still handle this case for gbm users 209 * that might want to use resources shared with panfrost 210 * on video processing hardware that does. 211 */ 212 for (count = 0, cur = prsc; cur; cur = cur->next) 213 count++; 214 *value = count; 215 return true; 216 default: 217 return false; 218 } 219} 220 221static void 222panfrost_flush_resource(struct pipe_context *pctx, struct pipe_resource *prsc) 223{ 224 /* TODO */ 225} 226 227static struct pipe_surface * 228panfrost_create_surface(struct pipe_context *pipe, 229 struct pipe_resource *pt, 230 const struct pipe_surface *surf_tmpl) 231{ 232 struct panfrost_context *ctx = pan_context(pipe); 233 struct pipe_surface *ps = NULL; 234 235 pan_legalize_afbc_format(ctx, pan_resource(pt), surf_tmpl->format); 236 237 ps = CALLOC_STRUCT(pipe_surface); 238 239 if (ps) { 240 pipe_reference_init(&ps->reference, 1); 241 pipe_resource_reference(&ps->texture, pt); 242 ps->context = pipe; 243 ps->format = surf_tmpl->format; 244 245 if (pt->target != PIPE_BUFFER) { 246 assert(surf_tmpl->u.tex.level <= pt->last_level); 247 ps->width = u_minify(pt->width0, surf_tmpl->u.tex.level); 248 ps->height = u_minify(pt->height0, surf_tmpl->u.tex.level); 249 ps->nr_samples = surf_tmpl->nr_samples; 250 ps->u.tex.level = surf_tmpl->u.tex.level; 251 ps->u.tex.first_layer = surf_tmpl->u.tex.first_layer; 252 ps->u.tex.last_layer = surf_tmpl->u.tex.last_layer; 253 } else { 254 /* setting width as number of elements should get us correct renderbuffer width */ 255 ps->width = surf_tmpl->u.buf.last_element - surf_tmpl->u.buf.first_element + 1; 256 ps->height = pt->height0; 257 ps->u.buf.first_element = surf_tmpl->u.buf.first_element; 258 ps->u.buf.last_element = surf_tmpl->u.buf.last_element; 259 assert(ps->u.buf.first_element <= ps->u.buf.last_element); 260 assert(ps->u.buf.last_element < ps->width); 261 } 262 } 263 264 return ps; 265} 266 267static void 268panfrost_surface_destroy(struct pipe_context *pipe, 269 struct pipe_surface *surf) 270{ 271 assert(surf->texture); 272 pipe_resource_reference(&surf->texture, NULL); 273 free(surf); 274} 275 276static inline bool 277panfrost_is_2d(const struct panfrost_resource *pres) 278{ 279 return (pres->base.target == PIPE_TEXTURE_2D) 280 || (pres->base.target == PIPE_TEXTURE_RECT); 281} 282 283/* Based on the usage, determine if it makes sense to use u-inteleaved tiling. 284 * We only have routines to tile 2D textures of sane bpps. On the hardware 285 * level, not all usages are valid for tiling. Finally, if the app is hinting 286 * that the contents frequently change, tiling will be a loss. 287 * 288 * On platforms where it is supported, AFBC is even better. */ 289 290static bool 291panfrost_should_afbc(struct panfrost_device *dev, 292 const struct panfrost_resource *pres, 293 enum pipe_format fmt) 294{ 295 /* AFBC resources may be rendered to, textured from, or shared across 296 * processes, but may not be used as e.g buffers */ 297 const unsigned valid_binding = 298 PIPE_BIND_DEPTH_STENCIL | 299 PIPE_BIND_RENDER_TARGET | 300 PIPE_BIND_BLENDABLE | 301 PIPE_BIND_SAMPLER_VIEW | 302 PIPE_BIND_DISPLAY_TARGET | 303 PIPE_BIND_SCANOUT | 304 PIPE_BIND_SHARED; 305 306 if (pres->base.bind & ~valid_binding) 307 return false; 308 309 /* AFBC support is optional */ 310 if (!dev->has_afbc) 311 return false; 312 313 /* AFBC<-->staging is expensive */ 314 if (pres->base.usage == PIPE_USAGE_STREAM) 315 return false; 316 317 /* Only a small selection of formats are AFBC'able */ 318 if (!panfrost_format_supports_afbc(dev, fmt)) 319 return false; 320 321 /* AFBC does not support layered (GLES3 style) multisampling. Use 322 * EXT_multisampled_render_to_texture instead */ 323 if (pres->base.nr_samples > 1) 324 return false; 325 326 switch (pres->base.target) { 327 case PIPE_TEXTURE_2D: 328 case PIPE_TEXTURE_2D_ARRAY: 329 case PIPE_TEXTURE_RECT: 330 break; 331 332 case PIPE_TEXTURE_3D: 333 /* 3D AFBC is only supported on Bifrost v7+. It's supposed to 334 * be supported on Midgard but it doesn't seem to work */ 335 if (dev->arch != 7) 336 return false; 337 338 break; 339 340 default: 341 return false; 342 } 343 344 /* For one tile, AFBC is a loss compared to u-interleaved */ 345 if (pres->base.width0 <= 16 && pres->base.height0 <= 16) 346 return false; 347 348 /* Otherwise, we'd prefer AFBC as it is dramatically more efficient 349 * than linear or usually even u-interleaved */ 350 return true; 351} 352 353/* 354 * For a resource we want to use AFBC with, should we use AFBC with tiled 355 * headers? On GPUs that support it, this is believed to be beneficial for 356 * images that are at least 128x128. 357 */ 358static bool 359panfrost_should_tile_afbc(const struct panfrost_device *dev, 360 const struct panfrost_resource *pres) 361{ 362 return panfrost_afbc_can_tile(dev) && 363 pres->base.width0 >= 128 && 364 pres->base.height0 >= 128; 365} 366 367static bool 368panfrost_should_tile(struct panfrost_device *dev, 369 const struct panfrost_resource *pres, 370 enum pipe_format fmt) 371{ 372 const unsigned valid_binding = 373 PIPE_BIND_DEPTH_STENCIL | 374 PIPE_BIND_RENDER_TARGET | 375 PIPE_BIND_BLENDABLE | 376 PIPE_BIND_SAMPLER_VIEW | 377 PIPE_BIND_DISPLAY_TARGET | 378 PIPE_BIND_SCANOUT | 379 PIPE_BIND_SHARED; 380 381 /* The purpose of tiling is improving locality in both X- and 382 * Y-directions. If there is only a single pixel in either direction, 383 * tiling does not make sense; using a linear layout instead is optimal 384 * for both memory usage and performance. 385 */ 386 if (MIN2(pres->base.width0, pres->base.height0) < 2) 387 return false; 388 389 bool can_tile = (pres->base.target != PIPE_BUFFER) 390 && ((pres->base.bind & ~valid_binding) == 0); 391 392 return can_tile && (pres->base.usage != PIPE_USAGE_STREAM); 393} 394 395static uint64_t 396panfrost_best_modifier(struct panfrost_device *dev, 397 const struct panfrost_resource *pres, 398 enum pipe_format fmt) 399{ 400 /* Force linear textures when debugging tiling/compression */ 401 if (unlikely(dev->debug & PAN_DBG_LINEAR)) 402 return DRM_FORMAT_MOD_LINEAR; 403 404 if (panfrost_should_afbc(dev, pres, fmt)) { 405 uint64_t afbc = 406 AFBC_FORMAT_MOD_BLOCK_SIZE_16x16 | 407 AFBC_FORMAT_MOD_SPARSE; 408 409 if (panfrost_afbc_can_ytr(pres->base.format)) 410 afbc |= AFBC_FORMAT_MOD_YTR; 411 412 if (panfrost_should_tile_afbc(dev, pres)) 413 afbc |= AFBC_FORMAT_MOD_TILED | AFBC_FORMAT_MOD_SC; 414 415 return DRM_FORMAT_MOD_ARM_AFBC(afbc); 416 } else if (panfrost_should_tile(dev, pres, fmt)) 417 return DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED; 418 else 419 return DRM_FORMAT_MOD_LINEAR; 420} 421 422static bool 423panfrost_should_checksum(const struct panfrost_device *dev, const struct panfrost_resource *pres) 424{ 425 /* When checksumming is enabled, the tile data must fit in the 426 * size of the writeback buffer, so don't checksum formats 427 * that use too much space. */ 428 429 unsigned bytes_per_pixel_max = (dev->arch == 6) ? 6 : 4; 430 431 unsigned bytes_per_pixel = MAX2(pres->base.nr_samples, 1) * 432 util_format_get_blocksize(pres->base.format); 433 434 return pres->base.bind & PIPE_BIND_RENDER_TARGET && 435 panfrost_is_2d(pres) && 436 bytes_per_pixel <= bytes_per_pixel_max && 437 pres->base.last_level == 0 && 438 !(dev->debug & PAN_DBG_NO_CRC); 439} 440 441static void 442panfrost_resource_setup(struct panfrost_device *dev, 443 struct panfrost_resource *pres, 444 uint64_t modifier, enum pipe_format fmt) 445{ 446 uint64_t chosen_mod = modifier != DRM_FORMAT_MOD_INVALID ? 447 modifier : panfrost_best_modifier(dev, pres, fmt); 448 enum pan_image_crc_mode crc_mode = 449 panfrost_should_checksum(dev, pres) ? 450 PAN_IMAGE_CRC_INBAND : PAN_IMAGE_CRC_NONE; 451 enum mali_texture_dimension dim = 452 panfrost_translate_texture_dimension(pres->base.target); 453 454 /* We can only switch tiled->linear if the resource isn't already 455 * linear and if we control the modifier */ 456 pres->modifier_constant = 457 !(chosen_mod != DRM_FORMAT_MOD_LINEAR && 458 modifier == DRM_FORMAT_MOD_INVALID); 459 460 /* Z32_S8X24 variants are actually stored in 2 planes (one per 461 * component), we have to adjust the format on the first plane. 462 */ 463 if (fmt == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT) 464 fmt = PIPE_FORMAT_Z32_FLOAT; 465 466 pres->image.layout = (struct pan_image_layout) { 467 .modifier = chosen_mod, 468 .format = fmt, 469 .dim = dim, 470 .width = pres->base.width0, 471 .height = pres->base.height0, 472 .depth = pres->base.depth0, 473 .array_size = pres->base.array_size, 474 .nr_samples = MAX2(pres->base.nr_samples, 1), 475 .nr_slices = pres->base.last_level + 1, 476 .crc_mode = crc_mode 477 }; 478 479 ASSERTED bool valid = pan_image_layout_init(&pres->image.layout, NULL); 480 assert(valid); 481} 482 483static void 484panfrost_resource_init_afbc_headers(struct panfrost_resource *pres) 485{ 486 panfrost_bo_mmap(pres->image.data.bo); 487 488 unsigned nr_samples = MAX2(pres->base.nr_samples, 1); 489 490 for (unsigned i = 0; i < pres->base.array_size; ++i) { 491 for (unsigned l = 0; l <= pres->base.last_level; ++l) { 492 struct pan_image_slice_layout *slice = &pres->image.layout.slices[l]; 493 494 for (unsigned s = 0; s < nr_samples; ++s) { 495 void *ptr = pres->image.data.bo->ptr.cpu + 496 (i * pres->image.layout.array_stride) + 497 slice->offset + 498 (s * slice->afbc.surface_stride); 499 500 /* Zero-ed AFBC headers seem to encode a plain 501 * black. Let's use this pattern to keep the 502 * initialization simple. 503 */ 504 memset(ptr, 0, slice->afbc.header_size); 505 } 506 } 507 } 508} 509 510void 511panfrost_resource_set_damage_region(struct pipe_screen *screen, 512 struct pipe_resource *res, 513 unsigned int nrects, 514 const struct pipe_box *rects) 515{ 516 struct panfrost_device *dev = pan_device(screen); 517 struct panfrost_resource *pres = pan_resource(res); 518 struct pipe_scissor_state *damage_extent = &pres->damage.extent; 519 unsigned int i; 520 521 /* Partial updates are implemented with a tile enable map only on v5. 522 * Later architectures have a more efficient method of implementing 523 * partial updates (frame shaders), while earlier architectures lack 524 * tile enable maps altogether. 525 */ 526 if (dev->arch == 5 && nrects > 1) { 527 if (!pres->damage.tile_map.data) { 528 pres->damage.tile_map.stride = 529 ALIGN_POT(DIV_ROUND_UP(res->width0, 32 * 8), 64); 530 pres->damage.tile_map.size = 531 pres->damage.tile_map.stride * 532 DIV_ROUND_UP(res->height0, 32); 533 pres->damage.tile_map.data = 534 malloc(pres->damage.tile_map.size); 535 } 536 537 memset(pres->damage.tile_map.data, 0, pres->damage.tile_map.size); 538 pres->damage.tile_map.enable = true; 539 } else { 540 pres->damage.tile_map.enable = false; 541 } 542 543 /* Track the damage extent: the quad including all damage regions. Will 544 * be used restrict the rendering area */ 545 546 damage_extent->minx = 0xffff; 547 damage_extent->miny = 0xffff; 548 549 unsigned enable_count = 0; 550 551 for (i = 0; i < nrects; i++) { 552 int x = rects[i].x, w = rects[i].width, h = rects[i].height; 553 int y = res->height0 - (rects[i].y + h); 554 555 damage_extent->minx = MIN2(damage_extent->minx, x); 556 damage_extent->miny = MIN2(damage_extent->miny, y); 557 damage_extent->maxx = MAX2(damage_extent->maxx, 558 MIN2(x + w, res->width0)); 559 damage_extent->maxy = MAX2(damage_extent->maxy, 560 MIN2(y + h, res->height0)); 561 562 if (!pres->damage.tile_map.enable) 563 continue; 564 565 unsigned t_x_start = x / 32; 566 unsigned t_x_end = (x + w - 1) / 32; 567 unsigned t_y_start = y / 32; 568 unsigned t_y_end = (y + h - 1) / 32; 569 570 for (unsigned t_y = t_y_start; t_y <= t_y_end; t_y++) { 571 for (unsigned t_x = t_x_start; t_x <= t_x_end; t_x++) { 572 unsigned b = (t_y * pres->damage.tile_map.stride * 8) + t_x; 573 574 if (BITSET_TEST(pres->damage.tile_map.data, b)) 575 continue; 576 577 BITSET_SET(pres->damage.tile_map.data, b); 578 enable_count++; 579 } 580 } 581 } 582 583 if (nrects == 0) { 584 damage_extent->minx = 0; 585 damage_extent->miny = 0; 586 damage_extent->maxx = res->width0; 587 damage_extent->maxy = res->height0; 588 } 589 590 if (pres->damage.tile_map.enable) { 591 unsigned t_x_start = damage_extent->minx / 32; 592 unsigned t_x_end = damage_extent->maxx / 32; 593 unsigned t_y_start = damage_extent->miny / 32; 594 unsigned t_y_end = damage_extent->maxy / 32; 595 unsigned tile_count = (t_x_end - t_x_start + 1) * 596 (t_y_end - t_y_start + 1); 597 598 /* Don't bother passing a tile-enable-map if the amount of 599 * tiles to reload is to close to the total number of tiles. 600 */ 601 if (tile_count - enable_count < 10) 602 pres->damage.tile_map.enable = false; 603 } 604 605} 606 607static struct pipe_resource * 608panfrost_resource_create_with_modifier(struct pipe_screen *screen, 609 const struct pipe_resource *template, 610 uint64_t modifier) 611{ 612 struct panfrost_device *dev = pan_device(screen); 613 614 struct panfrost_resource *so = CALLOC_STRUCT(panfrost_resource); 615 so->base = *template; 616 so->base.screen = screen; 617 618 pipe_reference_init(&so->base.reference, 1); 619 620 util_range_init(&so->valid_buffer_range); 621 622 if (template->bind & PAN_BIND_SHARED_MASK) { 623 /* For compatibility with older consumers that may not be 624 * modifiers aware, treat INVALID as LINEAR for shared 625 * resources. 626 */ 627 if (modifier == DRM_FORMAT_MOD_INVALID) 628 modifier = DRM_FORMAT_MOD_LINEAR; 629 630 /* At any rate, we can't change the modifier later for shared 631 * resources, since we have no way to propagate the modifier 632 * change. 633 */ 634 so->modifier_constant = true; 635 } 636 637 panfrost_resource_setup(dev, so, modifier, template->format); 638 639 /* Guess a label based on the bind */ 640 unsigned bind = template->bind; 641 const char *label = 642 (bind & PIPE_BIND_INDEX_BUFFER) ? "Index buffer" : 643 (bind & PIPE_BIND_SCANOUT) ? "Scanout" : 644 (bind & PIPE_BIND_DISPLAY_TARGET) ? "Display target" : 645 (bind & PIPE_BIND_SHARED) ? "Shared resource" : 646 (bind & PIPE_BIND_RENDER_TARGET) ? "Render target" : 647 (bind & PIPE_BIND_DEPTH_STENCIL) ? "Depth/stencil buffer" : 648 (bind & PIPE_BIND_SAMPLER_VIEW) ? "Texture" : 649 (bind & PIPE_BIND_VERTEX_BUFFER) ? "Vertex buffer" : 650 (bind & PIPE_BIND_CONSTANT_BUFFER) ? "Constant buffer" : 651 (bind & PIPE_BIND_GLOBAL) ? "Global memory" : 652 (bind & PIPE_BIND_SHADER_BUFFER) ? "Shader buffer" : 653 (bind & PIPE_BIND_SHADER_IMAGE) ? "Shader image" : 654 "Other resource"; 655 656 if (dev->ro && (template->bind & PIPE_BIND_SCANOUT)) { 657 struct winsys_handle handle; 658 struct pan_block_size blocksize = panfrost_block_size(modifier, template->format); 659 660 /* Block-based texture formats are only used for texture 661 * compression (not framebuffer compression!), which doesn't 662 * make sense to share across processes. 663 */ 664 assert(util_format_get_blockwidth(template->format) == 1); 665 assert(util_format_get_blockheight(template->format) == 1); 666 667 /* Present a resource with similar dimensions that, if allocated 668 * as a linear image, is big enough to fit the resource in the 669 * actual layout. For linear images, this is a no-op. For 16x16 670 * tiling, this aligns the dimensions to 16x16. 671 * 672 * For AFBC, this aligns the width to the superblock width (as 673 * expected) and adds extra rows to account for the header. This 674 * is a bit of a lie, but it's the best we can do with dumb 675 * buffers, which are extremely not meant for AFBC. And yet this 676 * has to work anyway... 677 * 678 * Moral of the story: if you're reading this comment, that 679 * means you're working on WSI and so it's already too late for 680 * you. I'm sorry. 681 */ 682 unsigned width = ALIGN_POT(template->width0, blocksize.width); 683 unsigned stride = ALIGN_POT(template->width0, blocksize.width) * 684 util_format_get_blocksize(template->format); 685 unsigned size = so->image.layout.data_size; 686 unsigned effective_rows = DIV_ROUND_UP(size, stride); 687 688 struct pipe_resource scanout_tmpl = { 689 .target = so->base.target, 690 .format = template->format, 691 .width0 = width, 692 .height0 = effective_rows, 693 .depth0 = 1, 694 .array_size = 1, 695 }; 696 697 so->scanout = 698 renderonly_scanout_for_resource(&scanout_tmpl, 699 dev->ro, 700 &handle); 701 702 if (!so->scanout) { 703 fprintf(stderr, "Failed to create scanout resource\n"); 704 free(so); 705 return NULL; 706 } 707 assert(handle.type == WINSYS_HANDLE_TYPE_FD); 708 so->image.data.bo = panfrost_bo_import(dev, handle.handle); 709 close(handle.handle); 710 711 if (!so->image.data.bo) { 712 free(so); 713 return NULL; 714 } 715 } else { 716 /* We create a BO immediately but don't bother mapping, since we don't 717 * care to map e.g. FBOs which the CPU probably won't touch */ 718 719 so->image.data.bo = 720 panfrost_bo_create(dev, so->image.layout.data_size, PAN_BO_DELAY_MMAP, label); 721 722 so->constant_stencil = true; 723 } 724 725 if (drm_is_afbc(so->image.layout.modifier)) 726 panfrost_resource_init_afbc_headers(so); 727 728 panfrost_resource_set_damage_region(screen, &so->base, 0, NULL); 729 730 if (template->bind & PIPE_BIND_INDEX_BUFFER) 731 so->index_cache = CALLOC_STRUCT(panfrost_minmax_cache); 732 733 return (struct pipe_resource *)so; 734} 735 736/* Default is to create a resource as don't care */ 737 738static struct pipe_resource * 739panfrost_resource_create(struct pipe_screen *screen, 740 const struct pipe_resource *template) 741{ 742 return panfrost_resource_create_with_modifier(screen, template, 743 DRM_FORMAT_MOD_INVALID); 744} 745 746/* If no modifier is specified, we'll choose. Otherwise, the order of 747 * preference is compressed, tiled, linear. */ 748 749static struct pipe_resource * 750panfrost_resource_create_with_modifiers(struct pipe_screen *screen, 751 const struct pipe_resource *template, 752 const uint64_t *modifiers, int count) 753{ 754 for (unsigned i = 0; i < PAN_MODIFIER_COUNT; ++i) { 755 if (drm_find_modifier(pan_best_modifiers[i], modifiers, count)) { 756 return panfrost_resource_create_with_modifier(screen, template, 757 pan_best_modifiers[i]); 758 } 759 } 760 761 /* If we didn't find one, app specified invalid */ 762 assert(count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID); 763 return panfrost_resource_create(screen, template); 764} 765 766static void 767panfrost_resource_destroy(struct pipe_screen *screen, 768 struct pipe_resource *pt) 769{ 770 struct panfrost_device *dev = pan_device(screen); 771 struct panfrost_resource *rsrc = (struct panfrost_resource *) pt; 772 773 if (rsrc->scanout) 774 renderonly_scanout_destroy(rsrc->scanout, dev->ro); 775 776 if (rsrc->image.data.bo) 777 panfrost_bo_unreference(rsrc->image.data.bo); 778 779 if (rsrc->image.crc.bo) 780 panfrost_bo_unreference(rsrc->image.crc.bo); 781 782 free(rsrc->index_cache); 783 free(rsrc->damage.tile_map.data); 784 785 util_range_destroy(&rsrc->valid_buffer_range); 786 free(rsrc); 787} 788 789/* Most of the time we can do CPU-side transfers, but sometimes we need to use 790 * the 3D pipe for this. Let's wrap u_blitter to blit to/from staging textures. 791 * Code adapted from freedreno */ 792 793static struct panfrost_resource * 794pan_alloc_staging(struct panfrost_context *ctx, struct panfrost_resource *rsc, 795 unsigned level, const struct pipe_box *box) 796{ 797 struct pipe_context *pctx = &ctx->base; 798 struct pipe_resource tmpl = rsc->base; 799 800 tmpl.width0 = box->width; 801 tmpl.height0 = box->height; 802 /* for array textures, box->depth is the array_size, otherwise 803 * for 3d textures, it is the depth: 804 */ 805 if (tmpl.array_size > 1) { 806 if (tmpl.target == PIPE_TEXTURE_CUBE) 807 tmpl.target = PIPE_TEXTURE_2D_ARRAY; 808 tmpl.array_size = box->depth; 809 tmpl.depth0 = 1; 810 } else { 811 tmpl.array_size = 1; 812 tmpl.depth0 = box->depth; 813 } 814 tmpl.last_level = 0; 815 tmpl.bind |= PIPE_BIND_LINEAR; 816 tmpl.bind &= ~PAN_BIND_SHARED_MASK; 817 818 struct pipe_resource *pstaging = 819 pctx->screen->resource_create(pctx->screen, &tmpl); 820 if (!pstaging) 821 return NULL; 822 823 return pan_resource(pstaging); 824} 825 826static enum pipe_format 827pan_blit_format(enum pipe_format fmt) 828{ 829 const struct util_format_description *desc; 830 desc = util_format_description(fmt); 831 832 /* This must be an emulated format (using u_transfer_helper) as if it 833 * was real RGTC we wouldn't have used AFBC and needed a blit. */ 834 if (desc->layout == UTIL_FORMAT_LAYOUT_RGTC) 835 fmt = PIPE_FORMAT_R8G8B8A8_UNORM; 836 837 return fmt; 838} 839 840static void 841pan_blit_from_staging(struct pipe_context *pctx, struct panfrost_transfer *trans) 842{ 843 struct pipe_resource *dst = trans->base.resource; 844 struct pipe_blit_info blit = {0}; 845 846 blit.dst.resource = dst; 847 blit.dst.format = pan_blit_format(dst->format); 848 blit.dst.level = trans->base.level; 849 blit.dst.box = trans->base.box; 850 blit.src.resource = trans->staging.rsrc; 851 blit.src.format = pan_blit_format(trans->staging.rsrc->format); 852 blit.src.level = 0; 853 blit.src.box = trans->staging.box; 854 blit.mask = util_format_get_mask(blit.src.format); 855 blit.filter = PIPE_TEX_FILTER_NEAREST; 856 857 panfrost_blit(pctx, &blit); 858} 859 860static void 861pan_blit_to_staging(struct pipe_context *pctx, struct panfrost_transfer *trans) 862{ 863 struct pipe_resource *src = trans->base.resource; 864 struct pipe_blit_info blit = {0}; 865 866 blit.src.resource = src; 867 blit.src.format = pan_blit_format(src->format); 868 blit.src.level = trans->base.level; 869 blit.src.box = trans->base.box; 870 blit.dst.resource = trans->staging.rsrc; 871 blit.dst.format = pan_blit_format(trans->staging.rsrc->format); 872 blit.dst.level = 0; 873 blit.dst.box = trans->staging.box; 874 blit.mask = util_format_get_mask(blit.dst.format); 875 blit.filter = PIPE_TEX_FILTER_NEAREST; 876 877 panfrost_blit(pctx, &blit); 878} 879 880static void 881panfrost_load_tiled_images(struct panfrost_transfer *transfer, 882 struct panfrost_resource *rsrc) 883{ 884 struct pipe_transfer *ptrans = &transfer->base; 885 unsigned level = ptrans->level; 886 887 /* If the requested level of the image is uninitialized, it's not 888 * necessary to copy it. Leave the result unintiialized too. 889 */ 890 if (!BITSET_TEST(rsrc->valid.data, level)) 891 return; 892 893 struct panfrost_bo *bo = rsrc->image.data.bo; 894 unsigned stride = panfrost_get_layer_stride(&rsrc->image.layout, level); 895 896 /* Otherwise, load each layer separately, required to load from 3D and 897 * array textures. 898 */ 899 for (unsigned z = 0; z < ptrans->box.depth; ++z) { 900 void *dst = transfer->map + (ptrans->layer_stride * z); 901 uint8_t *map = bo->ptr.cpu + 902 rsrc->image.layout.slices[level].offset + 903 (z + ptrans->box.z) * stride; 904 905 panfrost_load_tiled_image(dst, map, ptrans->box.x, 906 ptrans->box.y, ptrans->box.width, 907 ptrans->box.height, ptrans->stride, 908 rsrc->image.layout.slices[level].row_stride, 909 rsrc->image.layout.format); 910 } 911} 912 913static void 914panfrost_store_tiled_images(struct panfrost_transfer *transfer, 915 struct panfrost_resource *rsrc) 916{ 917 struct panfrost_bo *bo = rsrc->image.data.bo; 918 struct pipe_transfer *ptrans = &transfer->base; 919 unsigned level = ptrans->level; 920 unsigned stride = panfrost_get_layer_stride(&rsrc->image.layout, level); 921 922 /* Otherwise, store each layer separately, required to store to 3D and 923 * array textures. 924 */ 925 for (unsigned z = 0; z < ptrans->box.depth; ++z) { 926 void *src = transfer->map + (ptrans->layer_stride * z); 927 uint8_t *map = bo->ptr.cpu + 928 rsrc->image.layout.slices[level].offset + 929 (z + ptrans->box.z) * stride; 930 931 panfrost_store_tiled_image(map, src, 932 ptrans->box.x, ptrans->box.y, 933 ptrans->box.width, ptrans->box.height, 934 rsrc->image.layout.slices[level].row_stride, 935 ptrans->stride, rsrc->image.layout.format); 936 } 937} 938 939static bool 940panfrost_box_covers_resource(const struct pipe_resource *resource, 941 const struct pipe_box *box) 942{ 943 return resource->last_level == 0 && 944 util_texrange_covers_whole_level(resource, 0, box->x, box->y, 945 box->z, box->width, box->height, 946 box->depth); 947} 948 949static void * 950panfrost_ptr_map(struct pipe_context *pctx, 951 struct pipe_resource *resource, 952 unsigned level, 953 unsigned usage, /* a combination of PIPE_MAP_x */ 954 const struct pipe_box *box, 955 struct pipe_transfer **out_transfer) 956{ 957 struct panfrost_context *ctx = pan_context(pctx); 958 struct panfrost_device *dev = pan_device(pctx->screen); 959 struct panfrost_resource *rsrc = pan_resource(resource); 960 enum pipe_format format = rsrc->image.layout.format; 961 int bytes_per_block = util_format_get_blocksize(format); 962 struct panfrost_bo *bo = rsrc->image.data.bo; 963 964 /* Can't map tiled/compressed directly */ 965 if ((usage & PIPE_MAP_DIRECTLY) && rsrc->image.layout.modifier != DRM_FORMAT_MOD_LINEAR) 966 return NULL; 967 968 struct panfrost_transfer *transfer = rzalloc(pctx, struct panfrost_transfer); 969 transfer->base.level = level; 970 transfer->base.usage = usage; 971 transfer->base.box = *box; 972 973 pipe_resource_reference(&transfer->base.resource, resource); 974 *out_transfer = &transfer->base; 975 976 if (usage & PIPE_MAP_WRITE) 977 rsrc->constant_stencil = false; 978 979 /* We don't have s/w routines for AFBC, so use a staging texture */ 980 if (drm_is_afbc(rsrc->image.layout.modifier)) { 981 struct panfrost_resource *staging = pan_alloc_staging(ctx, rsrc, level, box); 982 assert(staging); 983 984 /* Staging resources have one LOD: level 0. Query the strides 985 * on this LOD. 986 */ 987 transfer->base.stride = staging->image.layout.slices[0].row_stride; 988 transfer->base.layer_stride = 989 panfrost_get_layer_stride(&staging->image.layout, 0); 990 991 transfer->staging.rsrc = &staging->base; 992 993 transfer->staging.box = *box; 994 transfer->staging.box.x = 0; 995 transfer->staging.box.y = 0; 996 transfer->staging.box.z = 0; 997 998 assert(transfer->staging.rsrc != NULL); 999 1000 bool valid = BITSET_TEST(rsrc->valid.data, level); 1001 1002 if ((usage & PIPE_MAP_READ) && (valid || rsrc->track.nr_writers > 0)) { 1003 pan_blit_to_staging(pctx, transfer); 1004 panfrost_flush_writer(ctx, staging, "AFBC read staging blit"); 1005 panfrost_bo_wait(staging->image.data.bo, INT64_MAX, false); 1006 } 1007 1008 panfrost_bo_mmap(staging->image.data.bo); 1009 return staging->image.data.bo->ptr.cpu; 1010 } 1011 1012 /* If we haven't already mmaped, now's the time */ 1013 panfrost_bo_mmap(bo); 1014 1015 if (dev->debug & (PAN_DBG_TRACE | PAN_DBG_SYNC)) 1016 pandecode_inject_mmap(bo->ptr.gpu, bo->ptr.cpu, bo->size, NULL); 1017 1018 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is 1019 * being mapped. 1020 */ 1021 if ((usage & PIPE_MAP_DISCARD_RANGE) && 1022 !(usage & PIPE_MAP_UNSYNCHRONIZED) && 1023 !(resource->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) && 1024 panfrost_box_covers_resource(resource, box) && 1025 !(rsrc->image.data.bo->flags & PAN_BO_SHARED)) { 1026 1027 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE; 1028 } 1029 1030 bool create_new_bo = usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE; 1031 bool copy_resource = false; 1032 1033 if (!create_new_bo && 1034 !(usage & PIPE_MAP_UNSYNCHRONIZED) && 1035 (usage & PIPE_MAP_WRITE) && 1036 !(resource->target == PIPE_BUFFER 1037 && !util_ranges_intersect(&rsrc->valid_buffer_range, box->x, box->x + box->width)) && 1038 rsrc->track.nr_users > 0) { 1039 1040 /* When a resource to be modified is already being used by a 1041 * pending batch, it is often faster to copy the whole BO than 1042 * to flush and split the frame in two. 1043 */ 1044 1045 panfrost_flush_writer(ctx, rsrc, "Shadow resource creation"); 1046 panfrost_bo_wait(bo, INT64_MAX, false); 1047 1048 create_new_bo = true; 1049 copy_resource = !(usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE); 1050 } 1051 1052 /* Shadowing with separate stencil may require additional accounting. 1053 * Bail in these exotic cases. 1054 */ 1055 if (rsrc->separate_stencil) { 1056 create_new_bo = false; 1057 copy_resource = false; 1058 } 1059 1060 if (create_new_bo) { 1061 /* Make sure we re-emit any descriptors using this resource */ 1062 panfrost_dirty_state_all(ctx); 1063 1064 /* If the BO is used by one of the pending batches or if it's 1065 * not ready yet (still accessed by one of the already flushed 1066 * batches), we try to allocate a new one to avoid waiting. 1067 */ 1068 if (rsrc->track.nr_users > 0 || 1069 !panfrost_bo_wait(bo, 0, true)) { 1070 /* We want the BO to be MMAPed. */ 1071 uint32_t flags = bo->flags & ~PAN_BO_DELAY_MMAP; 1072 struct panfrost_bo *newbo = NULL; 1073 1074 /* When the BO has been imported/exported, we can't 1075 * replace it by another one, otherwise the 1076 * importer/exporter wouldn't see the change we're 1077 * doing to it. 1078 */ 1079 if (!(bo->flags & PAN_BO_SHARED)) 1080 newbo = panfrost_bo_create(dev, bo->size, 1081 flags, bo->label); 1082 1083 if (newbo) { 1084 if (copy_resource) 1085 memcpy(newbo->ptr.cpu, rsrc->image.data.bo->ptr.cpu, bo->size); 1086 1087 panfrost_resource_swap_bo(ctx, rsrc, newbo); 1088 1089 if (!copy_resource && 1090 drm_is_afbc(rsrc->image.layout.modifier)) 1091 panfrost_resource_init_afbc_headers(rsrc); 1092 1093 bo = newbo; 1094 } else { 1095 /* Allocation failed or was impossible, let's 1096 * fall back on a flush+wait. 1097 */ 1098 panfrost_flush_batches_accessing_rsrc(ctx, rsrc, 1099 "Resource access with high memory pressure"); 1100 panfrost_bo_wait(bo, INT64_MAX, true); 1101 } 1102 } 1103 } else if ((usage & PIPE_MAP_WRITE) 1104 && resource->target == PIPE_BUFFER 1105 && !util_ranges_intersect(&rsrc->valid_buffer_range, box->x, box->x + box->width)) { 1106 /* No flush for writes to uninitialized */ 1107 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) { 1108 if (usage & PIPE_MAP_WRITE) { 1109 panfrost_flush_batches_accessing_rsrc(ctx, rsrc, "Synchronized write"); 1110 panfrost_bo_wait(bo, INT64_MAX, true); 1111 } else if (usage & PIPE_MAP_READ) { 1112 panfrost_flush_writer(ctx, rsrc, "Synchronized read"); 1113 panfrost_bo_wait(bo, INT64_MAX, false); 1114 } 1115 } 1116 1117 /* For access to compressed textures, we want the (x, y, w, h) 1118 * region-of-interest in blocks, not pixels. Then we compute the stride 1119 * between rows of blocks as the width in blocks times the width per 1120 * block, etc. 1121 */ 1122 struct pipe_box box_blocks; 1123 u_box_pixels_to_blocks(&box_blocks, box, format); 1124 1125 if (rsrc->image.layout.modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { 1126 transfer->base.stride = box_blocks.width * bytes_per_block; 1127 transfer->base.layer_stride = transfer->base.stride * box_blocks.height; 1128 transfer->map = ralloc_size(transfer, transfer->base.layer_stride * box->depth); 1129 1130 if (usage & PIPE_MAP_READ) 1131 panfrost_load_tiled_images(transfer, rsrc); 1132 1133 return transfer->map; 1134 } else { 1135 assert (rsrc->image.layout.modifier == DRM_FORMAT_MOD_LINEAR); 1136 1137 /* Direct, persistent writes create holes in time for 1138 * caching... I don't know if this is actually possible but we 1139 * should still get it right */ 1140 1141 unsigned dpw = PIPE_MAP_DIRECTLY | PIPE_MAP_WRITE | PIPE_MAP_PERSISTENT; 1142 1143 if ((usage & dpw) == dpw && rsrc->index_cache) 1144 return NULL; 1145 1146 transfer->base.stride = rsrc->image.layout.slices[level].row_stride; 1147 transfer->base.layer_stride = 1148 panfrost_get_layer_stride(&rsrc->image.layout, level); 1149 1150 /* By mapping direct-write, we're implicitly already 1151 * initialized (maybe), so be conservative */ 1152 1153 if (usage & PIPE_MAP_WRITE) { 1154 BITSET_SET(rsrc->valid.data, level); 1155 panfrost_minmax_cache_invalidate(rsrc->index_cache, &transfer->base); 1156 } 1157 1158 return bo->ptr.cpu 1159 + rsrc->image.layout.slices[level].offset 1160 + box->z * transfer->base.layer_stride 1161 + box_blocks.y * rsrc->image.layout.slices[level].row_stride 1162 + box_blocks.x * bytes_per_block; 1163 } 1164} 1165 1166void 1167pan_resource_modifier_convert(struct panfrost_context *ctx, 1168 struct panfrost_resource *rsrc, 1169 uint64_t modifier, const char *reason) 1170{ 1171 assert(!rsrc->modifier_constant); 1172 1173 perf_debug_ctx(ctx, "Disabling AFBC with a blit. Reason: %s", reason); 1174 1175 struct pipe_resource *tmp_prsrc = 1176 panfrost_resource_create_with_modifier( 1177 ctx->base.screen, &rsrc->base, modifier); 1178 struct panfrost_resource *tmp_rsrc = pan_resource(tmp_prsrc); 1179 enum pipe_format blit_fmt = pan_blit_format(tmp_rsrc->base.format); 1180 1181 unsigned depth = rsrc->base.target == PIPE_TEXTURE_3D ? 1182 rsrc->base.depth0 : rsrc->base.array_size; 1183 1184 struct pipe_box box = 1185 { 0, 0, 0, rsrc->base.width0, rsrc->base.height0, depth }; 1186 1187 struct pipe_blit_info blit = { 1188 .dst.resource = &tmp_rsrc->base, 1189 .dst.format = blit_fmt, 1190 .dst.box = box, 1191 .src.resource = &rsrc->base, 1192 .src.format = pan_blit_format(rsrc->base.format), 1193 .src.box = box, 1194 .mask = util_format_get_mask(blit_fmt), 1195 .filter = PIPE_TEX_FILTER_NEAREST 1196 }; 1197 1198 for (int i = 0; i <= rsrc->base.last_level; i++) { 1199 if (BITSET_TEST(rsrc->valid.data, i)) { 1200 blit.dst.level = blit.src.level = i; 1201 panfrost_blit(&ctx->base, &blit); 1202 } 1203 } 1204 1205 panfrost_bo_unreference(rsrc->image.data.bo); 1206 if (rsrc->image.crc.bo) 1207 panfrost_bo_unreference(rsrc->image.crc.bo); 1208 1209 rsrc->image.data.bo = tmp_rsrc->image.data.bo; 1210 panfrost_bo_reference(rsrc->image.data.bo); 1211 1212 panfrost_resource_setup(pan_device(ctx->base.screen), rsrc, modifier, 1213 blit.dst.format); 1214 pipe_resource_reference(&tmp_prsrc, NULL); 1215} 1216 1217/* Validate that an AFBC resource may be used as a particular format. If it may 1218 * not, decompress it on the fly. Failure to do so can produce wrong results or 1219 * invalid data faults when sampling or rendering to AFBC */ 1220 1221void 1222pan_legalize_afbc_format(struct panfrost_context *ctx, 1223 struct panfrost_resource *rsrc, 1224 enum pipe_format format) 1225{ 1226 struct panfrost_device *dev = pan_device(ctx->base.screen); 1227 1228 if (!drm_is_afbc(rsrc->image.layout.modifier)) 1229 return; 1230 1231 if (panfrost_afbc_format(dev->arch, pan_blit_format(rsrc->base.format)) == 1232 panfrost_afbc_format(dev->arch, pan_blit_format(format))) 1233 return; 1234 1235 pan_resource_modifier_convert(ctx, rsrc, 1236 DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED, 1237 "Reinterpreting AFBC surface as incompatible format"); 1238} 1239 1240static bool 1241panfrost_should_linear_convert(struct panfrost_device *dev, 1242 struct panfrost_resource *prsrc, 1243 struct pipe_transfer *transfer) 1244{ 1245 if (prsrc->modifier_constant) 1246 return false; 1247 1248 /* Overwriting the entire resource indicates streaming, for which 1249 * linear layout is most efficient due to the lack of expensive 1250 * conversion. 1251 * 1252 * For now we just switch to linear after a number of complete 1253 * overwrites to keep things simple, but we could do better. 1254 * 1255 * This mechanism is only implemented for 2D resources. This suffices 1256 * for video players, its intended use case. 1257 */ 1258 1259 bool entire_overwrite = 1260 panfrost_is_2d(prsrc) && 1261 prsrc->base.last_level == 0 && 1262 transfer->box.width == prsrc->base.width0 && 1263 transfer->box.height == prsrc->base.height0 && 1264 transfer->box.x == 0 && 1265 transfer->box.y == 0; 1266 1267 if (entire_overwrite) 1268 ++prsrc->modifier_updates; 1269 1270 if (prsrc->modifier_updates >= LAYOUT_CONVERT_THRESHOLD) { 1271 perf_debug(dev, "Transitioning to linear due to streaming usage"); 1272 return true; 1273 } else { 1274 return false; 1275 } 1276} 1277 1278static void 1279panfrost_ptr_unmap(struct pipe_context *pctx, 1280 struct pipe_transfer *transfer) 1281{ 1282 /* Gallium expects writeback here, so we tile */ 1283 1284 struct panfrost_transfer *trans = pan_transfer(transfer); 1285 struct panfrost_resource *prsrc = (struct panfrost_resource *) transfer->resource; 1286 struct panfrost_device *dev = pan_device(pctx->screen); 1287 1288 if (transfer->usage & PIPE_MAP_WRITE) 1289 prsrc->valid.crc = false; 1290 1291 /* AFBC will use a staging resource. `initialized` will be set when the 1292 * fragment job is created; this is deferred to prevent useless surface 1293 * reloads that can cascade into DATA_INVALID_FAULTs due to reading 1294 * malformed AFBC data if uninitialized */ 1295 1296 if (trans->staging.rsrc) { 1297 if (transfer->usage & PIPE_MAP_WRITE) { 1298 if (panfrost_should_linear_convert(dev, prsrc, transfer)) { 1299 1300 panfrost_bo_unreference(prsrc->image.data.bo); 1301 if (prsrc->image.crc.bo) 1302 panfrost_bo_unreference(prsrc->image.crc.bo); 1303 1304 panfrost_resource_setup(dev, prsrc, DRM_FORMAT_MOD_LINEAR, 1305 prsrc->image.layout.format); 1306 1307 prsrc->image.data.bo = pan_resource(trans->staging.rsrc)->image.data.bo; 1308 panfrost_bo_reference(prsrc->image.data.bo); 1309 } else { 1310 pan_blit_from_staging(pctx, trans); 1311 panfrost_flush_batches_accessing_rsrc(pan_context(pctx), 1312 pan_resource(trans->staging.rsrc), 1313 "AFBC write staging blit"); 1314 } 1315 } 1316 1317 pipe_resource_reference(&trans->staging.rsrc, NULL); 1318 } 1319 1320 /* Tiling will occur in software from a staging cpu buffer */ 1321 if (trans->map) { 1322 struct panfrost_bo *bo = prsrc->image.data.bo; 1323 1324 if (transfer->usage & PIPE_MAP_WRITE) { 1325 BITSET_SET(prsrc->valid.data, transfer->level); 1326 1327 if (prsrc->image.layout.modifier == DRM_FORMAT_MOD_ARM_16X16_BLOCK_U_INTERLEAVED) { 1328 if (panfrost_should_linear_convert(dev, prsrc, transfer)) { 1329 panfrost_resource_setup(dev, prsrc, DRM_FORMAT_MOD_LINEAR, 1330 prsrc->image.layout.format); 1331 if (prsrc->image.layout.data_size > bo->size) { 1332 const char *label = bo->label; 1333 panfrost_bo_unreference(bo); 1334 bo = prsrc->image.data.bo = 1335 panfrost_bo_create(dev, prsrc->image.layout.data_size, 0, label); 1336 assert(bo); 1337 } 1338 1339 util_copy_rect( 1340 bo->ptr.cpu + prsrc->image.layout.slices[0].offset, 1341 prsrc->base.format, 1342 prsrc->image.layout.slices[0].row_stride, 1343 0, 0, 1344 transfer->box.width, 1345 transfer->box.height, 1346 trans->map, 1347 transfer->stride, 1348 0, 0); 1349 } else { 1350 panfrost_store_tiled_images(trans, prsrc); 1351 } 1352 } 1353 } 1354 } 1355 1356 1357 util_range_add(&prsrc->base, &prsrc->valid_buffer_range, 1358 transfer->box.x, 1359 transfer->box.x + transfer->box.width); 1360 1361 panfrost_minmax_cache_invalidate(prsrc->index_cache, transfer); 1362 1363 /* Derefence the resource */ 1364 pipe_resource_reference(&transfer->resource, NULL); 1365 1366 /* Transfer itself is RALLOCed at the moment */ 1367 ralloc_free(transfer); 1368} 1369 1370static void 1371panfrost_ptr_flush_region(struct pipe_context *pctx, 1372 struct pipe_transfer *transfer, 1373 const struct pipe_box *box) 1374{ 1375 struct panfrost_resource *rsc = pan_resource(transfer->resource); 1376 1377 if (transfer->resource->target == PIPE_BUFFER) { 1378 util_range_add(&rsc->base, &rsc->valid_buffer_range, 1379 transfer->box.x + box->x, 1380 transfer->box.x + box->x + box->width); 1381 } else { 1382 BITSET_SET(rsc->valid.data, transfer->level); 1383 } 1384} 1385 1386static void 1387panfrost_invalidate_resource(struct pipe_context *pctx, struct pipe_resource *prsrc) 1388{ 1389 struct panfrost_context *ctx = pan_context(pctx); 1390 struct panfrost_batch *batch = panfrost_get_batch_for_fbo(ctx); 1391 struct panfrost_resource *rsrc = pan_resource(prsrc); 1392 1393 rsrc->constant_stencil = true; 1394 1395 /* Handle the glInvalidateFramebuffer case */ 1396 if (batch->key.zsbuf && batch->key.zsbuf->texture == prsrc) 1397 batch->resolve &= ~PIPE_CLEAR_DEPTHSTENCIL; 1398 1399 for (unsigned i = 0; i < batch->key.nr_cbufs; ++i) { 1400 struct pipe_surface *surf = batch->key.cbufs[i]; 1401 1402 if (surf && surf->texture == prsrc) 1403 batch->resolve &= ~(PIPE_CLEAR_COLOR0 << i); 1404 } 1405} 1406 1407static enum pipe_format 1408panfrost_resource_get_internal_format(struct pipe_resource *rsrc) 1409{ 1410 struct panfrost_resource *prsrc = (struct panfrost_resource *) rsrc; 1411 return prsrc->image.layout.format; 1412} 1413 1414static bool 1415panfrost_generate_mipmap( 1416 struct pipe_context *pctx, 1417 struct pipe_resource *prsrc, 1418 enum pipe_format format, 1419 unsigned base_level, 1420 unsigned last_level, 1421 unsigned first_layer, 1422 unsigned last_layer) 1423{ 1424 struct panfrost_resource *rsrc = pan_resource(prsrc); 1425 1426 /* Generating a mipmap invalidates the written levels, so make that 1427 * explicit so we don't try to wallpaper them back and end up with 1428 * u_blitter recursion */ 1429 1430 assert(rsrc->image.data.bo); 1431 for (unsigned l = base_level + 1; l <= last_level; ++l) 1432 BITSET_CLEAR(rsrc->valid.data, l); 1433 1434 /* Beyond that, we just delegate the hard stuff. */ 1435 1436 bool blit_res = util_gen_mipmap( 1437 pctx, prsrc, format, 1438 base_level, last_level, 1439 first_layer, last_layer, 1440 PIPE_TEX_FILTER_LINEAR); 1441 1442 return blit_res; 1443} 1444 1445static void 1446panfrost_resource_set_stencil(struct pipe_resource *prsrc, 1447 struct pipe_resource *stencil) 1448{ 1449 pan_resource(prsrc)->separate_stencil = pan_resource(stencil); 1450} 1451 1452static struct pipe_resource * 1453panfrost_resource_get_stencil(struct pipe_resource *prsrc) 1454{ 1455 if (!pan_resource(prsrc)->separate_stencil) 1456 return NULL; 1457 1458 return &pan_resource(prsrc)->separate_stencil->base; 1459} 1460 1461static const struct u_transfer_vtbl transfer_vtbl = { 1462 .resource_create = panfrost_resource_create, 1463 .resource_destroy = panfrost_resource_destroy, 1464 .transfer_map = panfrost_ptr_map, 1465 .transfer_unmap = panfrost_ptr_unmap, 1466 .transfer_flush_region = panfrost_ptr_flush_region, 1467 .get_internal_format = panfrost_resource_get_internal_format, 1468 .set_stencil = panfrost_resource_set_stencil, 1469 .get_stencil = panfrost_resource_get_stencil, 1470}; 1471 1472void 1473panfrost_resource_screen_init(struct pipe_screen *pscreen) 1474{ 1475 struct panfrost_device *dev = pan_device(pscreen); 1476 1477 bool fake_rgtc = !panfrost_supports_compressed_format(dev, MALI_BC4_UNORM); 1478 1479 pscreen->resource_create_with_modifiers = 1480 panfrost_resource_create_with_modifiers; 1481 pscreen->resource_create = u_transfer_helper_resource_create; 1482 pscreen->resource_destroy = u_transfer_helper_resource_destroy; 1483 pscreen->resource_from_handle = panfrost_resource_from_handle; 1484 pscreen->resource_get_handle = panfrost_resource_get_handle; 1485 pscreen->resource_get_param = panfrost_resource_get_param; 1486 pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl, 1487 true, false, 1488 fake_rgtc, true, false); 1489} 1490void 1491panfrost_resource_screen_destroy(struct pipe_screen *pscreen) 1492{ 1493 u_transfer_helper_destroy(pscreen->transfer_helper); 1494} 1495 1496void 1497panfrost_resource_context_init(struct pipe_context *pctx) 1498{ 1499 pctx->buffer_map = u_transfer_helper_transfer_map; 1500 pctx->buffer_unmap = u_transfer_helper_transfer_unmap; 1501 pctx->texture_map = u_transfer_helper_transfer_map; 1502 pctx->texture_unmap = u_transfer_helper_transfer_unmap; 1503 pctx->create_surface = panfrost_create_surface; 1504 pctx->surface_destroy = panfrost_surface_destroy; 1505 pctx->resource_copy_region = util_resource_copy_region; 1506 pctx->blit = panfrost_blit; 1507 pctx->generate_mipmap = panfrost_generate_mipmap; 1508 pctx->flush_resource = panfrost_flush_resource; 1509 pctx->invalidate_resource = panfrost_invalidate_resource; 1510 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region; 1511 pctx->buffer_subdata = u_default_buffer_subdata; 1512 pctx->texture_subdata = u_default_texture_subdata; 1513 pctx->clear_buffer = u_default_clear_buffer; 1514} 1515