1/* 2 * Copyright (C) 2017 Rob Clark <robclark@freedesktop.org> 3 * Copyright © 2018 Google, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 * SOFTWARE. 23 * 24 * Authors: 25 * Rob Clark <robclark@freedesktop.org> 26 */ 27 28#include "util/format_srgb.h" 29#include "util/half_float.h" 30#include "util/u_dump.h" 31#include "util/u_log.h" 32 33#include "freedreno_blitter.h" 34#include "freedreno_fence.h" 35#include "freedreno_resource.h" 36#include "freedreno_tracepoints.h" 37 38#include "fd6_blitter.h" 39#include "fd6_emit.h" 40#include "fd6_resource.h" 41 42static inline enum a6xx_2d_ifmt 43fd6_ifmt(enum a6xx_format fmt) 44{ 45 switch (fmt) { 46 case FMT6_A8_UNORM: 47 case FMT6_8_UNORM: 48 case FMT6_8_SNORM: 49 case FMT6_8_8_UNORM: 50 case FMT6_8_8_SNORM: 51 case FMT6_8_8_8_8_UNORM: 52 case FMT6_8_8_8_X8_UNORM: 53 case FMT6_8_8_8_8_SNORM: 54 case FMT6_4_4_4_4_UNORM: 55 case FMT6_5_5_5_1_UNORM: 56 case FMT6_5_6_5_UNORM: 57 return R2D_UNORM8; 58 59 case FMT6_32_UINT: 60 case FMT6_32_SINT: 61 case FMT6_32_32_UINT: 62 case FMT6_32_32_SINT: 63 case FMT6_32_32_32_32_UINT: 64 case FMT6_32_32_32_32_SINT: 65 return R2D_INT32; 66 67 case FMT6_16_UINT: 68 case FMT6_16_SINT: 69 case FMT6_16_16_UINT: 70 case FMT6_16_16_SINT: 71 case FMT6_16_16_16_16_UINT: 72 case FMT6_16_16_16_16_SINT: 73 case FMT6_10_10_10_2_UINT: 74 return R2D_INT16; 75 76 case FMT6_8_UINT: 77 case FMT6_8_SINT: 78 case FMT6_8_8_UINT: 79 case FMT6_8_8_SINT: 80 case FMT6_8_8_8_8_UINT: 81 case FMT6_8_8_8_8_SINT: 82 case FMT6_Z24_UNORM_S8_UINT: 83 case FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8: 84 return R2D_INT8; 85 86 case FMT6_16_UNORM: 87 case FMT6_16_SNORM: 88 case FMT6_16_16_UNORM: 89 case FMT6_16_16_SNORM: 90 case FMT6_16_16_16_16_UNORM: 91 case FMT6_16_16_16_16_SNORM: 92 case FMT6_32_FLOAT: 93 case FMT6_32_32_FLOAT: 94 case FMT6_32_32_32_32_FLOAT: 95 return R2D_FLOAT32; 96 97 case FMT6_16_FLOAT: 98 case FMT6_16_16_FLOAT: 99 case FMT6_16_16_16_16_FLOAT: 100 case FMT6_11_11_10_FLOAT: 101 case FMT6_10_10_10_2_UNORM_DEST: 102 return R2D_FLOAT16; 103 104 default: 105 unreachable("bad format"); 106 return 0; 107 } 108} 109 110/* Make sure none of the requested dimensions extend beyond the size of the 111 * resource. Not entirely sure why this happens, but sometimes it does, and 112 * w/ 2d blt doesn't have wrap modes like a sampler, so force those cases 113 * back to u_blitter 114 */ 115static bool 116ok_dims(const struct pipe_resource *r, const struct pipe_box *b, int lvl) 117{ 118 int last_layer = 119 r->target == PIPE_TEXTURE_3D ? u_minify(r->depth0, lvl) : r->array_size; 120 121 return (b->x >= 0) && (b->x + b->width <= u_minify(r->width0, lvl)) && 122 (b->y >= 0) && (b->y + b->height <= u_minify(r->height0, lvl)) && 123 (b->z >= 0) && (b->z + b->depth <= last_layer); 124} 125 126static bool 127ok_format(enum pipe_format pfmt) 128{ 129 enum a6xx_format fmt = fd6_color_format(pfmt, TILE6_LINEAR); 130 131 if (util_format_is_compressed(pfmt)) 132 return true; 133 134 switch (pfmt) { 135 case PIPE_FORMAT_Z24_UNORM_S8_UINT: 136 case PIPE_FORMAT_Z24X8_UNORM: 137 case PIPE_FORMAT_Z16_UNORM: 138 case PIPE_FORMAT_Z32_UNORM: 139 case PIPE_FORMAT_Z32_FLOAT: 140 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: 141 case PIPE_FORMAT_S8_UINT: 142 return true; 143 default: 144 break; 145 } 146 147 if (fmt == FMT6_NONE) 148 return false; 149 150 return true; 151} 152 153#define DEBUG_BLIT 0 154#define DEBUG_BLIT_FALLBACK 0 155 156#define fail_if(cond) \ 157 do { \ 158 if (cond) { \ 159 if (DEBUG_BLIT_FALLBACK) { \ 160 fprintf(stderr, "falling back: %s for blit:\n", #cond); \ 161 dump_blit_info(info); \ 162 } \ 163 return false; \ 164 } \ 165 } while (0) 166 167static bool 168is_ubwc(struct pipe_resource *prsc, unsigned level) 169{ 170 return fd_resource_ubwc_enabled(fd_resource(prsc), level); 171} 172 173static void 174dump_blit_info(const struct pipe_blit_info *info) 175{ 176 util_dump_blit_info(stderr, info); 177 fprintf(stderr, "\ndst resource: "); 178 util_dump_resource(stderr, info->dst.resource); 179 if (is_ubwc(info->dst.resource, info->dst.level)) 180 fprintf(stderr, " (ubwc)"); 181 fprintf(stderr, "\nsrc resource: "); 182 util_dump_resource(stderr, info->src.resource); 183 if (is_ubwc(info->src.resource, info->src.level)) 184 fprintf(stderr, " (ubwc)"); 185 fprintf(stderr, "\n"); 186} 187 188static bool 189can_do_blit(const struct pipe_blit_info *info) 190{ 191 /* I think we can do scaling, but not in z dimension since that would 192 * require blending.. 193 */ 194 fail_if(info->dst.box.depth != info->src.box.depth); 195 196 /* Fail if unsupported format: */ 197 fail_if(!ok_format(info->src.format)); 198 fail_if(!ok_format(info->dst.format)); 199 200 assert(!util_format_is_compressed(info->src.format)); 201 assert(!util_format_is_compressed(info->dst.format)); 202 203 fail_if(!ok_dims(info->src.resource, &info->src.box, info->src.level)); 204 205 fail_if(!ok_dims(info->dst.resource, &info->dst.box, info->dst.level)); 206 207 assert(info->dst.box.width >= 0); 208 assert(info->dst.box.height >= 0); 209 assert(info->dst.box.depth >= 0); 210 211 fail_if(info->dst.resource->nr_samples > 1); 212 213 fail_if(info->window_rectangle_include); 214 215 const struct util_format_description *src_desc = 216 util_format_description(info->src.format); 217 const struct util_format_description *dst_desc = 218 util_format_description(info->dst.format); 219 const int common_channels = 220 MIN2(src_desc->nr_channels, dst_desc->nr_channels); 221 222 if (info->mask & PIPE_MASK_RGBA) { 223 for (int i = 0; i < common_channels; i++) { 224 fail_if(memcmp(&src_desc->channel[i], &dst_desc->channel[i], 225 sizeof(src_desc->channel[0]))); 226 } 227 } 228 229 fail_if(info->alpha_blend); 230 231 return true; 232} 233 234static void 235emit_setup(struct fd_batch *batch) 236{ 237 struct fd_ringbuffer *ring = batch->draw; 238 struct fd_screen *screen = batch->ctx->screen; 239 240 fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true); 241 fd6_event_write(batch, ring, PC_CCU_FLUSH_DEPTH_TS, true); 242 fd6_event_write(batch, ring, PC_CCU_INVALIDATE_COLOR, false); 243 fd6_event_write(batch, ring, PC_CCU_INVALIDATE_DEPTH, false); 244 245 /* normal BLIT_OP_SCALE operation needs bypass RB_CCU_CNTL */ 246 OUT_WFI5(ring); 247 OUT_PKT4(ring, REG_A6XX_RB_CCU_CNTL, 1); 248 OUT_RING(ring, A6XX_RB_CCU_CNTL_COLOR_OFFSET(screen->ccu_offset_bypass)); 249} 250 251static void 252emit_blit_setup(struct fd_ringbuffer *ring, enum pipe_format pfmt, 253 bool scissor_enable, union pipe_color_union *color, 254 uint32_t unknown_8c01) 255{ 256 enum a6xx_format fmt = fd6_color_format(pfmt, TILE6_LINEAR); 257 bool is_srgb = util_format_is_srgb(pfmt); 258 enum a6xx_2d_ifmt ifmt = fd6_ifmt(fmt); 259 260 if (is_srgb) { 261 assert(ifmt == R2D_UNORM8); 262 ifmt = R2D_UNORM8_SRGB; 263 } 264 265 uint32_t blit_cntl = A6XX_RB_2D_BLIT_CNTL_MASK(0xf) | 266 A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT(fmt) | 267 A6XX_RB_2D_BLIT_CNTL_IFMT(ifmt) | 268 COND(color, A6XX_RB_2D_BLIT_CNTL_SOLID_COLOR) | 269 COND(scissor_enable, A6XX_RB_2D_BLIT_CNTL_SCISSOR); 270 271 OUT_PKT4(ring, REG_A6XX_RB_2D_BLIT_CNTL, 1); 272 OUT_RING(ring, blit_cntl); 273 274 OUT_PKT4(ring, REG_A6XX_GRAS_2D_BLIT_CNTL, 1); 275 OUT_RING(ring, blit_cntl); 276 277 if (fmt == FMT6_10_10_10_2_UNORM_DEST) 278 fmt = FMT6_16_16_16_16_FLOAT; 279 280 /* This register is probably badly named... it seems that it's 281 * controlling the internal/accumulator format or something like 282 * that. It's certainly not tied to only the src format. 283 */ 284 OUT_PKT4(ring, REG_A6XX_SP_2D_DST_FORMAT, 1); 285 OUT_RING( 286 ring, 287 A6XX_SP_2D_DST_FORMAT_COLOR_FORMAT(fmt) | 288 COND(util_format_is_pure_sint(pfmt), A6XX_SP_2D_DST_FORMAT_SINT) | 289 COND(util_format_is_pure_uint(pfmt), A6XX_SP_2D_DST_FORMAT_UINT) | 290 COND(is_srgb, A6XX_SP_2D_DST_FORMAT_SRGB) | 291 A6XX_SP_2D_DST_FORMAT_MASK(0xf)); 292 293 OUT_PKT4(ring, REG_A6XX_RB_2D_UNKNOWN_8C01, 1); 294 OUT_RING(ring, unknown_8c01); 295} 296 297/* buffers need to be handled specially since x/width can exceed the bounds 298 * supported by hw.. if necessary decompose into (potentially) two 2D blits 299 */ 300static void 301emit_blit_buffer(struct fd_context *ctx, struct fd_ringbuffer *ring, 302 const struct pipe_blit_info *info) 303{ 304 const struct pipe_box *sbox = &info->src.box; 305 const struct pipe_box *dbox = &info->dst.box; 306 struct fd_resource *src, *dst; 307 unsigned sshift, dshift; 308 309 if (DEBUG_BLIT) { 310 fprintf(stderr, "buffer blit: "); 311 dump_blit_info(info); 312 } 313 314 src = fd_resource(info->src.resource); 315 dst = fd_resource(info->dst.resource); 316 317 assert(src->layout.cpp == 1); 318 assert(dst->layout.cpp == 1); 319 assert(info->src.resource->format == info->dst.resource->format); 320 assert((sbox->y == 0) && (sbox->height == 1)); 321 assert((dbox->y == 0) && (dbox->height == 1)); 322 assert((sbox->z == 0) && (sbox->depth == 1)); 323 assert((dbox->z == 0) && (dbox->depth == 1)); 324 assert(sbox->width == dbox->width); 325 assert(info->src.level == 0); 326 assert(info->dst.level == 0); 327 328 /* 329 * Buffers can have dimensions bigger than max width, remap into 330 * multiple 1d blits to fit within max dimension 331 * 332 * Note that blob uses .ARRAY_PITCH=128 for blitting buffers, which 333 * seems to prevent overfetch related faults. Not quite sure what 334 * the deal is there. 335 * 336 * Low 6 bits of SRC/DST addresses need to be zero (ie. address 337 * aligned to 64) so we need to shift src/dst x1/x2 to make up the 338 * difference. On top of already splitting up the blit so width 339 * isn't > 16k. 340 * 341 * We perhaps could do a bit better, if src and dst are aligned but 342 * in the worst case this means we have to split the copy up into 343 * 16k (0x4000) minus 64 (0x40). 344 */ 345 346 sshift = sbox->x & 0x3f; 347 dshift = dbox->x & 0x3f; 348 349 emit_blit_setup(ring, PIPE_FORMAT_R8_UNORM, false, NULL, 0); 350 351 for (unsigned off = 0; off < sbox->width; off += (0x4000 - 0x40)) { 352 unsigned soff, doff, w, p; 353 354 soff = (sbox->x + off) & ~0x3f; 355 doff = (dbox->x + off) & ~0x3f; 356 357 w = MIN2(sbox->width - off, (0x4000 - 0x40)); 358 p = align(w, 64); 359 360 assert((soff + w) <= fd_bo_size(src->bo)); 361 assert((doff + w) <= fd_bo_size(dst->bo)); 362 363 /* 364 * Emit source: 365 */ 366 OUT_PKT4(ring, REG_A6XX_SP_PS_2D_SRC_INFO, 10); 367 OUT_RING(ring, A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(FMT6_8_UNORM) | 368 A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(TILE6_LINEAR) | 369 A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP(WZYX) | 0x500000); 370 OUT_RING(ring, 371 A6XX_SP_PS_2D_SRC_SIZE_WIDTH(sshift + w) | 372 A6XX_SP_PS_2D_SRC_SIZE_HEIGHT(1)); /* SP_PS_2D_SRC_SIZE */ 373 OUT_RELOC(ring, src->bo, soff, 0, 0); /* SP_PS_2D_SRC_LO/HI */ 374 OUT_RING(ring, A6XX_SP_PS_2D_SRC_PITCH_PITCH(p)); 375 376 OUT_RING(ring, 0x00000000); 377 OUT_RING(ring, 0x00000000); 378 OUT_RING(ring, 0x00000000); 379 OUT_RING(ring, 0x00000000); 380 OUT_RING(ring, 0x00000000); 381 382 /* 383 * Emit destination: 384 */ 385 OUT_PKT4(ring, REG_A6XX_RB_2D_DST_INFO, 9); 386 OUT_RING(ring, A6XX_RB_2D_DST_INFO_COLOR_FORMAT(FMT6_8_UNORM) | 387 A6XX_RB_2D_DST_INFO_TILE_MODE(TILE6_LINEAR) | 388 A6XX_RB_2D_DST_INFO_COLOR_SWAP(WZYX)); 389 OUT_RELOC(ring, dst->bo, doff, 0, 0); /* RB_2D_DST_LO/HI */ 390 OUT_RING(ring, A6XX_RB_2D_DST_PITCH(p)); 391 OUT_RING(ring, 0x00000000); 392 OUT_RING(ring, 0x00000000); 393 OUT_RING(ring, 0x00000000); 394 OUT_RING(ring, 0x00000000); 395 OUT_RING(ring, 0x00000000); 396 397 /* 398 * Blit command: 399 */ 400 OUT_PKT4(ring, REG_A6XX_GRAS_2D_SRC_TL_X, 4); 401 OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_X(sshift)); 402 OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_X(sshift + w - 1)); 403 OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_Y(0)); 404 OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_Y(0)); 405 406 OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2); 407 OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(dshift) | A6XX_GRAS_2D_DST_TL_Y(0)); 408 OUT_RING(ring, A6XX_GRAS_2D_DST_BR_X(dshift + w - 1) | 409 A6XX_GRAS_2D_DST_BR_Y(0)); 410 411 OUT_PKT7(ring, CP_EVENT_WRITE, 1); 412 OUT_RING(ring, 0x3f); 413 OUT_WFI5(ring); 414 415 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); 416 OUT_RING(ring, ctx->screen->info->a6xx.magic.RB_UNKNOWN_8E04_blit); 417 418 OUT_PKT7(ring, CP_BLIT, 1); 419 OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE)); 420 421 OUT_WFI5(ring); 422 423 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); 424 OUT_RING(ring, 0); /* RB_UNKNOWN_8E04 */ 425 } 426} 427 428static void 429fd6_clear_ubwc(struct fd_batch *batch, struct fd_resource *rsc) assert_dt 430{ 431 struct fd_ringbuffer *ring = fd_batch_get_prologue(batch); 432 union pipe_color_union color = {}; 433 434 emit_blit_setup(ring, PIPE_FORMAT_R8_UNORM, false, &color, 0); 435 436 OUT_PKT4(ring, REG_A6XX_SP_PS_2D_SRC_INFO, 13); 437 OUT_RING(ring, 0x00000000); 438 OUT_RING(ring, 0x00000000); 439 OUT_RING(ring, 0x00000000); 440 OUT_RING(ring, 0x00000000); 441 OUT_RING(ring, 0x00000000); 442 OUT_RING(ring, 0x00000000); 443 OUT_RING(ring, 0x00000000); 444 OUT_RING(ring, 0x00000000); 445 OUT_RING(ring, 0x00000000); 446 OUT_RING(ring, 0x00000000); 447 OUT_RING(ring, 0x00000000); 448 OUT_RING(ring, 0x00000000); 449 OUT_RING(ring, 0x00000000); 450 451 OUT_PKT4(ring, REG_A6XX_RB_2D_SRC_SOLID_C0, 4); 452 OUT_RING(ring, 0x00000000); 453 OUT_RING(ring, 0x00000000); 454 OUT_RING(ring, 0x00000000); 455 OUT_RING(ring, 0x00000000); 456 457 OUT_PKT4(ring, REG_A6XX_GRAS_2D_SRC_TL_X, 4); 458 OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_X(0)); 459 OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_X(0)); 460 OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_Y(0)); 461 OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_Y(0)); 462 463 unsigned size = rsc->layout.slices[0].offset; 464 unsigned offset = 0; 465 466 /* We could be more clever here and realize that we could use a 467 * larger width if the size is aligned to something more than a 468 * single page.. or even use a format larger than r8 in those 469 * cases. But for normal sized textures and even up to 16k x 16k 470 * at <= 4byte/pixel, we'll only go thru the loop once 471 */ 472 const unsigned w = 0x1000; 473 474 /* ubwc size should always be page aligned: */ 475 assert((size % w) == 0); 476 477 while (size > 0) { 478 const unsigned h = MIN2(0x4000, size / w); 479 /* width is already aligned to a suitable pitch: */ 480 const unsigned p = w; 481 482 /* 483 * Emit destination: 484 */ 485 OUT_PKT4(ring, REG_A6XX_RB_2D_DST_INFO, 9); 486 OUT_RING(ring, A6XX_RB_2D_DST_INFO_COLOR_FORMAT(FMT6_8_UNORM) | 487 A6XX_RB_2D_DST_INFO_TILE_MODE(TILE6_LINEAR) | 488 A6XX_RB_2D_DST_INFO_COLOR_SWAP(WZYX)); 489 OUT_RELOC(ring, rsc->bo, offset, 0, 0); /* RB_2D_DST_LO/HI */ 490 OUT_RING(ring, A6XX_RB_2D_DST_PITCH(p)); 491 OUT_RING(ring, 0x00000000); 492 OUT_RING(ring, 0x00000000); 493 OUT_RING(ring, 0x00000000); 494 OUT_RING(ring, 0x00000000); 495 OUT_RING(ring, 0x00000000); 496 497 /* 498 * Blit command: 499 */ 500 501 OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2); 502 OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(0) | A6XX_GRAS_2D_DST_TL_Y(0)); 503 OUT_RING(ring, 504 A6XX_GRAS_2D_DST_BR_X(w - 1) | A6XX_GRAS_2D_DST_BR_Y(h - 1)); 505 506 OUT_PKT7(ring, CP_EVENT_WRITE, 1); 507 OUT_RING(ring, 0x3f); 508 OUT_WFI5(ring); 509 510 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); 511 OUT_RING(ring, batch->ctx->screen->info->a6xx.magic.RB_UNKNOWN_8E04_blit); 512 513 OUT_PKT7(ring, CP_BLIT, 1); 514 OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE)); 515 516 OUT_WFI5(ring); 517 518 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); 519 OUT_RING(ring, 0); /* RB_UNKNOWN_8E04 */ 520 521 offset += w * h; 522 size -= w * h; 523 } 524 525 fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true); 526 fd6_event_write(batch, ring, PC_CCU_FLUSH_DEPTH_TS, true); 527 fd6_event_write(batch, ring, CACHE_FLUSH_TS, true); 528 fd_wfi(batch, ring); 529 fd6_cache_inv(batch, ring); 530} 531 532static void 533emit_blit_dst(struct fd_ringbuffer *ring, struct pipe_resource *prsc, 534 enum pipe_format pfmt, unsigned level, unsigned layer) 535{ 536 struct fd_resource *dst = fd_resource(prsc); 537 enum a6xx_format fmt = fd6_color_format(pfmt, dst->layout.tile_mode); 538 enum a6xx_tile_mode tile = fd_resource_tile_mode(prsc, level); 539 enum a3xx_color_swap swap = fd6_color_swap(pfmt, dst->layout.tile_mode); 540 uint32_t pitch = fd_resource_pitch(dst, level); 541 bool ubwc_enabled = fd_resource_ubwc_enabled(dst, level); 542 unsigned off = fd_resource_offset(dst, level, layer); 543 544 if (fmt == FMT6_Z24_UNORM_S8_UINT) 545 fmt = FMT6_Z24_UNORM_S8_UINT_AS_R8G8B8A8; 546 547 OUT_PKT4(ring, REG_A6XX_RB_2D_DST_INFO, 9); 548 OUT_RING(ring, A6XX_RB_2D_DST_INFO_COLOR_FORMAT(fmt) | 549 A6XX_RB_2D_DST_INFO_TILE_MODE(tile) | 550 A6XX_RB_2D_DST_INFO_COLOR_SWAP(swap) | 551 COND(util_format_is_srgb(pfmt), A6XX_RB_2D_DST_INFO_SRGB) | 552 COND(ubwc_enabled, A6XX_RB_2D_DST_INFO_FLAGS)); 553 OUT_RELOC(ring, dst->bo, off, 0, 0); /* RB_2D_DST_LO/HI */ 554 OUT_RING(ring, A6XX_RB_2D_DST_PITCH(pitch)); 555 OUT_RING(ring, 0x00000000); 556 OUT_RING(ring, 0x00000000); 557 OUT_RING(ring, 0x00000000); 558 OUT_RING(ring, 0x00000000); 559 OUT_RING(ring, 0x00000000); 560 561 if (ubwc_enabled) { 562 OUT_PKT4(ring, REG_A6XX_RB_2D_DST_FLAGS, 6); 563 fd6_emit_flag_reference(ring, dst, level, layer); 564 OUT_RING(ring, 0x00000000); 565 OUT_RING(ring, 0x00000000); 566 OUT_RING(ring, 0x00000000); 567 } 568} 569 570static void 571emit_blit_src(struct fd_ringbuffer *ring, const struct pipe_blit_info *info, 572 unsigned layer, unsigned nr_samples, bool sample_0) 573{ 574 struct fd_resource *src = fd_resource(info->src.resource); 575 enum a6xx_format sfmt = fd6_texture_format(info->src.format, src->layout.tile_mode); 576 enum a6xx_tile_mode stile = 577 fd_resource_tile_mode(info->src.resource, info->src.level); 578 enum a3xx_color_swap sswap = fd6_texture_swap(info->src.format, src->layout.tile_mode); 579 uint32_t pitch = fd_resource_pitch(src, info->src.level); 580 bool subwc_enabled = fd_resource_ubwc_enabled(src, info->src.level); 581 unsigned soff = fd_resource_offset(src, info->src.level, layer); 582 uint32_t width = u_minify(src->b.b.width0, info->src.level) * nr_samples; 583 uint32_t height = u_minify(src->b.b.height0, info->src.level); 584 uint32_t filter = 0; 585 586 if (info->filter == PIPE_TEX_FILTER_LINEAR) 587 filter = A6XX_SP_PS_2D_SRC_INFO_FILTER; 588 589 enum a3xx_msaa_samples samples = fd_msaa_samples(src->b.b.nr_samples); 590 591 if (info->src.format == PIPE_FORMAT_A8_UNORM) 592 sfmt = FMT6_A8_UNORM; 593 594 OUT_PKT4(ring, REG_A6XX_SP_PS_2D_SRC_INFO, 10); 595 OUT_RING(ring, A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(sfmt) | 596 A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(stile) | 597 A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP(sswap) | 598 A6XX_SP_PS_2D_SRC_INFO_SAMPLES(samples) | 599 COND(samples > MSAA_ONE && !sample_0, 600 A6XX_SP_PS_2D_SRC_INFO_SAMPLES_AVERAGE) | 601 COND(subwc_enabled, A6XX_SP_PS_2D_SRC_INFO_FLAGS) | 602 COND(util_format_is_srgb(info->src.format), 603 A6XX_SP_PS_2D_SRC_INFO_SRGB) | 604 0x500000 | filter); 605 OUT_RING(ring, 606 A6XX_SP_PS_2D_SRC_SIZE_WIDTH(width) | 607 A6XX_SP_PS_2D_SRC_SIZE_HEIGHT(height)); /* SP_PS_2D_SRC_SIZE */ 608 OUT_RELOC(ring, src->bo, soff, 0, 0); /* SP_PS_2D_SRC_LO/HI */ 609 OUT_RING(ring, A6XX_SP_PS_2D_SRC_PITCH_PITCH(pitch)); 610 611 OUT_RING(ring, 0x00000000); 612 OUT_RING(ring, 0x00000000); 613 OUT_RING(ring, 0x00000000); 614 OUT_RING(ring, 0x00000000); 615 OUT_RING(ring, 0x00000000); 616 617 if (subwc_enabled) { 618 OUT_PKT4(ring, REG_A6XX_SP_PS_2D_SRC_FLAGS, 6); 619 fd6_emit_flag_reference(ring, src, info->src.level, layer); 620 OUT_RING(ring, 0x00000000); 621 OUT_RING(ring, 0x00000000); 622 OUT_RING(ring, 0x00000000); 623 } 624} 625 626static void 627emit_blit_texture(struct fd_context *ctx, struct fd_ringbuffer *ring, 628 const struct pipe_blit_info *info, bool sample_0) 629{ 630 const struct pipe_box *sbox = &info->src.box; 631 const struct pipe_box *dbox = &info->dst.box; 632 struct fd_resource *dst; 633 int sx1, sy1, sx2, sy2; 634 int dx1, dy1, dx2, dy2; 635 636 if (DEBUG_BLIT) { 637 fprintf(stderr, "texture blit: "); 638 dump_blit_info(info); 639 } 640 641 dst = fd_resource(info->dst.resource); 642 643 uint32_t nr_samples = fd_resource_nr_samples(&dst->b.b); 644 645 sx1 = sbox->x * nr_samples; 646 sy1 = sbox->y; 647 sx2 = (sbox->x + sbox->width) * nr_samples - 1; 648 sy2 = sbox->y + sbox->height - 1; 649 650 OUT_PKT4(ring, REG_A6XX_GRAS_2D_SRC_TL_X, 4); 651 OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_X(sx1)); 652 OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_X(sx2)); 653 OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_Y(sy1)); 654 OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_Y(sy2)); 655 656 dx1 = dbox->x * nr_samples; 657 dy1 = dbox->y; 658 dx2 = (dbox->x + dbox->width) * nr_samples - 1; 659 dy2 = dbox->y + dbox->height - 1; 660 661 OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2); 662 OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(dx1) | A6XX_GRAS_2D_DST_TL_Y(dy1)); 663 OUT_RING(ring, A6XX_GRAS_2D_DST_BR_X(dx2) | A6XX_GRAS_2D_DST_BR_Y(dy2)); 664 665 if (info->scissor_enable) { 666 OUT_PKT4(ring, REG_A6XX_GRAS_2D_RESOLVE_CNTL_1, 2); 667 OUT_RING(ring, A6XX_GRAS_2D_RESOLVE_CNTL_1_X(info->scissor.minx) | 668 A6XX_GRAS_2D_RESOLVE_CNTL_1_Y(info->scissor.miny)); 669 OUT_RING(ring, A6XX_GRAS_2D_RESOLVE_CNTL_1_X(info->scissor.maxx - 1) | 670 A6XX_GRAS_2D_RESOLVE_CNTL_1_Y(info->scissor.maxy - 1)); 671 } 672 673 emit_blit_setup(ring, info->dst.format, info->scissor_enable, NULL, 0); 674 675 for (unsigned i = 0; i < info->dst.box.depth; i++) { 676 677 emit_blit_src(ring, info, sbox->z + i, nr_samples, sample_0); 678 emit_blit_dst(ring, info->dst.resource, info->dst.format, info->dst.level, 679 dbox->z + i); 680 681 /* 682 * Blit command: 683 */ 684 OUT_PKT7(ring, CP_EVENT_WRITE, 1); 685 OUT_RING(ring, 0x3f); 686 OUT_WFI5(ring); 687 688 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); 689 OUT_RING(ring, ctx->screen->info->a6xx.magic.RB_UNKNOWN_8E04_blit); 690 691 OUT_PKT7(ring, CP_BLIT, 1); 692 OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE)); 693 694 OUT_WFI5(ring); 695 696 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); 697 OUT_RING(ring, 0); /* RB_UNKNOWN_8E04 */ 698 } 699} 700 701static void 702emit_clear_color(struct fd_ringbuffer *ring, enum pipe_format pfmt, 703 union pipe_color_union *color) 704{ 705 switch (pfmt) { 706 case PIPE_FORMAT_Z24X8_UNORM: 707 case PIPE_FORMAT_Z24_UNORM_S8_UINT: 708 case PIPE_FORMAT_X24S8_UINT: { 709 uint32_t depth_unorm24 = color->f[0] * ((1u << 24) - 1); 710 uint8_t stencil = color->ui[1]; 711 color->ui[0] = depth_unorm24 & 0xff; 712 color->ui[1] = (depth_unorm24 >> 8) & 0xff; 713 color->ui[2] = (depth_unorm24 >> 16) & 0xff; 714 color->ui[3] = stencil; 715 break; 716 } 717 default: 718 break; 719 } 720 721 OUT_PKT4(ring, REG_A6XX_RB_2D_SRC_SOLID_C0, 4); 722 switch (fd6_ifmt(fd6_color_format(pfmt, TILE6_LINEAR))) { 723 case R2D_UNORM8: 724 case R2D_UNORM8_SRGB: 725 /* The r2d ifmt is badly named, it also covers the signed case: */ 726 if (util_format_is_snorm(pfmt)) { 727 OUT_RING(ring, float_to_byte_tex(color->f[0])); 728 OUT_RING(ring, float_to_byte_tex(color->f[1])); 729 OUT_RING(ring, float_to_byte_tex(color->f[2])); 730 OUT_RING(ring, float_to_byte_tex(color->f[3])); 731 } else { 732 OUT_RING(ring, float_to_ubyte(color->f[0])); 733 OUT_RING(ring, float_to_ubyte(color->f[1])); 734 OUT_RING(ring, float_to_ubyte(color->f[2])); 735 OUT_RING(ring, float_to_ubyte(color->f[3])); 736 } 737 break; 738 case R2D_FLOAT16: 739 OUT_RING(ring, _mesa_float_to_half(color->f[0])); 740 OUT_RING(ring, _mesa_float_to_half(color->f[1])); 741 OUT_RING(ring, _mesa_float_to_half(color->f[2])); 742 OUT_RING(ring, _mesa_float_to_half(color->f[3])); 743 break; 744 case R2D_FLOAT32: 745 case R2D_INT32: 746 case R2D_INT16: 747 case R2D_INT8: 748 default: 749 OUT_RING(ring, color->ui[0]); 750 OUT_RING(ring, color->ui[1]); 751 OUT_RING(ring, color->ui[2]); 752 OUT_RING(ring, color->ui[3]); 753 break; 754 } 755} 756 757/** 758 * Handle conversion of clear color 759 */ 760static union pipe_color_union 761convert_color(enum pipe_format format, union pipe_color_union *pcolor) 762{ 763 union pipe_color_union color = *pcolor; 764 765 /* For solid-fill blits, the hw isn't going to convert from 766 * linear to srgb for us: 767 */ 768 if (util_format_is_srgb(format)) { 769 for (int i = 0; i < 3; i++) 770 color.f[i] = util_format_linear_to_srgb_float(color.f[i]); 771 } 772 773 if (util_format_is_snorm(format)) { 774 for (int i = 0; i < 3; i++) 775 color.f[i] = CLAMP(color.f[i], -1.0f, 1.0f); 776 } 777 778 /* Note that float_to_ubyte() already clamps, for the unorm case */ 779 780 return color; 781} 782 783void 784fd6_clear_surface(struct fd_context *ctx, struct fd_ringbuffer *ring, 785 struct pipe_surface *psurf, uint32_t width, uint32_t height, 786 union pipe_color_union *color, uint32_t unknown_8c01) 787{ 788 if (DEBUG_BLIT) { 789 fprintf(stderr, "surface clear:\ndst resource: "); 790 util_dump_resource(stderr, psurf->texture); 791 fprintf(stderr, "\n"); 792 } 793 794 uint32_t nr_samples = fd_resource_nr_samples(psurf->texture); 795 OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2); 796 OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(0) | A6XX_GRAS_2D_DST_TL_Y(0)); 797 OUT_RING(ring, A6XX_GRAS_2D_DST_BR_X(width * nr_samples - 1) | 798 A6XX_GRAS_2D_DST_BR_Y(height - 1)); 799 800 union pipe_color_union clear_color = convert_color(psurf->format, color); 801 802 emit_clear_color(ring, psurf->format, &clear_color); 803 emit_blit_setup(ring, psurf->format, false, &clear_color, unknown_8c01); 804 805 for (unsigned i = psurf->u.tex.first_layer; i <= psurf->u.tex.last_layer; 806 i++) { 807 emit_blit_dst(ring, psurf->texture, psurf->format, psurf->u.tex.level, i); 808 809 /* 810 * Blit command: 811 */ 812 OUT_PKT7(ring, CP_EVENT_WRITE, 1); 813 OUT_RING(ring, 0x3f); 814 OUT_WFI5(ring); 815 816 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); 817 OUT_RING(ring, ctx->screen->info->a6xx.magic.RB_UNKNOWN_8E04_blit); 818 819 OUT_PKT7(ring, CP_BLIT, 1); 820 OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE)); 821 822 OUT_WFI5(ring); 823 824 OUT_PKT4(ring, REG_A6XX_RB_UNKNOWN_8E04, 1); 825 OUT_RING(ring, 0); /* RB_UNKNOWN_8E04 */ 826 } 827} 828 829void 830fd6_resolve_tile(struct fd_batch *batch, struct fd_ringbuffer *ring, 831 uint32_t base, struct pipe_surface *psurf, uint32_t unknown_8c01) 832{ 833 const struct fd_gmem_stateobj *gmem = batch->gmem_state; 834 uint64_t gmem_base = batch->ctx->screen->gmem_base + base; 835 uint32_t gmem_pitch = gmem->bin_w * batch->framebuffer.samples * 836 util_format_get_blocksize(psurf->format); 837 838 OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2); 839 OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(0) | A6XX_GRAS_2D_DST_TL_Y(0)); 840 OUT_RING(ring, A6XX_GRAS_2D_DST_BR_X(psurf->width - 1) | 841 A6XX_GRAS_2D_DST_BR_Y(psurf->height - 1)); 842 843 OUT_PKT4(ring, REG_A6XX_GRAS_2D_SRC_TL_X, 4); 844 OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_X(0)); 845 OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_X(psurf->width - 1)); 846 OUT_RING(ring, A6XX_GRAS_2D_SRC_TL_Y(0)); 847 OUT_RING(ring, A6XX_GRAS_2D_SRC_BR_Y(psurf->height - 1)); 848 849 /* Enable scissor bit, which will take into account the window scissor 850 * which is set per-tile 851 */ 852 emit_blit_setup(ring, psurf->format, true, NULL, unknown_8c01); 853 854 /* We shouldn't be using GMEM in the layered rendering case: */ 855 assert(psurf->u.tex.first_layer == psurf->u.tex.last_layer); 856 857 emit_blit_dst(ring, psurf->texture, psurf->format, psurf->u.tex.level, 858 psurf->u.tex.first_layer); 859 860 enum a6xx_format sfmt = fd6_color_format(psurf->format, TILE6_LINEAR); 861 enum a3xx_msaa_samples samples = fd_msaa_samples(batch->framebuffer.samples); 862 863 OUT_PKT4(ring, REG_A6XX_SP_PS_2D_SRC_INFO, 10); 864 OUT_RING(ring, 865 A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(sfmt) | 866 A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(TILE6_2) | 867 A6XX_SP_PS_2D_SRC_INFO_SAMPLES(samples) | 868 COND(samples > MSAA_ONE, A6XX_SP_PS_2D_SRC_INFO_SAMPLES_AVERAGE) | 869 COND(util_format_is_srgb(psurf->format), A6XX_SP_PS_2D_SRC_INFO_SRGB) | 870 A6XX_SP_PS_2D_SRC_INFO_UNK20 | A6XX_SP_PS_2D_SRC_INFO_UNK22); 871 OUT_RING(ring, A6XX_SP_PS_2D_SRC_SIZE_WIDTH(psurf->width) | 872 A6XX_SP_PS_2D_SRC_SIZE_HEIGHT(psurf->height)); 873 OUT_RING(ring, gmem_base); /* SP_PS_2D_SRC_LO */ 874 OUT_RING(ring, gmem_base >> 32); /* SP_PS_2D_SRC_HI */ 875 OUT_RING(ring, A6XX_SP_PS_2D_SRC_PITCH_PITCH(gmem_pitch)); 876 OUT_RING(ring, 0x00000000); 877 OUT_RING(ring, 0x00000000); 878 OUT_RING(ring, 0x00000000); 879 OUT_RING(ring, 0x00000000); 880 OUT_RING(ring, 0x00000000); 881 882 /* sync GMEM writes with CACHE. */ 883 fd6_cache_inv(batch, ring); 884 885 /* Wait for CACHE_INVALIDATE to land */ 886 fd_wfi(batch, ring); 887 888 OUT_PKT7(ring, CP_BLIT, 1); 889 OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE)); 890 891 OUT_WFI5(ring); 892 893 /* CP_BLIT writes to the CCU, unlike CP_EVENT_WRITE::BLIT which writes to 894 * sysmem, and we generally assume that GMEM renderpasses leave their 895 * results in sysmem, so we need to flush manually here. 896 */ 897 fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true); 898 fd_wfi(batch, ring); 899} 900 901static bool 902handle_rgba_blit(struct fd_context *ctx, 903 const struct pipe_blit_info *info, bool sample_0) assert_dt 904{ 905 struct fd_batch *batch; 906 907 assert(!(info->mask & PIPE_MASK_ZS)); 908 909 if (!can_do_blit(info)) 910 return false; 911 912 struct fd_resource *src = fd_resource(info->src.resource); 913 struct fd_resource *dst = fd_resource(info->dst.resource); 914 915 fd6_validate_format(ctx, src, info->src.format); 916 fd6_validate_format(ctx, dst, info->dst.format); 917 918 batch = fd_bc_alloc_batch(ctx, true); 919 920 fd_screen_lock(ctx->screen); 921 922 fd_batch_resource_read(batch, src); 923 fd_batch_resource_write(batch, dst); 924 925 fd_screen_unlock(ctx->screen); 926 927 ASSERTED bool ret = fd_batch_lock_submit(batch); 928 assert(ret); 929 930 /* Marking the batch as needing flush must come after the batch 931 * dependency tracking (resource_read()/resource_write()), as that 932 * can trigger a flush 933 */ 934 fd_batch_needs_flush(batch); 935 936 fd_batch_update_queries(batch); 937 938 emit_setup(batch); 939 940 DBG_BLIT(info, batch); 941 942 trace_start_blit(&batch->trace, batch->draw, info->src.resource->target, 943 info->dst.resource->target); 944 945 if ((info->src.resource->target == PIPE_BUFFER) && 946 (info->dst.resource->target == PIPE_BUFFER)) { 947 assert(src->layout.tile_mode == TILE6_LINEAR); 948 assert(dst->layout.tile_mode == TILE6_LINEAR); 949 emit_blit_buffer(ctx, batch->draw, info); 950 } else { 951 /* I don't *think* we need to handle blits between buffer <-> !buffer */ 952 assert(info->src.resource->target != PIPE_BUFFER); 953 assert(info->dst.resource->target != PIPE_BUFFER); 954 emit_blit_texture(ctx, batch->draw, info, sample_0); 955 } 956 957 trace_end_blit(&batch->trace, batch->draw); 958 959 fd6_event_write(batch, batch->draw, PC_CCU_FLUSH_COLOR_TS, true); 960 fd6_event_write(batch, batch->draw, PC_CCU_FLUSH_DEPTH_TS, true); 961 fd6_event_write(batch, batch->draw, CACHE_FLUSH_TS, true); 962 fd_wfi(batch, batch->draw); 963 fd6_cache_inv(batch, batch->draw); 964 965 fd_batch_unlock_submit(batch); 966 967 fd_batch_flush(batch); 968 fd_batch_reference(&batch, NULL); 969 970 /* Acc query state will have been dirtied by our fd_batch_update_queries, so 971 * the ctx->batch may need to turn its queries back on. 972 */ 973 ctx->update_active_queries = true; 974 975 return true; 976} 977 978/** 979 * Re-written z/s blits can still fail for various reasons (for example MSAA). 980 * But we want to do the fallback blit with the re-written pipe_blit_info, 981 * in particular as u_blitter cannot blit stencil. So handle the fallback 982 * ourself and never "fail". 983 */ 984static bool 985do_rewritten_blit(struct fd_context *ctx, 986 const struct pipe_blit_info *info, bool sample_0) assert_dt 987{ 988 bool success = handle_rgba_blit(ctx, info, sample_0); 989 if (!success) { 990 if (sample_0 && !util_format_is_pure_integer(info->src.format)) 991 mesa_logw("sample averaging on fallback blit when we shouldn't."); 992 success = fd_blitter_blit(ctx, info); 993 } 994 assert(success); /* fallback should never fail! */ 995 return success; 996} 997 998/** 999 * Handle depth/stencil blits either via u_blitter and/or re-writing the 1000 * blit into an equivilant format that we can handle 1001 */ 1002static bool 1003handle_zs_blit(struct fd_context *ctx, 1004 const struct pipe_blit_info *info) assert_dt 1005{ 1006 struct pipe_blit_info blit = *info; 1007 1008 if (DEBUG_BLIT) { 1009 fprintf(stderr, "---- handle_zs_blit: "); 1010 dump_blit_info(info); 1011 } 1012 1013 if (info->src.format != info->dst.format) 1014 return false; 1015 1016 struct fd_resource *src = fd_resource(info->src.resource); 1017 struct fd_resource *dst = fd_resource(info->dst.resource); 1018 1019 switch (info->dst.format) { 1020 case PIPE_FORMAT_S8_UINT: 1021 assert(info->mask == PIPE_MASK_S); 1022 blit.mask = PIPE_MASK_R; 1023 blit.src.format = PIPE_FORMAT_R8_UINT; 1024 blit.dst.format = PIPE_FORMAT_R8_UINT; 1025 return do_rewritten_blit(ctx, &blit, true); 1026 1027 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT: 1028 if (info->mask & PIPE_MASK_Z) { 1029 blit.mask = PIPE_MASK_R; 1030 blit.src.format = PIPE_FORMAT_R32_FLOAT; 1031 blit.dst.format = PIPE_FORMAT_R32_FLOAT; 1032 do_rewritten_blit(ctx, &blit, true); 1033 } 1034 1035 if (info->mask & PIPE_MASK_S) { 1036 blit.mask = PIPE_MASK_R; 1037 blit.src.format = PIPE_FORMAT_R8_UINT; 1038 blit.dst.format = PIPE_FORMAT_R8_UINT; 1039 blit.src.resource = &src->stencil->b.b; 1040 blit.dst.resource = &dst->stencil->b.b; 1041 do_rewritten_blit(ctx, &blit, true); 1042 } 1043 1044 return true; 1045 1046 case PIPE_FORMAT_Z16_UNORM: 1047 blit.mask = PIPE_MASK_R; 1048 blit.src.format = PIPE_FORMAT_R16_UNORM; 1049 blit.dst.format = PIPE_FORMAT_R16_UNORM; 1050 return do_rewritten_blit(ctx, &blit, true); 1051 1052 case PIPE_FORMAT_Z32_UNORM: 1053 case PIPE_FORMAT_Z32_FLOAT: 1054 assert(info->mask == PIPE_MASK_Z); 1055 blit.mask = PIPE_MASK_R; 1056 blit.src.format = PIPE_FORMAT_R32_UINT; 1057 blit.dst.format = PIPE_FORMAT_R32_UINT; 1058 return do_rewritten_blit(ctx, &blit, true); 1059 1060 case PIPE_FORMAT_Z24X8_UNORM: 1061 case PIPE_FORMAT_Z24_UNORM_S8_UINT: 1062 blit.mask = 0; 1063 if (info->mask & PIPE_MASK_Z) 1064 blit.mask |= PIPE_MASK_R | PIPE_MASK_G | PIPE_MASK_B; 1065 if (info->mask & PIPE_MASK_S) 1066 blit.mask |= PIPE_MASK_A; 1067 blit.src.format = PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8; 1068 blit.dst.format = PIPE_FORMAT_Z24_UNORM_S8_UINT_AS_R8G8B8A8; 1069 /* non-UBWC Z24_UNORM_S8_UINT_AS_R8G8B8A8 is broken on a630, fall back to 1070 * 8888_unorm. 1071 */ 1072 if (!ctx->screen->info->a6xx.has_z24uint_s8uint) { 1073 if (!src->layout.ubwc && !dst->layout.ubwc) { 1074 blit.src.format = PIPE_FORMAT_RGBA8888_UINT; 1075 blit.dst.format = PIPE_FORMAT_RGBA8888_UINT; 1076 } else { 1077 if (!src->layout.ubwc) 1078 blit.src.format = PIPE_FORMAT_RGBA8888_UNORM; 1079 if (!dst->layout.ubwc) 1080 blit.dst.format = PIPE_FORMAT_RGBA8888_UNORM; 1081 } 1082 } 1083 if (info->src.resource->nr_samples > 1 && blit.src.format != PIPE_FORMAT_RGBA8888_UINT) 1084 mesa_logw("sample averaging on fallback z24s8 blit when we shouldn't."); 1085 return fd_blitter_blit(ctx, &blit); 1086 1087 default: 1088 return false; 1089 } 1090} 1091 1092static bool 1093handle_compressed_blit(struct fd_context *ctx, 1094 const struct pipe_blit_info *info) assert_dt 1095{ 1096 struct pipe_blit_info blit = *info; 1097 1098 if (DEBUG_BLIT) { 1099 fprintf(stderr, "---- handle_compressed_blit: "); 1100 dump_blit_info(info); 1101 } 1102 1103 if (info->src.format != info->dst.format) 1104 return fd_blitter_blit(ctx, info); 1105 1106 if (util_format_get_blocksize(info->src.format) == 8) { 1107 blit.src.format = blit.dst.format = PIPE_FORMAT_R16G16B16A16_UINT; 1108 } else { 1109 assert(util_format_get_blocksize(info->src.format) == 16); 1110 blit.src.format = blit.dst.format = PIPE_FORMAT_R32G32B32A32_UINT; 1111 } 1112 1113 int bw = util_format_get_blockwidth(info->src.format); 1114 int bh = util_format_get_blockheight(info->src.format); 1115 1116 /* NOTE: x/y *must* be aligned to block boundary (ie. in 1117 * glCompressedTexSubImage2D()) but width/height may not 1118 * be: 1119 */ 1120 1121 assert((blit.src.box.x % bw) == 0); 1122 assert((blit.src.box.y % bh) == 0); 1123 1124 blit.src.box.x /= bw; 1125 blit.src.box.y /= bh; 1126 blit.src.box.width = DIV_ROUND_UP(blit.src.box.width, bw); 1127 blit.src.box.height = DIV_ROUND_UP(blit.src.box.height, bh); 1128 1129 assert((blit.dst.box.x % bw) == 0); 1130 assert((blit.dst.box.y % bh) == 0); 1131 1132 blit.dst.box.x /= bw; 1133 blit.dst.box.y /= bh; 1134 blit.dst.box.width = DIV_ROUND_UP(blit.dst.box.width, bw); 1135 blit.dst.box.height = DIV_ROUND_UP(blit.dst.box.height, bh); 1136 1137 return do_rewritten_blit(ctx, &blit, false); 1138} 1139 1140/** 1141 * For SNORM formats, copy them as the equivalent UNORM format. If we treat 1142 * them as snorm then the 0x80 (-1.0 snorm8) value will get clamped to 0x81 1143 * (also -1.0), when we're supposed to be memcpying the bits. See 1144 * https://gitlab.khronos.org/Tracker/vk-gl-cts/-/issues/2917 for discussion. 1145 */ 1146static bool 1147handle_snorm_copy_blit(struct fd_context *ctx, 1148 const struct pipe_blit_info *info) 1149 assert_dt 1150{ 1151 /* If we're interpolating the pixels, we can't just treat the values as unorm. */ 1152 if (info->filter == PIPE_TEX_FILTER_LINEAR) 1153 return false; 1154 1155 struct pipe_blit_info blit = *info; 1156 1157 blit.src.format = blit.dst.format = util_format_snorm_to_unorm(info->src.format); 1158 1159 return do_rewritten_blit(ctx, &blit, false); 1160} 1161 1162static bool 1163fd6_blit(struct fd_context *ctx, const struct pipe_blit_info *info) assert_dt 1164{ 1165 if (info->mask & PIPE_MASK_ZS) 1166 return handle_zs_blit(ctx, info); 1167 1168 if (util_format_is_compressed(info->src.format) || 1169 util_format_is_compressed(info->dst.format)) 1170 return handle_compressed_blit(ctx, info); 1171 1172 if ((info->src.format == info->dst.format) && 1173 util_format_is_snorm(info->src.format)) 1174 return handle_snorm_copy_blit(ctx, info); 1175 1176 return handle_rgba_blit(ctx, info, false); 1177} 1178 1179void 1180fd6_blitter_init(struct pipe_context *pctx) disable_thread_safety_analysis 1181{ 1182 struct fd_context *ctx = fd_context(pctx); 1183 1184 ctx->clear_ubwc = fd6_clear_ubwc; 1185 ctx->validate_format = fd6_validate_format; 1186 1187 if (FD_DBG(NOBLIT)) 1188 return; 1189 1190 ctx->blit = fd6_blit; 1191} 1192 1193unsigned 1194fd6_tile_mode(const struct pipe_resource *tmpl) 1195{ 1196 /* if the mipmap level 0 is still too small to be tiled, then don't 1197 * bother pretending: 1198 */ 1199 if (fd_resource_level_linear(tmpl, 0)) 1200 return TILE6_LINEAR; 1201 1202 /* basically just has to be a format we can blit, so uploads/downloads 1203 * via linear staging buffer works: 1204 */ 1205 if (ok_format(tmpl->format)) 1206 return TILE6_3; 1207 1208 return TILE6_LINEAR; 1209} 1210