1/* 2 * Copyright © 2017 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included 12 * in all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 * DEALINGS IN THE SOFTWARE. 21 */ 22 23#include <stdio.h> 24#include "pipe/p_defines.h" 25#include "pipe/p_state.h" 26#include "pipe/p_context.h" 27#include "pipe/p_screen.h" 28#include "util/format/u_format.h" 29#include "util/u_inlines.h" 30#include "util/u_surface.h" 31#include "util/ralloc.h" 32#include "intel/blorp/blorp.h" 33#include "iris_context.h" 34#include "iris_resource.h" 35#include "iris_screen.h" 36 37/** 38 * Helper function for handling mirror image blits. 39 * 40 * If coord0 > coord1, swap them and return "true" (mirrored). 41 */ 42static bool 43apply_mirror(float *coord0, float *coord1) 44{ 45 if (*coord0 > *coord1) { 46 float tmp = *coord0; 47 *coord0 = *coord1; 48 *coord1 = tmp; 49 return true; 50 } 51 return false; 52} 53 54/** 55 * Compute the number of pixels to clip for each side of a rect 56 * 57 * \param x0 The rect's left coordinate 58 * \param y0 The rect's bottom coordinate 59 * \param x1 The rect's right coordinate 60 * \param y1 The rect's top coordinate 61 * \param min_x The clipping region's left coordinate 62 * \param min_y The clipping region's bottom coordinate 63 * \param max_x The clipping region's right coordinate 64 * \param max_y The clipping region's top coordinate 65 * \param clipped_x0 The number of pixels to clip from the left side 66 * \param clipped_y0 The number of pixels to clip from the bottom side 67 * \param clipped_x1 The number of pixels to clip from the right side 68 * \param clipped_y1 The number of pixels to clip from the top side 69 * 70 * \return false if we clip everything away, true otherwise 71 */ 72static inline bool 73compute_pixels_clipped(float x0, float y0, float x1, float y1, 74 float min_x, float min_y, float max_x, float max_y, 75 float *clipped_x0, float *clipped_y0, 76 float *clipped_x1, float *clipped_y1) 77{ 78 /* If we are going to clip everything away, stop. */ 79 if (!(min_x <= max_x && 80 min_y <= max_y && 81 x0 <= max_x && 82 y0 <= max_y && 83 min_x <= x1 && 84 min_y <= y1 && 85 x0 <= x1 && 86 y0 <= y1)) { 87 return false; 88 } 89 90 if (x0 < min_x) 91 *clipped_x0 = min_x - x0; 92 else 93 *clipped_x0 = 0; 94 if (max_x < x1) 95 *clipped_x1 = x1 - max_x; 96 else 97 *clipped_x1 = 0; 98 99 if (y0 < min_y) 100 *clipped_y0 = min_y - y0; 101 else 102 *clipped_y0 = 0; 103 if (max_y < y1) 104 *clipped_y1 = y1 - max_y; 105 else 106 *clipped_y1 = 0; 107 108 return true; 109} 110 111/** 112 * Clips a coordinate (left, right, top or bottom) for the src or dst rect 113 * (whichever requires the largest clip) and adjusts the coordinate 114 * for the other rect accordingly. 115 * 116 * \param mirror true if mirroring is required 117 * \param src the source rect coordinate (for example src_x0) 118 * \param dst0 the dst rect coordinate (for example dst_x0) 119 * \param dst1 the opposite dst rect coordinate (for example dst_x1) 120 * \param clipped_dst0 number of pixels to clip from the dst coordinate 121 * \param clipped_dst1 number of pixels to clip from the opposite dst coordinate 122 * \param scale the src vs dst scale involved for that coordinate 123 * \param is_left_or_bottom true if we are clipping the left or bottom sides 124 * of the rect. 125 */ 126static void 127clip_coordinates(bool mirror, 128 float *src, float *dst0, float *dst1, 129 float clipped_dst0, 130 float clipped_dst1, 131 float scale, 132 bool is_left_or_bottom) 133{ 134 /* When clipping we need to add or subtract pixels from the original 135 * coordinates depending on whether we are acting on the left/bottom 136 * or right/top sides of the rect respectively. We assume we have to 137 * add them in the code below, and multiply by -1 when we should 138 * subtract. 139 */ 140 int mult = is_left_or_bottom ? 1 : -1; 141 142 if (!mirror) { 143 *dst0 += clipped_dst0 * mult; 144 *src += clipped_dst0 * scale * mult; 145 } else { 146 *dst1 -= clipped_dst1 * mult; 147 *src += clipped_dst1 * scale * mult; 148 } 149} 150 151/** 152 * Apply a scissor rectangle to blit coordinates. 153 * 154 * Returns true if the blit was entirely scissored away. 155 */ 156static bool 157apply_blit_scissor(const struct pipe_scissor_state *scissor, 158 float *src_x0, float *src_y0, 159 float *src_x1, float *src_y1, 160 float *dst_x0, float *dst_y0, 161 float *dst_x1, float *dst_y1, 162 bool mirror_x, bool mirror_y) 163{ 164 float clip_dst_x0, clip_dst_x1, clip_dst_y0, clip_dst_y1; 165 166 /* Compute number of pixels to scissor away. */ 167 if (!compute_pixels_clipped(*dst_x0, *dst_y0, *dst_x1, *dst_y1, 168 scissor->minx, scissor->miny, 169 scissor->maxx, scissor->maxy, 170 &clip_dst_x0, &clip_dst_y0, 171 &clip_dst_x1, &clip_dst_y1)) 172 return true; 173 174 // XXX: comments assume source clipping, which we don't do 175 176 /* When clipping any of the two rects we need to adjust the coordinates 177 * in the other rect considering the scaling factor involved. To obtain 178 * the best precision we want to make sure that we only clip once per 179 * side to avoid accumulating errors due to the scaling adjustment. 180 * 181 * For example, if src_x0 and dst_x0 need both to be clipped we want to 182 * avoid the situation where we clip src_x0 first, then adjust dst_x0 183 * accordingly but then we realize that the resulting dst_x0 still needs 184 * to be clipped, so we clip dst_x0 and adjust src_x0 again. Because we are 185 * applying scaling factors to adjust the coordinates in each clipping 186 * pass we lose some precision and that can affect the results of the 187 * blorp blit operation slightly. What we want to do here is detect the 188 * rect that we should clip first for each side so that when we adjust 189 * the other rect we ensure the resulting coordinate does not need to be 190 * clipped again. 191 * 192 * The code below implements this by comparing the number of pixels that 193 * we need to clip for each side of both rects considering the scales 194 * involved. For example, clip_src_x0 represents the number of pixels 195 * to be clipped for the src rect's left side, so if clip_src_x0 = 5, 196 * clip_dst_x0 = 4 and scale_x = 2 it means that we are clipping more 197 * from the dst rect so we should clip dst_x0 only and adjust src_x0. 198 * This is because clipping 4 pixels in the dst is equivalent to 199 * clipping 4 * 2 = 8 > 5 in the src. 200 */ 201 202 if (*src_x0 == *src_x1 || *src_y0 == *src_y1 203 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1) 204 return true; 205 206 float scale_x = (float) (*src_x1 - *src_x0) / (*dst_x1 - *dst_x0); 207 float scale_y = (float) (*src_y1 - *src_y0) / (*dst_y1 - *dst_y0); 208 209 /* Clip left side */ 210 clip_coordinates(mirror_x, src_x0, dst_x0, dst_x1, 211 clip_dst_x0, clip_dst_x1, scale_x, true); 212 213 /* Clip right side */ 214 clip_coordinates(mirror_x, src_x1, dst_x1, dst_x0, 215 clip_dst_x1, clip_dst_x0, scale_x, false); 216 217 /* Clip bottom side */ 218 clip_coordinates(mirror_y, src_y0, dst_y0, dst_y1, 219 clip_dst_y0, clip_dst_y1, scale_y, true); 220 221 /* Clip top side */ 222 clip_coordinates(mirror_y, src_y1, dst_y1, dst_y0, 223 clip_dst_y1, clip_dst_y0, scale_y, false); 224 225 /* Check for invalid bounds 226 * Can't blit for 0-dimensions 227 */ 228 return *src_x0 == *src_x1 || *src_y0 == *src_y1 229 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1; 230} 231 232void 233iris_blorp_surf_for_resource(struct isl_device *isl_dev, 234 struct blorp_surf *surf, 235 struct pipe_resource *p_res, 236 enum isl_aux_usage aux_usage, 237 unsigned level, 238 bool is_dest) 239{ 240 struct iris_resource *res = (void *) p_res; 241 const struct intel_device_info *devinfo = isl_dev->info; 242 243 *surf = (struct blorp_surf) { 244 .surf = &res->surf, 245 .addr = (struct blorp_address) { 246 .buffer = res->bo, 247 .offset = res->offset, 248 .reloc_flags = is_dest ? EXEC_OBJECT_WRITE : 0, 249 .mocs = iris_mocs(res->bo, isl_dev, 250 is_dest ? ISL_SURF_USAGE_RENDER_TARGET_BIT 251 : ISL_SURF_USAGE_TEXTURE_BIT), 252 .local_hint = iris_bo_likely_local(res->bo), 253 }, 254 .aux_usage = aux_usage, 255 }; 256 257 if (aux_usage != ISL_AUX_USAGE_NONE) { 258 surf->aux_surf = &res->aux.surf; 259 surf->aux_addr = (struct blorp_address) { 260 .buffer = res->aux.bo, 261 .offset = res->aux.offset, 262 .reloc_flags = is_dest ? EXEC_OBJECT_WRITE : 0, 263 .mocs = iris_mocs(res->bo, isl_dev, 0), 264 .local_hint = devinfo->has_flat_ccs || 265 iris_bo_likely_local(res->aux.bo), 266 }; 267 surf->clear_color = res->aux.clear_color; 268 surf->clear_color_addr = (struct blorp_address) { 269 .buffer = res->aux.clear_color_bo, 270 .offset = res->aux.clear_color_offset, 271 .reloc_flags = 0, 272 .mocs = iris_mocs(res->aux.clear_color_bo, isl_dev, 0), 273 .local_hint = devinfo->has_flat_ccs || 274 iris_bo_likely_local(res->aux.clear_color_bo), 275 }; 276 } 277} 278 279static bool 280is_astc(enum isl_format format) 281{ 282 return format != ISL_FORMAT_UNSUPPORTED && 283 isl_format_get_layout(format)->txc == ISL_TXC_ASTC; 284} 285 286static void 287tex_cache_flush_hack(struct iris_batch *batch, 288 enum isl_format view_format, 289 enum isl_format surf_format) 290{ 291 const struct intel_device_info *devinfo = &batch->screen->devinfo; 292 293 /* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says: 294 * 295 * "Currently Sampler assumes that a surface would not have two 296 * different format associate with it. It will not properly cache 297 * the different views in the MT cache, causing a data corruption." 298 * 299 * We may need to handle this for texture views in general someday, but 300 * for now we handle it here, as it hurts copies and blits particularly 301 * badly because they ofter reinterpret formats. 302 * 303 * If the BO hasn't been referenced yet this batch, we assume that the 304 * texture cache doesn't contain any relevant data nor need flushing. 305 * 306 * Icelake (Gfx11+) claims to fix this issue, but seems to still have 307 * issues with ASTC formats. 308 */ 309 bool need_flush = devinfo->ver >= 11 ? 310 is_astc(surf_format) != is_astc(view_format) : 311 view_format != surf_format; 312 if (!need_flush) 313 return; 314 315 const char *reason = 316 "workaround: WaSamplerCacheFlushBetweenRedescribedSurfaceReads"; 317 318 iris_emit_pipe_control_flush(batch, reason, PIPE_CONTROL_CS_STALL); 319 iris_emit_pipe_control_flush(batch, reason, 320 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); 321} 322 323static struct iris_resource * 324iris_resource_for_aspect(struct pipe_resource *p_res, unsigned pipe_mask) 325{ 326 if (pipe_mask == PIPE_MASK_S) { 327 struct iris_resource *junk, *s_res; 328 iris_get_depth_stencil_resources(p_res, &junk, &s_res); 329 return s_res; 330 } else { 331 return (struct iris_resource *)p_res; 332 } 333} 334 335static enum pipe_format 336pipe_format_for_aspect(enum pipe_format format, unsigned pipe_mask) 337{ 338 if (pipe_mask == PIPE_MASK_S) { 339 return util_format_stencil_only(format); 340 } else if (pipe_mask == PIPE_MASK_Z) { 341 return util_format_get_depth_only(format); 342 } else { 343 return format; 344 } 345} 346 347static bool 348clear_color_is_fully_zero(const struct iris_resource *res) 349{ 350 return !res->aux.clear_color_unknown && 351 res->aux.clear_color.u32[0] == 0 && 352 res->aux.clear_color.u32[1] == 0 && 353 res->aux.clear_color.u32[2] == 0 && 354 res->aux.clear_color.u32[3] == 0; 355} 356 357/** 358 * The pipe->blit() driver hook. 359 * 360 * This performs a blit between two surfaces, which copies data but may 361 * also perform format conversion, scaling, flipping, and so on. 362 */ 363static void 364iris_blit(struct pipe_context *ctx, const struct pipe_blit_info *info) 365{ 366 struct iris_context *ice = (void *) ctx; 367 struct iris_screen *screen = (struct iris_screen *)ctx->screen; 368 const struct intel_device_info *devinfo = &screen->devinfo; 369 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; 370 enum blorp_batch_flags blorp_flags = iris_blorp_flags_for_batch(batch); 371 372 /* We don't support color masking. */ 373 assert((info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA || 374 (info->mask & PIPE_MASK_RGBA) == 0); 375 376 if (info->render_condition_enable) { 377 if (ice->state.predicate == IRIS_PREDICATE_STATE_DONT_RENDER) 378 return; 379 380 if (ice->state.predicate == IRIS_PREDICATE_STATE_USE_BIT) 381 blorp_flags |= BLORP_BATCH_PREDICATE_ENABLE; 382 } 383 384 float src_x0 = info->src.box.x; 385 float src_x1 = info->src.box.x + info->src.box.width; 386 float src_y0 = info->src.box.y; 387 float src_y1 = info->src.box.y + info->src.box.height; 388 float dst_x0 = info->dst.box.x; 389 float dst_x1 = info->dst.box.x + info->dst.box.width; 390 float dst_y0 = info->dst.box.y; 391 float dst_y1 = info->dst.box.y + info->dst.box.height; 392 bool mirror_x = apply_mirror(&src_x0, &src_x1); 393 bool mirror_y = apply_mirror(&src_y0, &src_y1); 394 enum blorp_filter filter; 395 396 if (info->scissor_enable) { 397 bool noop = apply_blit_scissor(&info->scissor, 398 &src_x0, &src_y0, &src_x1, &src_y1, 399 &dst_x0, &dst_y0, &dst_x1, &dst_y1, 400 mirror_x, mirror_y); 401 if (noop) 402 return; 403 } 404 405 /* Do DRI PRIME blits on the hardware blitter on Gfx12+ */ 406 if (devinfo->ver >= 12 && 407 (info->dst.resource->bind & PIPE_BIND_PRIME_BLIT_DST)) { 408 assert(!info->render_condition_enable); 409 assert(util_can_blit_via_copy_region(info, false, false)); 410 iris_copy_region(&ice->blorp, &ice->batches[IRIS_BATCH_BLITTER], 411 info->dst.resource, info->dst.level, 412 info->dst.box.x, info->dst.box.y, info->dst.box.z, 413 info->src.resource, info->src.level, 414 &info->src.box); 415 return; 416 } 417 418 if (abs(info->dst.box.width) == abs(info->src.box.width) && 419 abs(info->dst.box.height) == abs(info->src.box.height)) { 420 if (info->src.resource->nr_samples > 1 && 421 info->dst.resource->nr_samples <= 1) { 422 /* The OpenGL ES 3.2 specification, section 16.2.1, says: 423 * 424 * "If the read framebuffer is multisampled (its effective 425 * value of SAMPLE_BUFFERS is one) and the draw framebuffer 426 * is not (its value of SAMPLE_BUFFERS is zero), the samples 427 * corresponding to each pixel location in the source are 428 * converted to a single sample before being written to the 429 * destination. The filter parameter is ignored. If the 430 * source formats are integer types or stencil values, a 431 * single sample’s value is selected for each pixel. If the 432 * source formats are floating-point or normalized types, 433 * the sample values for each pixel are resolved in an 434 * implementation-dependent manner. If the source formats 435 * are depth values, sample values are resolved in an 436 * implementation-dependent manner where the result will be 437 * between the minimum and maximum depth values in the pixel." 438 * 439 * When selecting a single sample, we always choose sample 0. 440 */ 441 if (util_format_is_depth_or_stencil(info->src.format) || 442 util_format_is_pure_integer(info->src.format)) { 443 filter = BLORP_FILTER_SAMPLE_0; 444 } else { 445 filter = BLORP_FILTER_AVERAGE; 446 } 447 } else { 448 /* The OpenGL 4.6 specification, section 18.3.1, says: 449 * 450 * "If the source and destination dimensions are identical, 451 * no filtering is applied." 452 * 453 * Using BLORP_FILTER_NONE will also handle the upsample case by 454 * replicating the one value in the source to all values in the 455 * destination. 456 */ 457 filter = BLORP_FILTER_NONE; 458 } 459 } else if (info->filter == PIPE_TEX_FILTER_LINEAR) { 460 filter = BLORP_FILTER_BILINEAR; 461 } else { 462 filter = BLORP_FILTER_NEAREST; 463 } 464 465 struct blorp_batch blorp_batch; 466 blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags); 467 468 float src_z_step = (float)info->src.box.depth / (float)info->dst.box.depth; 469 470 /* There is no interpolation to the pixel center during rendering, so 471 * add the 0.5 offset ourselves here. 472 */ 473 float depth_center_offset = 0; 474 if (info->src.resource->target == PIPE_TEXTURE_3D) 475 depth_center_offset = 0.5 / info->dst.box.depth * info->src.box.depth; 476 477 /* Perform a blit for each aspect requested by the caller. PIPE_MASK_R is 478 * used to represent the color aspect. */ 479 unsigned aspect_mask = info->mask & (PIPE_MASK_R | PIPE_MASK_ZS); 480 while (aspect_mask) { 481 unsigned aspect = 1 << u_bit_scan(&aspect_mask); 482 483 struct iris_resource *src_res = 484 iris_resource_for_aspect(info->src.resource, aspect); 485 struct iris_resource *dst_res = 486 iris_resource_for_aspect(info->dst.resource, aspect); 487 488 enum pipe_format src_pfmt = 489 pipe_format_for_aspect(info->src.format, aspect); 490 enum pipe_format dst_pfmt = 491 pipe_format_for_aspect(info->dst.format, aspect); 492 493 struct iris_format_info src_fmt = 494 iris_format_for_usage(devinfo, src_pfmt, ISL_SURF_USAGE_TEXTURE_BIT); 495 enum isl_aux_usage src_aux_usage = 496 iris_resource_texture_aux_usage(ice, src_res, src_fmt.fmt); 497 498 iris_resource_prepare_texture(ice, src_res, src_fmt.fmt, 499 info->src.level, 1, info->src.box.z, 500 info->src.box.depth); 501 iris_emit_buffer_barrier_for(batch, src_res->bo, 502 IRIS_DOMAIN_SAMPLER_READ); 503 504 struct iris_format_info dst_fmt = 505 iris_format_for_usage(devinfo, dst_pfmt, 506 ISL_SURF_USAGE_RENDER_TARGET_BIT); 507 enum isl_aux_usage dst_aux_usage = 508 iris_resource_render_aux_usage(ice, dst_res, info->dst.level, 509 dst_fmt.fmt, false); 510 511 struct blorp_surf src_surf, dst_surf; 512 iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf, 513 &src_res->base.b, src_aux_usage, 514 info->src.level, false); 515 iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf, 516 &dst_res->base.b, dst_aux_usage, 517 info->dst.level, true); 518 519 iris_resource_prepare_render(ice, dst_res, info->dst.level, 520 info->dst.box.z, info->dst.box.depth, 521 dst_aux_usage); 522 iris_emit_buffer_barrier_for(batch, dst_res->bo, 523 IRIS_DOMAIN_RENDER_WRITE); 524 525 if (iris_batch_references(batch, src_res->bo)) 526 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format); 527 528 if (dst_res->base.b.target == PIPE_BUFFER) { 529 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, 530 dst_x0, dst_x1); 531 } 532 533 for (int slice = 0; slice < info->dst.box.depth; slice++) { 534 unsigned dst_z = info->dst.box.z + slice; 535 float src_z = info->src.box.z + slice * src_z_step + 536 depth_center_offset; 537 538 iris_batch_maybe_flush(batch, 1500); 539 iris_batch_sync_region_start(batch); 540 541 blorp_blit(&blorp_batch, 542 &src_surf, info->src.level, src_z, 543 src_fmt.fmt, src_fmt.swizzle, 544 &dst_surf, info->dst.level, dst_z, 545 dst_fmt.fmt, dst_fmt.swizzle, 546 src_x0, src_y0, src_x1, src_y1, 547 dst_x0, dst_y0, dst_x1, dst_y1, 548 filter, mirror_x, mirror_y); 549 550 iris_batch_sync_region_end(batch); 551 } 552 553 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format); 554 555 iris_resource_finish_render(ice, dst_res, info->dst.level, 556 info->dst.box.z, info->dst.box.depth, 557 dst_aux_usage); 558 } 559 560 blorp_batch_finish(&blorp_batch); 561 562 iris_dirty_for_history(ice, (struct iris_resource *)info->dst.resource); 563} 564 565static void 566get_copy_region_aux_settings(struct iris_context *ice, 567 const struct iris_batch *batch, 568 struct iris_resource *res, 569 unsigned level, 570 enum isl_aux_usage *out_aux_usage, 571 bool *out_clear_supported, 572 bool is_dest) 573{ 574 struct iris_screen *screen = (void *) ice->ctx.screen; 575 struct intel_device_info *devinfo = &screen->devinfo; 576 577 switch (res->aux.usage) { 578 case ISL_AUX_USAGE_HIZ: 579 case ISL_AUX_USAGE_HIZ_CCS: 580 case ISL_AUX_USAGE_HIZ_CCS_WT: 581 case ISL_AUX_USAGE_STC_CCS: 582 if (is_dest) { 583 *out_aux_usage = iris_resource_render_aux_usage(ice, res, level, 584 res->surf.format, 585 false); 586 } else { 587 *out_aux_usage = iris_resource_texture_aux_usage(ice, res, 588 res->surf.format); 589 } 590 *out_clear_supported = isl_aux_usage_has_fast_clears(*out_aux_usage); 591 break; 592 case ISL_AUX_USAGE_MCS: 593 case ISL_AUX_USAGE_MCS_CCS: 594 if (!is_dest && !iris_can_sample_mcs_with_clear(devinfo, res)) { 595 *out_aux_usage = res->aux.usage; 596 *out_clear_supported = false; 597 break; 598 } 599 FALLTHROUGH; 600 case ISL_AUX_USAGE_CCS_E: 601 case ISL_AUX_USAGE_GFX12_CCS_E: { 602 /* blorp_copy may reinterpret the surface format and has limited support 603 * for adjusting the clear color, so clear support may only be enabled 604 * in some cases: 605 * 606 * - On gfx11+, the clear color is indirect and comes in two forms: a 607 * 32bpc representation used for rendering and a pixel representation 608 * used for sampling. blorp_copy doesn't change indirect clear colors, 609 * so clears are only supported in the sampling case. 610 * 611 * - A clear color of zeroes holds the same meaning regardless of the 612 * format. Although it could avoid more resolves, we don't use 613 * isl_color_value_is_zero because the surface format used by 614 * blorp_copy isn't guaranteed to access the same components as the 615 * original format (e.g. A8_UNORM/R8_UINT). 616 */ 617 bool is_zero = clear_color_is_fully_zero(res); 618 619 if (batch->name == IRIS_BATCH_BLITTER) { 620 if (devinfo->verx10 >= 125) { 621 *out_aux_usage = res->aux.usage; 622 *out_clear_supported = is_zero; 623 } else { 624 *out_aux_usage = ISL_AUX_USAGE_NONE; 625 *out_clear_supported = false; 626 } 627 } else { 628 *out_aux_usage = res->aux.usage; 629 *out_clear_supported = is_zero || (devinfo->ver >= 11 && !is_dest); 630 } 631 break; 632 } 633 default: 634 *out_aux_usage = ISL_AUX_USAGE_NONE; 635 *out_clear_supported = false; 636 break; 637 } 638} 639 640/** 641 * Perform a GPU-based raw memory copy between compatible view classes. 642 * 643 * Does not perform any flushing - the new data may still be left in the 644 * render cache, and old data may remain in other caches. 645 * 646 * Wraps blorp_copy() and blorp_buffer_copy(). 647 */ 648void 649iris_copy_region(struct blorp_context *blorp, 650 struct iris_batch *batch, 651 struct pipe_resource *dst, 652 unsigned dst_level, 653 unsigned dstx, unsigned dsty, unsigned dstz, 654 struct pipe_resource *src, 655 unsigned src_level, 656 const struct pipe_box *src_box) 657{ 658 struct blorp_batch blorp_batch; 659 struct iris_context *ice = blorp->driver_ctx; 660 struct iris_screen *screen = (void *) ice->ctx.screen; 661 struct iris_resource *src_res = (void *) src; 662 struct iris_resource *dst_res = (void *) dst; 663 664 enum iris_domain write_domain = 665 batch->name == IRIS_BATCH_BLITTER ? IRIS_DOMAIN_OTHER_WRITE 666 : IRIS_DOMAIN_RENDER_WRITE; 667 668 enum isl_aux_usage src_aux_usage, dst_aux_usage; 669 bool src_clear_supported, dst_clear_supported; 670 get_copy_region_aux_settings(ice, batch, src_res, src_level, 671 &src_aux_usage, &src_clear_supported, false); 672 get_copy_region_aux_settings(ice, batch, dst_res, dst_level, 673 &dst_aux_usage, &dst_clear_supported, true); 674 675 if (iris_batch_references(batch, src_res->bo)) 676 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format); 677 678 if (dst->target == PIPE_BUFFER) 679 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, dstx, dstx + src_box->width); 680 681 enum blorp_batch_flags blorp_flags = iris_blorp_flags_for_batch(batch); 682 683 blorp_batch_init(blorp, &blorp_batch, batch, blorp_flags); 684 685 if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) { 686 struct blorp_address src_addr = { 687 .buffer = src_res->bo, .offset = src_res->offset + src_box->x, 688 .mocs = iris_mocs(src_res->bo, &screen->isl_dev, 689 ISL_SURF_USAGE_TEXTURE_BIT), 690 .local_hint = iris_bo_likely_local(src_res->bo), 691 }; 692 struct blorp_address dst_addr = { 693 .buffer = dst_res->bo, .offset = dst_res->offset + dstx, 694 .reloc_flags = EXEC_OBJECT_WRITE, 695 .mocs = iris_mocs(dst_res->bo, &screen->isl_dev, 696 ISL_SURF_USAGE_RENDER_TARGET_BIT), 697 .local_hint = iris_bo_likely_local(dst_res->bo), 698 }; 699 700 iris_emit_buffer_barrier_for(batch, src_res->bo, 701 IRIS_DOMAIN_SAMPLER_READ); 702 iris_emit_buffer_barrier_for(batch, dst_res->bo, write_domain); 703 704 iris_batch_maybe_flush(batch, 1500); 705 706 iris_batch_sync_region_start(batch); 707 blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width); 708 iris_batch_sync_region_end(batch); 709 } else { 710 // XXX: what about one surface being a buffer and not the other? 711 712 struct blorp_surf src_surf, dst_surf; 713 iris_blorp_surf_for_resource(&screen->isl_dev, &src_surf, 714 src, src_aux_usage, src_level, false); 715 iris_blorp_surf_for_resource(&screen->isl_dev, &dst_surf, 716 dst, dst_aux_usage, dst_level, true); 717 718 iris_resource_prepare_access(ice, src_res, src_level, 1, 719 src_box->z, src_box->depth, 720 src_aux_usage, src_clear_supported); 721 iris_resource_prepare_access(ice, dst_res, dst_level, 1, 722 dstz, src_box->depth, 723 dst_aux_usage, dst_clear_supported); 724 725 iris_emit_buffer_barrier_for(batch, src_res->bo, 726 IRIS_DOMAIN_SAMPLER_READ); 727 iris_emit_buffer_barrier_for(batch, dst_res->bo, write_domain); 728 729 for (int slice = 0; slice < src_box->depth; slice++) { 730 iris_batch_maybe_flush(batch, 1500); 731 732 iris_batch_sync_region_start(batch); 733 blorp_copy(&blorp_batch, &src_surf, src_level, src_box->z + slice, 734 &dst_surf, dst_level, dstz + slice, 735 src_box->x, src_box->y, dstx, dsty, 736 src_box->width, src_box->height); 737 iris_batch_sync_region_end(batch); 738 } 739 740 iris_resource_finish_write(ice, dst_res, dst_level, dstz, 741 src_box->depth, dst_aux_usage); 742 } 743 744 blorp_batch_finish(&blorp_batch); 745 746 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format); 747} 748 749/** 750 * The pipe->resource_copy_region() driver hook. 751 * 752 * This implements ARB_copy_image semantics - a raw memory copy between 753 * compatible view classes. 754 */ 755static void 756iris_resource_copy_region(struct pipe_context *ctx, 757 struct pipe_resource *p_dst, 758 unsigned dst_level, 759 unsigned dstx, unsigned dsty, unsigned dstz, 760 struct pipe_resource *p_src, 761 unsigned src_level, 762 const struct pipe_box *src_box) 763{ 764 struct iris_context *ice = (void *) ctx; 765 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; 766 767 iris_copy_region(&ice->blorp, batch, p_dst, dst_level, dstx, dsty, dstz, 768 p_src, src_level, src_box); 769 770 if (util_format_is_depth_and_stencil(p_dst->format) && 771 util_format_has_stencil(util_format_description(p_src->format))) { 772 struct iris_resource *junk, *s_src_res, *s_dst_res; 773 iris_get_depth_stencil_resources(p_src, &junk, &s_src_res); 774 iris_get_depth_stencil_resources(p_dst, &junk, &s_dst_res); 775 776 iris_copy_region(&ice->blorp, batch, &s_dst_res->base.b, dst_level, dstx, 777 dsty, dstz, &s_src_res->base.b, src_level, src_box); 778 } 779 780 iris_dirty_for_history(ice, (struct iris_resource *) p_dst); 781} 782 783void 784iris_init_blit_functions(struct pipe_context *ctx) 785{ 786 ctx->blit = iris_blit; 787 ctx->resource_copy_region = iris_resource_copy_region; 788} 789