1/* 2 * Copyright © 2017 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included 12 * in all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 * DEALINGS IN THE SOFTWARE. 21 */ 22 23#include <stdio.h> 24#include "pipe/p_defines.h" 25#include "pipe/p_state.h" 26#include "pipe/p_context.h" 27#include "pipe/p_screen.h" 28#include "util/format/u_format.h" 29#include "util/u_inlines.h" 30#include "util/u_surface.h" 31#include "util/ralloc.h" 32#include "intel/blorp/blorp.h" 33#include "crocus_context.h" 34#include "crocus_resource.h" 35#include "crocus_screen.h" 36 37void crocus_blitter_begin(struct crocus_context *ice, enum crocus_blitter_op op, bool render_cond) 38{ 39 util_blitter_save_vertex_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_VERTEX]); 40 util_blitter_save_tessctrl_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_TESS_CTRL]); 41 util_blitter_save_tesseval_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_TESS_EVAL]); 42 util_blitter_save_geometry_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_GEOMETRY]); 43 util_blitter_save_so_targets(ice->blitter, ice->state.so_targets, 44 (struct pipe_stream_output_target**)ice->state.so_target); 45 util_blitter_save_vertex_buffer_slot(ice->blitter, ice->state.vertex_buffers); 46 util_blitter_save_vertex_elements(ice->blitter, (void *)ice->state.cso_vertex_elements); 47 if (op & CROCUS_SAVE_FRAGMENT_STATE) { 48 util_blitter_save_blend(ice->blitter, ice->state.cso_blend); 49 util_blitter_save_depth_stencil_alpha(ice->blitter, ice->state.cso_zsa); 50 util_blitter_save_stencil_ref(ice->blitter, &ice->state.stencil_ref); 51 util_blitter_save_fragment_shader(ice->blitter, ice->shaders.uncompiled[MESA_SHADER_FRAGMENT]); 52 util_blitter_save_sample_mask(ice->blitter, ice->state.sample_mask, 0); 53 util_blitter_save_rasterizer(ice->blitter, ice->state.cso_rast); 54 util_blitter_save_scissor(ice->blitter, &ice->state.scissors[0]); 55 util_blitter_save_viewport(ice->blitter, &ice->state.viewports[0]); 56 util_blitter_save_fragment_constant_buffer_slot(ice->blitter, &ice->state.shaders[MESA_SHADER_FRAGMENT].constbufs[0]); 57 } 58 59 if (!render_cond) 60 util_blitter_save_render_condition(ice->blitter, 61 (struct pipe_query *)ice->condition.query, 62 ice->condition.condition, 63 ice->condition.mode); 64 65// util_blitter_save_scissor(ice->blitter, &ice->scissors[0]); 66 if (op & CROCUS_SAVE_FRAMEBUFFER) 67 util_blitter_save_framebuffer(ice->blitter, &ice->state.framebuffer); 68 69 if (op & CROCUS_SAVE_TEXTURES) { 70 util_blitter_save_fragment_sampler_states(ice->blitter, 1, (void **)ice->state.shaders[MESA_SHADER_FRAGMENT].samplers); 71 util_blitter_save_fragment_sampler_views(ice->blitter, 1, (struct pipe_sampler_view **)ice->state.shaders[MESA_SHADER_FRAGMENT].textures); 72 } 73} 74 75/** 76 * Helper function for handling mirror image blits. 77 * 78 * If coord0 > coord1, swap them and return "true" (mirrored). 79 */ 80static bool 81apply_mirror(float *coord0, float *coord1) 82{ 83 if (*coord0 > *coord1) { 84 float tmp = *coord0; 85 *coord0 = *coord1; 86 *coord1 = tmp; 87 return true; 88 } 89 return false; 90} 91 92/** 93 * Compute the number of pixels to clip for each side of a rect 94 * 95 * \param x0 The rect's left coordinate 96 * \param y0 The rect's bottom coordinate 97 * \param x1 The rect's right coordinate 98 * \param y1 The rect's top coordinate 99 * \param min_x The clipping region's left coordinate 100 * \param min_y The clipping region's bottom coordinate 101 * \param max_x The clipping region's right coordinate 102 * \param max_y The clipping region's top coordinate 103 * \param clipped_x0 The number of pixels to clip from the left side 104 * \param clipped_y0 The number of pixels to clip from the bottom side 105 * \param clipped_x1 The number of pixels to clip from the right side 106 * \param clipped_y1 The number of pixels to clip from the top side 107 * 108 * \return false if we clip everything away, true otherwise 109 */ 110static inline bool 111compute_pixels_clipped(float x0, float y0, float x1, float y1, 112 float min_x, float min_y, float max_x, float max_y, 113 float *clipped_x0, float *clipped_y0, 114 float *clipped_x1, float *clipped_y1) 115{ 116 /* If we are going to clip everything away, stop. */ 117 if (!(min_x <= max_x && 118 min_y <= max_y && 119 x0 <= max_x && 120 y0 <= max_y && 121 min_x <= x1 && 122 min_y <= y1 && 123 x0 <= x1 && 124 y0 <= y1)) { 125 return false; 126 } 127 128 if (x0 < min_x) 129 *clipped_x0 = min_x - x0; 130 else 131 *clipped_x0 = 0; 132 if (max_x < x1) 133 *clipped_x1 = x1 - max_x; 134 else 135 *clipped_x1 = 0; 136 137 if (y0 < min_y) 138 *clipped_y0 = min_y - y0; 139 else 140 *clipped_y0 = 0; 141 if (max_y < y1) 142 *clipped_y1 = y1 - max_y; 143 else 144 *clipped_y1 = 0; 145 146 return true; 147} 148 149/** 150 * Clips a coordinate (left, right, top or bottom) for the src or dst rect 151 * (whichever requires the largest clip) and adjusts the coordinate 152 * for the other rect accordingly. 153 * 154 * \param mirror true if mirroring is required 155 * \param src the source rect coordinate (for example src_x0) 156 * \param dst0 the dst rect coordinate (for example dst_x0) 157 * \param dst1 the opposite dst rect coordinate (for example dst_x1) 158 * \param clipped_dst0 number of pixels to clip from the dst coordinate 159 * \param clipped_dst1 number of pixels to clip from the opposite dst coordinate 160 * \param scale the src vs dst scale involved for that coordinate 161 * \param is_left_or_bottom true if we are clipping the left or bottom sides 162 * of the rect. 163 */ 164static void 165clip_coordinates(bool mirror, 166 float *src, float *dst0, float *dst1, 167 float clipped_dst0, 168 float clipped_dst1, 169 float scale, 170 bool is_left_or_bottom) 171{ 172 /* When clipping we need to add or subtract pixels from the original 173 * coordinates depending on whether we are acting on the left/bottom 174 * or right/top sides of the rect respectively. We assume we have to 175 * add them in the code below, and multiply by -1 when we should 176 * subtract. 177 */ 178 int mult = is_left_or_bottom ? 1 : -1; 179 180 if (!mirror) { 181 *dst0 += clipped_dst0 * mult; 182 *src += clipped_dst0 * scale * mult; 183 } else { 184 *dst1 -= clipped_dst1 * mult; 185 *src += clipped_dst1 * scale * mult; 186 } 187} 188 189/** 190 * Apply a scissor rectangle to blit coordinates. 191 * 192 * Returns true if the blit was entirely scissored away. 193 */ 194static bool 195apply_blit_scissor(const struct pipe_scissor_state *scissor, 196 float *src_x0, float *src_y0, 197 float *src_x1, float *src_y1, 198 float *dst_x0, float *dst_y0, 199 float *dst_x1, float *dst_y1, 200 bool mirror_x, bool mirror_y) 201{ 202 float clip_dst_x0, clip_dst_x1, clip_dst_y0, clip_dst_y1; 203 204 /* Compute number of pixels to scissor away. */ 205 if (!compute_pixels_clipped(*dst_x0, *dst_y0, *dst_x1, *dst_y1, 206 scissor->minx, scissor->miny, 207 scissor->maxx, scissor->maxy, 208 &clip_dst_x0, &clip_dst_y0, 209 &clip_dst_x1, &clip_dst_y1)) 210 return true; 211 212 // XXX: comments assume source clipping, which we don't do 213 214 /* When clipping any of the two rects we need to adjust the coordinates 215 * in the other rect considering the scaling factor involved. To obtain 216 * the best precision we want to make sure that we only clip once per 217 * side to avoid accumulating errors due to the scaling adjustment. 218 * 219 * For example, if src_x0 and dst_x0 need both to be clipped we want to 220 * avoid the situation where we clip src_x0 first, then adjust dst_x0 221 * accordingly but then we realize that the resulting dst_x0 still needs 222 * to be clipped, so we clip dst_x0 and adjust src_x0 again. Because we are 223 * applying scaling factors to adjust the coordinates in each clipping 224 * pass we lose some precision and that can affect the results of the 225 * blorp blit operation slightly. What we want to do here is detect the 226 * rect that we should clip first for each side so that when we adjust 227 * the other rect we ensure the resulting coordinate does not need to be 228 * clipped again. 229 * 230 * The code below implements this by comparing the number of pixels that 231 * we need to clip for each side of both rects considering the scales 232 * involved. For example, clip_src_x0 represents the number of pixels 233 * to be clipped for the src rect's left side, so if clip_src_x0 = 5, 234 * clip_dst_x0 = 4 and scale_x = 2 it means that we are clipping more 235 * from the dst rect so we should clip dst_x0 only and adjust src_x0. 236 * This is because clipping 4 pixels in the dst is equivalent to 237 * clipping 4 * 2 = 8 > 5 in the src. 238 */ 239 240 if (*src_x0 == *src_x1 || *src_y0 == *src_y1 241 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1) 242 return true; 243 244 float scale_x = (float) (*src_x1 - *src_x0) / (*dst_x1 - *dst_x0); 245 float scale_y = (float) (*src_y1 - *src_y0) / (*dst_y1 - *dst_y0); 246 247 /* Clip left side */ 248 clip_coordinates(mirror_x, src_x0, dst_x0, dst_x1, 249 clip_dst_x0, clip_dst_x1, scale_x, true); 250 251 /* Clip right side */ 252 clip_coordinates(mirror_x, src_x1, dst_x1, dst_x0, 253 clip_dst_x1, clip_dst_x0, scale_x, false); 254 255 /* Clip bottom side */ 256 clip_coordinates(mirror_y, src_y0, dst_y0, dst_y1, 257 clip_dst_y0, clip_dst_y1, scale_y, true); 258 259 /* Clip top side */ 260 clip_coordinates(mirror_y, src_y1, dst_y1, dst_y0, 261 clip_dst_y1, clip_dst_y0, scale_y, false); 262 263 /* Check for invalid bounds 264 * Can't blit for 0-dimensions 265 */ 266 return *src_x0 == *src_x1 || *src_y0 == *src_y1 267 || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1; 268} 269 270void 271crocus_blorp_surf_for_resource(struct crocus_vtable *vtbl, 272 struct isl_device *isl_dev, 273 struct blorp_surf *surf, 274 struct pipe_resource *p_res, 275 enum isl_aux_usage aux_usage, 276 unsigned level, 277 bool is_render_target) 278{ 279 struct crocus_resource *res = (void *) p_res; 280 281 assert(!crocus_resource_unfinished_aux_import(res)); 282 283 if (isl_aux_usage_has_hiz(aux_usage) && 284 !crocus_resource_level_has_hiz(res, level)) 285 aux_usage = ISL_AUX_USAGE_NONE; 286 287 *surf = (struct blorp_surf) { 288 .surf = &res->surf, 289 .addr = (struct blorp_address) { 290 .buffer = res->bo, 291 .offset = res->offset, 292 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0, 293 .mocs = crocus_mocs(res->bo, isl_dev), 294 }, 295 .aux_usage = aux_usage, 296 }; 297 298 if (aux_usage != ISL_AUX_USAGE_NONE) { 299 surf->aux_surf = &res->aux.surf; 300 surf->aux_addr = (struct blorp_address) { 301 .buffer = res->aux.bo, 302 .offset = res->aux.offset, 303 .reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0, 304 .mocs = crocus_mocs(res->bo, isl_dev), 305 }; 306 surf->clear_color = 307 crocus_resource_get_clear_color(res); 308 } 309} 310 311static void 312tex_cache_flush_hack(struct crocus_batch *batch, 313 enum isl_format view_format, 314 enum isl_format surf_format) 315{ 316 /* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says: 317 * 318 * "Currently Sampler assumes that a surface would not have two 319 * different format associate with it. It will not properly cache 320 * the different views in the MT cache, causing a data corruption." 321 * 322 * We may need to handle this for texture views in general someday, but 323 * for now we handle it here, as it hurts copies and blits particularly 324 * badly because they ofter reinterpret formats. 325 * 326 * If the BO hasn't been referenced yet this batch, we assume that the 327 * texture cache doesn't contain any relevant data nor need flushing. 328 * 329 * Icelake (Gen11+) claims to fix this issue, but seems to still have 330 * issues with ASTC formats. 331 */ 332 bool need_flush = view_format != surf_format; 333 if (!need_flush) 334 return; 335 336 const char *reason = 337 "workaround: WaSamplerCacheFlushBetweenRedescribedSurfaceReads"; 338 339 crocus_emit_pipe_control_flush(batch, reason, PIPE_CONTROL_CS_STALL); 340 crocus_emit_pipe_control_flush(batch, reason, 341 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE); 342} 343 344static struct crocus_resource * 345crocus_resource_for_aspect(const struct intel_device_info *devinfo, 346 struct pipe_resource *p_res, unsigned pipe_mask) 347{ 348 if (pipe_mask == PIPE_MASK_S) { 349 struct crocus_resource *junk, *s_res; 350 crocus_get_depth_stencil_resources(devinfo, p_res, &junk, &s_res); 351 return s_res; 352 } else { 353 return (struct crocus_resource *)p_res; 354 } 355} 356 357static enum pipe_format 358pipe_format_for_aspect(enum pipe_format format, unsigned pipe_mask) 359{ 360 if (pipe_mask == PIPE_MASK_S) { 361 return util_format_stencil_only(format); 362 } else if (pipe_mask == PIPE_MASK_Z) { 363 return util_format_get_depth_only(format); 364 } else { 365 return format; 366 } 367} 368 369static void 370crocus_u_blitter(struct crocus_context *ice, 371 const struct pipe_blit_info *info) 372{ 373 struct pipe_blit_info dinfo = *info; 374 if (!util_format_has_alpha(dinfo.dst.resource->format)) 375 dinfo.mask &= ~PIPE_MASK_A; 376 crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable); 377 util_blitter_blit(ice->blitter, &dinfo); 378} 379 380/** 381 * The pipe->blit() driver hook. 382 * 383 * This performs a blit between two surfaces, which copies data but may 384 * also perform format conversion, scaling, flipping, and so on. 385 */ 386static void 387crocus_blit(struct pipe_context *ctx, const struct pipe_blit_info *info) 388{ 389 struct crocus_context *ice = (void *) ctx; 390 struct crocus_screen *screen = (struct crocus_screen *)ctx->screen; 391 const struct intel_device_info *devinfo = &screen->devinfo; 392 struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER]; 393 enum blorp_batch_flags blorp_flags = 0; 394 395 /* We don't support color masking. */ 396 assert((info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA || 397 (info->mask & PIPE_MASK_RGBA) == 0); 398 399 if (info->render_condition_enable) 400 if (!crocus_check_conditional_render(ice)) 401 return; 402 403 if (devinfo->ver <= 5) { 404 if (!screen->vtbl.blit_blt(batch, info)) { 405 406 if (!util_format_is_depth_or_stencil(info->src.resource->format) && 407 info->dst.resource->target != PIPE_TEXTURE_3D) 408 goto use_blorp; 409 410 if (!util_blitter_is_blit_supported(ice->blitter, info)) { 411 if (util_format_is_depth_or_stencil(info->src.resource->format)) { 412 413 struct pipe_blit_info depth_blit = *info; 414 depth_blit.mask = PIPE_MASK_Z; 415 crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable); 416 util_blitter_blit(ice->blitter, &depth_blit); 417 418 struct pipe_surface *dst_view, dst_templ; 419 util_blitter_default_dst_texture(&dst_templ, info->dst.resource, info->dst.level, info->dst.box.z); 420 dst_view = ctx->create_surface(ctx, info->dst.resource, &dst_templ); 421 422 crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable); 423 424 util_blitter_clear_depth_stencil(ice->blitter, dst_view, PIPE_CLEAR_STENCIL, 425 0, 0, info->dst.box.x, info->dst.box.y, 426 info->dst.box.width, info->dst.box.height); 427 crocus_blitter_begin(ice, CROCUS_SAVE_FRAMEBUFFER | CROCUS_SAVE_TEXTURES | CROCUS_SAVE_FRAGMENT_STATE, info->render_condition_enable); 428 util_blitter_stencil_fallback(ice->blitter, 429 info->dst.resource, 430 info->dst.level, 431 &info->dst.box, 432 info->src.resource, 433 info->src.level, 434 &info->src.box, NULL); 435 436 pipe_surface_release(ctx, &dst_view); 437 } 438 return; 439 } 440 441 crocus_u_blitter(ice, info); 442 } 443 return; 444 } 445 446 if (devinfo->ver == 6) { 447 if (info->src.resource->target == PIPE_TEXTURE_3D && 448 info->dst.resource->target == PIPE_TEXTURE_3D) { 449 crocus_u_blitter(ice, info); 450 return; 451 } 452 } 453 454use_blorp: 455 if (info->render_condition_enable) { 456 if (ice->state.predicate == CROCUS_PREDICATE_STATE_USE_BIT) 457 blorp_flags |= BLORP_BATCH_PREDICATE_ENABLE; 458 } 459 460 float src_x0 = info->src.box.x; 461 float src_x1 = info->src.box.x + info->src.box.width; 462 float src_y0 = info->src.box.y; 463 float src_y1 = info->src.box.y + info->src.box.height; 464 float dst_x0 = info->dst.box.x; 465 float dst_x1 = info->dst.box.x + info->dst.box.width; 466 float dst_y0 = info->dst.box.y; 467 float dst_y1 = info->dst.box.y + info->dst.box.height; 468 bool mirror_x = apply_mirror(&src_x0, &src_x1); 469 bool mirror_y = apply_mirror(&src_y0, &src_y1); 470 enum blorp_filter filter; 471 472 if (info->scissor_enable) { 473 bool noop = apply_blit_scissor(&info->scissor, 474 &src_x0, &src_y0, &src_x1, &src_y1, 475 &dst_x0, &dst_y0, &dst_x1, &dst_y1, 476 mirror_x, mirror_y); 477 if (noop) 478 return; 479 } 480 481 if (abs(info->dst.box.width) == abs(info->src.box.width) && 482 abs(info->dst.box.height) == abs(info->src.box.height)) { 483 if (info->src.resource->nr_samples > 1 && 484 info->dst.resource->nr_samples <= 1) { 485 /* The OpenGL ES 3.2 specification, section 16.2.1, says: 486 * 487 * "If the read framebuffer is multisampled (its effective 488 * value of SAMPLE_BUFFERS is one) and the draw framebuffer 489 * is not (its value of SAMPLE_BUFFERS is zero), the samples 490 * corresponding to each pixel location in the source are 491 * converted to a single sample before being written to the 492 * destination. The filter parameter is ignored. If the 493 * source formats are integer types or stencil values, a 494 * single sample’s value is selected for each pixel. If the 495 * source formats are floating-point or normalized types, 496 * the sample values for each pixel are resolved in an 497 * implementation-dependent manner. If the source formats 498 * are depth values, sample values are resolved in an 499 * implementation-dependent manner where the result will be 500 * between the minimum and maximum depth values in the pixel." 501 * 502 * When selecting a single sample, we always choose sample 0. 503 */ 504 if (util_format_is_depth_or_stencil(info->src.format) || 505 util_format_is_pure_integer(info->src.format)) { 506 filter = BLORP_FILTER_SAMPLE_0; 507 } else { 508 filter = BLORP_FILTER_AVERAGE; 509 } 510 } else { 511 /* The OpenGL 4.6 specification, section 18.3.1, says: 512 * 513 * "If the source and destination dimensions are identical, 514 * no filtering is applied." 515 * 516 * Using BLORP_FILTER_NONE will also handle the upsample case by 517 * replicating the one value in the source to all values in the 518 * destination. 519 */ 520 filter = BLORP_FILTER_NONE; 521 } 522 } else if (info->filter == PIPE_TEX_FILTER_LINEAR) { 523 filter = BLORP_FILTER_BILINEAR; 524 } else { 525 filter = BLORP_FILTER_NEAREST; 526 } 527 528 struct blorp_batch blorp_batch; 529 blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags); 530 531 float src_z_step = (float)info->src.box.depth / (float)info->dst.box.depth; 532 533 /* There is no interpolation to the pixel center during rendering, so 534 * add the 0.5 offset ourselves here. 535 */ 536 float depth_center_offset = 0; 537 if (info->src.resource->target == PIPE_TEXTURE_3D) 538 depth_center_offset = 0.5 / info->dst.box.depth * info->src.box.depth; 539 540 /* Perform a blit for each aspect requested by the caller. PIPE_MASK_R is 541 * used to represent the color aspect. */ 542 unsigned aspect_mask = info->mask & (PIPE_MASK_R | PIPE_MASK_ZS); 543 while (aspect_mask) { 544 unsigned aspect = 1 << u_bit_scan(&aspect_mask); 545 546 struct crocus_resource *src_res = 547 crocus_resource_for_aspect(devinfo, info->src.resource, aspect); 548 struct crocus_resource *dst_res = 549 crocus_resource_for_aspect(devinfo, info->dst.resource, aspect); 550 551 enum pipe_format src_pfmt = 552 pipe_format_for_aspect(info->src.format, aspect); 553 enum pipe_format dst_pfmt = 554 pipe_format_for_aspect(info->dst.format, aspect); 555 556 if (crocus_resource_unfinished_aux_import(src_res)) 557 crocus_resource_finish_aux_import(ctx->screen, src_res); 558 if (crocus_resource_unfinished_aux_import(dst_res)) 559 crocus_resource_finish_aux_import(ctx->screen, dst_res); 560 561 struct crocus_format_info src_fmt = 562 crocus_format_for_usage(devinfo, src_pfmt, ISL_SURF_USAGE_TEXTURE_BIT); 563 enum isl_aux_usage src_aux_usage = 564 crocus_resource_texture_aux_usage(src_res); 565 566 crocus_resource_prepare_texture(ice, src_res, src_fmt.fmt, 567 info->src.level, 1, info->src.box.z, 568 info->src.box.depth); 569 // crocus_emit_buffer_barrier_for(batch, src_res->bo, 570 // CROCUS_DOMAIN_OTHER_READ); 571 572 bool dst_aux_disable = false; 573 /* on SNB blorp will use render target instead of depth 574 * so disable HiZ. 575 */ 576 if (devinfo->ver <= 6 && util_format_is_depth_or_stencil(dst_pfmt)) 577 dst_aux_disable = true; 578 struct crocus_format_info dst_fmt = 579 crocus_format_for_usage(devinfo, dst_pfmt, 580 ISL_SURF_USAGE_RENDER_TARGET_BIT); 581 enum isl_aux_usage dst_aux_usage = 582 crocus_resource_render_aux_usage(ice, dst_res, info->dst.level, 583 dst_fmt.fmt, dst_aux_disable); 584 585 struct blorp_surf src_surf, dst_surf; 586 crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &src_surf, 587 &src_res->base.b, src_aux_usage, 588 info->src.level, false); 589 crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &dst_surf, 590 &dst_res->base.b, dst_aux_usage, 591 info->dst.level, true); 592 593 crocus_resource_prepare_render(ice, dst_res, info->dst.level, 594 info->dst.box.z, info->dst.box.depth, 595 dst_aux_usage); 596 // crocus_emit_buffer_barrier_for(batch, dst_res->bo, 597 // CROCUS_DOMAIN_RENDER_WRITE); 598 599 if (crocus_batch_references(batch, src_res->bo)) 600 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format); 601 602 if (dst_res->base.b.target == PIPE_BUFFER) { 603 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, 604 dst_x0, dst_x1); 605 } 606 607 struct isl_swizzle src_swiz = pipe_to_isl_swizzles(src_fmt.swizzles); 608 struct isl_swizzle dst_swiz = pipe_to_isl_swizzles(dst_fmt.swizzles); 609 610 for (int slice = 0; slice < info->dst.box.depth; slice++) { 611 unsigned dst_z = info->dst.box.z + slice; 612 float src_z = info->src.box.z + slice * src_z_step + 613 depth_center_offset; 614 615 crocus_batch_maybe_flush(batch, 1500); 616 617 blorp_blit(&blorp_batch, 618 &src_surf, info->src.level, src_z, 619 src_fmt.fmt, src_swiz, 620 &dst_surf, info->dst.level, dst_z, 621 dst_fmt.fmt, dst_swiz, 622 src_x0, src_y0, src_x1, src_y1, 623 dst_x0, dst_y0, dst_x1, dst_y1, 624 filter, mirror_x, mirror_y); 625 626 } 627 628 tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format); 629 630 crocus_resource_finish_render(ice, dst_res, info->dst.level, 631 info->dst.box.z, info->dst.box.depth, 632 dst_aux_usage); 633 } 634 635 blorp_batch_finish(&blorp_batch); 636 637 crocus_flush_and_dirty_for_history(ice, batch, (struct crocus_resource *) 638 info->dst.resource, 639 PIPE_CONTROL_RENDER_TARGET_FLUSH, 640 "cache history: post-blit"); 641} 642 643static void 644get_copy_region_aux_settings(struct crocus_resource *res, 645 enum isl_aux_usage *out_aux_usage, 646 bool is_render_target) 647{ 648 switch (res->aux.usage) { 649 case ISL_AUX_USAGE_MCS: 650 /* A stencil resolve operation must be performed prior to doing resource 651 * copies or used by CPU. 652 * (see HSD 1209978162) 653 */ 654 if (is_render_target && isl_surf_usage_is_stencil(res->surf.usage)) { 655 *out_aux_usage = ISL_AUX_USAGE_NONE; 656 } else { 657 *out_aux_usage = res->aux.usage; 658 } 659 break; 660 default: 661 *out_aux_usage = ISL_AUX_USAGE_NONE; 662 break; 663 } 664} 665 666/** 667 * Perform a GPU-based raw memory copy between compatible view classes. 668 * 669 * Does not perform any flushing - the new data may still be left in the 670 * render cache, and old data may remain in other caches. 671 * 672 * Wraps blorp_copy() and blorp_buffer_copy(). 673 */ 674void 675crocus_copy_region(struct blorp_context *blorp, 676 struct crocus_batch *batch, 677 struct pipe_resource *dst, 678 unsigned dst_level, 679 unsigned dstx, unsigned dsty, unsigned dstz, 680 struct pipe_resource *src, 681 unsigned src_level, 682 const struct pipe_box *src_box) 683{ 684 struct blorp_batch blorp_batch; 685 struct crocus_context *ice = blorp->driver_ctx; 686 struct crocus_screen *screen = (void *) ice->ctx.screen; 687 const struct intel_device_info *devinfo = &screen->devinfo; 688 struct crocus_resource *src_res = (void *) src; 689 struct crocus_resource *dst_res = (void *) dst; 690 691 if (devinfo->ver <= 5) { 692 if (screen->vtbl.copy_region_blt(batch, dst_res, 693 dst_level, dstx, dsty, dstz, 694 src_res, src_level, src_box)) 695 return; 696 } 697 enum isl_aux_usage src_aux_usage, dst_aux_usage; 698 get_copy_region_aux_settings(src_res, &src_aux_usage, 699 false); 700 get_copy_region_aux_settings(dst_res, &dst_aux_usage, 701 true); 702 703 if (crocus_batch_references(batch, src_res->bo)) 704 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format); 705 706 if (dst->target == PIPE_BUFFER) 707 util_range_add(&dst_res->base.b, &dst_res->valid_buffer_range, dstx, dstx + src_box->width); 708 709 if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) { 710 struct blorp_address src_addr = { 711 .buffer = crocus_resource_bo(src), .offset = src_box->x, 712 .mocs = crocus_mocs(src_res->bo, &screen->isl_dev), 713 }; 714 struct blorp_address dst_addr = { 715 .buffer = crocus_resource_bo(dst), .offset = dstx, 716 .mocs = crocus_mocs(dst_res->bo, &screen->isl_dev), 717 .reloc_flags = EXEC_OBJECT_WRITE, 718 }; 719 720 crocus_batch_maybe_flush(batch, 1500); 721 722 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0); 723 blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width); 724 blorp_batch_finish(&blorp_batch); 725 } else { 726 // XXX: what about one surface being a buffer and not the other? 727 728 struct blorp_surf src_surf, dst_surf; 729 crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &src_surf, 730 src, src_aux_usage, src_level, false); 731 crocus_blorp_surf_for_resource(&screen->vtbl, &screen->isl_dev, &dst_surf, 732 dst, dst_aux_usage, dst_level, true); 733 734 crocus_resource_prepare_access(ice, src_res, src_level, 1, 735 src_box->z, src_box->depth, 736 src_aux_usage, false); 737 crocus_resource_prepare_access(ice, dst_res, dst_level, 1, 738 dstz, src_box->depth, 739 dst_aux_usage, false); 740 741 blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0); 742 743 for (int slice = 0; slice < src_box->depth; slice++) { 744 crocus_batch_maybe_flush(batch, 1500); 745 746 blorp_copy(&blorp_batch, &src_surf, src_level, src_box->z + slice, 747 &dst_surf, dst_level, dstz + slice, 748 src_box->x, src_box->y, dstx, dsty, 749 src_box->width, src_box->height); 750 } 751 blorp_batch_finish(&blorp_batch); 752 753 crocus_resource_finish_write(ice, dst_res, dst_level, dstz, 754 src_box->depth, dst_aux_usage); 755 } 756 757 tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format); 758} 759 760/** 761 * The pipe->resource_copy_region() driver hook. 762 * 763 * This implements ARB_copy_image semantics - a raw memory copy between 764 * compatible view classes. 765 */ 766static void 767crocus_resource_copy_region(struct pipe_context *ctx, 768 struct pipe_resource *p_dst, 769 unsigned dst_level, 770 unsigned dstx, unsigned dsty, unsigned dstz, 771 struct pipe_resource *p_src, 772 unsigned src_level, 773 const struct pipe_box *src_box) 774{ 775 struct crocus_context *ice = (void *) ctx; 776 struct crocus_batch *batch = &ice->batches[CROCUS_BATCH_RENDER]; 777 struct crocus_screen *screen = (struct crocus_screen *)ctx->screen; 778 const struct intel_device_info *devinfo = &screen->devinfo; 779 struct crocus_resource *src = (void *) p_src; 780 struct crocus_resource *dst = (void *) p_dst; 781 782 if (crocus_resource_unfinished_aux_import(src)) 783 crocus_resource_finish_aux_import(ctx->screen, src); 784 if (crocus_resource_unfinished_aux_import(dst)) 785 crocus_resource_finish_aux_import(ctx->screen, dst); 786 787 if (devinfo->ver < 6 && util_format_is_depth_or_stencil(p_dst->format)) { 788 util_resource_copy_region(ctx, p_dst, dst_level, dstx, dsty, dstz, 789 p_src, src_level, src_box); 790 return; 791 } 792 crocus_copy_region(&ice->blorp, batch, p_dst, dst_level, dstx, dsty, dstz, 793 p_src, src_level, src_box); 794 795 if (util_format_is_depth_and_stencil(p_dst->format) && 796 util_format_has_stencil(util_format_description(p_src->format)) && 797 devinfo->ver >= 6) { 798 struct crocus_resource *junk, *s_src_res, *s_dst_res; 799 crocus_get_depth_stencil_resources(devinfo, p_src, &junk, &s_src_res); 800 crocus_get_depth_stencil_resources(devinfo, p_dst, &junk, &s_dst_res); 801 802 crocus_copy_region(&ice->blorp, batch, &s_dst_res->base.b, dst_level, dstx, 803 dsty, dstz, &s_src_res->base.b, src_level, src_box); 804 } 805 806 crocus_flush_and_dirty_for_history(ice, batch, dst, 807 PIPE_CONTROL_RENDER_TARGET_FLUSH, 808 "cache history: post copy_region"); 809} 810 811void 812crocus_init_blit_functions(struct pipe_context *ctx) 813{ 814 ctx->blit = crocus_blit; 815 ctx->resource_copy_region = crocus_resource_copy_region; 816} 817