1/* 2 * Copyright © 2017 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included 12 * in all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 * DEALINGS IN THE SOFTWARE. 21 */ 22 23/** 24 * @file iris_resource.c 25 * 26 * Resources are images, buffers, and other objects used by the GPU. 27 * 28 * XXX: explain resources 29 */ 30 31#include <stdio.h> 32#include <errno.h> 33#include "pipe/p_defines.h" 34#include "pipe/p_state.h" 35#include "pipe/p_context.h" 36#include "pipe/p_screen.h" 37#include "util/os_memory.h" 38#include "util/u_cpu_detect.h" 39#include "util/u_inlines.h" 40#include "util/format/u_format.h" 41#include "util/u_memory.h" 42#include "util/u_threaded_context.h" 43#include "util/u_transfer.h" 44#include "util/u_transfer_helper.h" 45#include "util/u_upload_mgr.h" 46#include "util/ralloc.h" 47#include "iris_batch.h" 48#include "iris_context.h" 49#include "iris_resource.h" 50#include "iris_screen.h" 51#include "intel/common/intel_aux_map.h" 52#include "intel/dev/intel_debug.h" 53#include "isl/isl.h" 54#include "drm-uapi/drm_fourcc.h" 55#include "drm-uapi/i915_drm.h" 56 57enum modifier_priority { 58 MODIFIER_PRIORITY_INVALID = 0, 59 MODIFIER_PRIORITY_LINEAR, 60 MODIFIER_PRIORITY_X, 61 MODIFIER_PRIORITY_Y, 62 MODIFIER_PRIORITY_Y_CCS, 63 MODIFIER_PRIORITY_Y_GFX12_RC_CCS, 64 MODIFIER_PRIORITY_Y_GFX12_RC_CCS_CC, 65 MODIFIER_PRIORITY_4, 66 MODIFIER_PRIORITY_4_DG2_RC_CCS, 67 MODIFIER_PRIORITY_4_DG2_RC_CCS_CC, 68}; 69 70static const uint64_t priority_to_modifier[] = { 71 [MODIFIER_PRIORITY_INVALID] = DRM_FORMAT_MOD_INVALID, 72 [MODIFIER_PRIORITY_LINEAR] = DRM_FORMAT_MOD_LINEAR, 73 [MODIFIER_PRIORITY_X] = I915_FORMAT_MOD_X_TILED, 74 [MODIFIER_PRIORITY_Y] = I915_FORMAT_MOD_Y_TILED, 75 [MODIFIER_PRIORITY_Y_CCS] = I915_FORMAT_MOD_Y_TILED_CCS, 76 [MODIFIER_PRIORITY_Y_GFX12_RC_CCS] = I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS, 77 [MODIFIER_PRIORITY_Y_GFX12_RC_CCS_CC] = I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC, 78 [MODIFIER_PRIORITY_4] = I915_FORMAT_MOD_4_TILED, 79 [MODIFIER_PRIORITY_4_DG2_RC_CCS] = I915_FORMAT_MOD_4_TILED_DG2_RC_CCS, 80 [MODIFIER_PRIORITY_4_DG2_RC_CCS_CC] = I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC, 81}; 82 83static bool 84modifier_is_supported(const struct intel_device_info *devinfo, 85 enum pipe_format pfmt, unsigned bind, 86 uint64_t modifier) 87{ 88 /* Check for basic device support. */ 89 switch (modifier) { 90 case DRM_FORMAT_MOD_LINEAR: 91 case I915_FORMAT_MOD_X_TILED: 92 break; 93 case I915_FORMAT_MOD_Y_TILED: 94 if (devinfo->ver <= 8 && (bind & PIPE_BIND_SCANOUT)) 95 return false; 96 if (devinfo->verx10 >= 125) 97 return false; 98 break; 99 case I915_FORMAT_MOD_Y_TILED_CCS: 100 if (devinfo->ver <= 8 || devinfo->ver >= 12) 101 return false; 102 break; 103 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: 104 case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: 105 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: 106 if (devinfo->verx10 != 120) 107 return false; 108 break; 109 case I915_FORMAT_MOD_4_TILED: 110 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS: 111 case I915_FORMAT_MOD_4_TILED_DG2_MC_CCS: 112 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC: 113 if (devinfo->verx10 < 125) 114 return false; 115 break; 116 case DRM_FORMAT_MOD_INVALID: 117 default: 118 return false; 119 } 120 121 /* Check remaining requirements. */ 122 switch (modifier) { 123 case I915_FORMAT_MOD_4_TILED_DG2_MC_CCS: 124 case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: 125 if (INTEL_DEBUG(DEBUG_NO_CCS)) 126 return false; 127 128 if (pfmt != PIPE_FORMAT_BGRA8888_UNORM && 129 pfmt != PIPE_FORMAT_RGBA8888_UNORM && 130 pfmt != PIPE_FORMAT_BGRX8888_UNORM && 131 pfmt != PIPE_FORMAT_RGBX8888_UNORM && 132 pfmt != PIPE_FORMAT_NV12 && 133 pfmt != PIPE_FORMAT_P010 && 134 pfmt != PIPE_FORMAT_P012 && 135 pfmt != PIPE_FORMAT_P016 && 136 pfmt != PIPE_FORMAT_YUYV && 137 pfmt != PIPE_FORMAT_UYVY) { 138 return false; 139 } 140 break; 141 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC: 142 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS: 143 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: 144 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: 145 case I915_FORMAT_MOD_Y_TILED_CCS: { 146 if (INTEL_DEBUG(DEBUG_NO_CCS)) 147 return false; 148 149 enum isl_format rt_format = 150 iris_format_for_usage(devinfo, pfmt, 151 ISL_SURF_USAGE_RENDER_TARGET_BIT).fmt; 152 153 if (rt_format == ISL_FORMAT_UNSUPPORTED || 154 !isl_format_supports_ccs_e(devinfo, rt_format)) 155 return false; 156 break; 157 } 158 default: 159 break; 160 } 161 162 return true; 163} 164 165static uint64_t 166select_best_modifier(struct intel_device_info *devinfo, 167 const struct pipe_resource *templ, 168 const uint64_t *modifiers, 169 int count) 170{ 171 enum modifier_priority prio = MODIFIER_PRIORITY_INVALID; 172 173 for (int i = 0; i < count; i++) { 174 if (!modifier_is_supported(devinfo, templ->format, templ->bind, 175 modifiers[i])) 176 continue; 177 178 switch (modifiers[i]) { 179 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC: 180 prio = MAX2(prio, MODIFIER_PRIORITY_4_DG2_RC_CCS_CC); 181 break; 182 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS: 183 prio = MAX2(prio, MODIFIER_PRIORITY_4_DG2_RC_CCS); 184 break; 185 case I915_FORMAT_MOD_4_TILED: 186 prio = MAX2(prio, MODIFIER_PRIORITY_4); 187 break; 188 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: 189 prio = MAX2(prio, MODIFIER_PRIORITY_Y_GFX12_RC_CCS_CC); 190 break; 191 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: 192 prio = MAX2(prio, MODIFIER_PRIORITY_Y_GFX12_RC_CCS); 193 break; 194 case I915_FORMAT_MOD_Y_TILED_CCS: 195 prio = MAX2(prio, MODIFIER_PRIORITY_Y_CCS); 196 break; 197 case I915_FORMAT_MOD_Y_TILED: 198 prio = MAX2(prio, MODIFIER_PRIORITY_Y); 199 break; 200 case I915_FORMAT_MOD_X_TILED: 201 prio = MAX2(prio, MODIFIER_PRIORITY_X); 202 break; 203 case DRM_FORMAT_MOD_LINEAR: 204 prio = MAX2(prio, MODIFIER_PRIORITY_LINEAR); 205 break; 206 case DRM_FORMAT_MOD_INVALID: 207 default: 208 break; 209 } 210 } 211 212 return priority_to_modifier[prio]; 213} 214 215static inline bool is_modifier_external_only(enum pipe_format pfmt, 216 uint64_t modifier) 217{ 218 /* Only allow external usage for the following cases: YUV formats 219 * and the media-compression modifier. The render engine lacks 220 * support for rendering to a media-compressed surface if the 221 * compression ratio is large enough. By requiring external usage 222 * of media-compressed surfaces, resolves are avoided. 223 */ 224 return util_format_is_yuv(pfmt) || 225 isl_drm_modifier_get_info(modifier)->aux_usage == ISL_AUX_USAGE_MC; 226} 227 228static void 229iris_query_dmabuf_modifiers(struct pipe_screen *pscreen, 230 enum pipe_format pfmt, 231 int max, 232 uint64_t *modifiers, 233 unsigned int *external_only, 234 int *count) 235{ 236 struct iris_screen *screen = (void *) pscreen; 237 const struct intel_device_info *devinfo = &screen->devinfo; 238 239 uint64_t all_modifiers[] = { 240 DRM_FORMAT_MOD_LINEAR, 241 I915_FORMAT_MOD_X_TILED, 242 I915_FORMAT_MOD_4_TILED, 243 I915_FORMAT_MOD_4_TILED_DG2_RC_CCS, 244 I915_FORMAT_MOD_4_TILED_DG2_MC_CCS, 245 I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC, 246 I915_FORMAT_MOD_Y_TILED, 247 I915_FORMAT_MOD_Y_TILED_CCS, 248 I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS, 249 I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS, 250 I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC, 251 }; 252 253 int supported_mods = 0; 254 255 for (int i = 0; i < ARRAY_SIZE(all_modifiers); i++) { 256 if (!modifier_is_supported(devinfo, pfmt, 0, all_modifiers[i])) 257 continue; 258 259 if (supported_mods < max) { 260 if (modifiers) 261 modifiers[supported_mods] = all_modifiers[i]; 262 263 if (external_only) { 264 external_only[supported_mods] = 265 is_modifier_external_only(pfmt, all_modifiers[i]); 266 } 267 } 268 269 supported_mods++; 270 } 271 272 *count = supported_mods; 273} 274 275static bool 276iris_is_dmabuf_modifier_supported(struct pipe_screen *pscreen, 277 uint64_t modifier, enum pipe_format pfmt, 278 bool *external_only) 279{ 280 struct iris_screen *screen = (void *) pscreen; 281 const struct intel_device_info *devinfo = &screen->devinfo; 282 283 if (modifier_is_supported(devinfo, pfmt, 0, modifier)) { 284 if (external_only) 285 *external_only = is_modifier_external_only(pfmt, modifier); 286 287 return true; 288 } 289 290 return false; 291} 292 293static unsigned int 294iris_get_dmabuf_modifier_planes(struct pipe_screen *pscreen, uint64_t modifier, 295 enum pipe_format format) 296{ 297 unsigned int planes = util_format_get_num_planes(format); 298 299 switch (modifier) { 300 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: 301 return 3; 302 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC: 303 case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: 304 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: 305 case I915_FORMAT_MOD_Y_TILED_CCS: 306 return 2 * planes; 307 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS: 308 case I915_FORMAT_MOD_4_TILED_DG2_MC_CCS: 309 default: 310 return planes; 311 } 312} 313 314enum isl_format 315iris_image_view_get_format(struct iris_context *ice, 316 const struct pipe_image_view *img) 317{ 318 struct iris_screen *screen = (struct iris_screen *)ice->ctx.screen; 319 const struct intel_device_info *devinfo = &screen->devinfo; 320 321 isl_surf_usage_flags_t usage = ISL_SURF_USAGE_STORAGE_BIT; 322 enum isl_format isl_fmt = 323 iris_format_for_usage(devinfo, img->format, usage).fmt; 324 325 if (img->shader_access & PIPE_IMAGE_ACCESS_READ) { 326 /* On Gfx8, try to use typed surfaces reads (which support a 327 * limited number of formats), and if not possible, fall back 328 * to untyped reads. 329 */ 330 if (devinfo->ver == 8 && 331 !isl_has_matching_typed_storage_image_format(devinfo, isl_fmt)) 332 return ISL_FORMAT_RAW; 333 else 334 return isl_lower_storage_image_format(devinfo, isl_fmt); 335 } 336 337 return isl_fmt; 338} 339 340static struct pipe_memory_object * 341iris_memobj_create_from_handle(struct pipe_screen *pscreen, 342 struct winsys_handle *whandle, 343 bool dedicated) 344{ 345 struct iris_screen *screen = (struct iris_screen *)pscreen; 346 struct iris_memory_object *memobj = CALLOC_STRUCT(iris_memory_object); 347 struct iris_bo *bo; 348 349 if (!memobj) 350 return NULL; 351 352 switch (whandle->type) { 353 case WINSYS_HANDLE_TYPE_SHARED: 354 bo = iris_bo_gem_create_from_name(screen->bufmgr, "winsys image", 355 whandle->handle); 356 break; 357 case WINSYS_HANDLE_TYPE_FD: 358 bo = iris_bo_import_dmabuf(screen->bufmgr, whandle->handle); 359 break; 360 default: 361 unreachable("invalid winsys handle type"); 362 } 363 364 if (!bo) { 365 free(memobj); 366 return NULL; 367 } 368 369 memobj->b.dedicated = dedicated; 370 memobj->bo = bo; 371 memobj->format = whandle->format; 372 memobj->stride = whandle->stride; 373 374 return &memobj->b; 375} 376 377static void 378iris_memobj_destroy(struct pipe_screen *pscreen, 379 struct pipe_memory_object *pmemobj) 380{ 381 struct iris_memory_object *memobj = (struct iris_memory_object *)pmemobj; 382 383 iris_bo_unreference(memobj->bo); 384 free(memobj); 385} 386 387struct pipe_resource * 388iris_resource_get_separate_stencil(struct pipe_resource *p_res) 389{ 390 /* For packed depth-stencil, we treat depth as the primary resource 391 * and store S8 as the "second plane" resource. 392 */ 393 if (p_res->next && p_res->next->format == PIPE_FORMAT_S8_UINT) 394 return p_res->next; 395 396 return NULL; 397 398} 399 400static void 401iris_resource_set_separate_stencil(struct pipe_resource *p_res, 402 struct pipe_resource *stencil) 403{ 404 assert(util_format_has_depth(util_format_description(p_res->format))); 405 pipe_resource_reference(&p_res->next, stencil); 406} 407 408void 409iris_get_depth_stencil_resources(struct pipe_resource *res, 410 struct iris_resource **out_z, 411 struct iris_resource **out_s) 412{ 413 if (!res) { 414 *out_z = NULL; 415 *out_s = NULL; 416 return; 417 } 418 419 if (res->format != PIPE_FORMAT_S8_UINT) { 420 *out_z = (void *) res; 421 *out_s = (void *) iris_resource_get_separate_stencil(res); 422 } else { 423 *out_z = NULL; 424 *out_s = (void *) res; 425 } 426} 427 428void 429iris_resource_disable_aux(struct iris_resource *res) 430{ 431 iris_bo_unreference(res->aux.bo); 432 iris_bo_unreference(res->aux.clear_color_bo); 433 free(res->aux.state); 434 435 res->aux.usage = ISL_AUX_USAGE_NONE; 436 res->aux.surf.size_B = 0; 437 res->aux.bo = NULL; 438 res->aux.extra_aux.surf.size_B = 0; 439 res->aux.clear_color_bo = NULL; 440 res->aux.state = NULL; 441} 442 443static uint32_t 444iris_resource_alloc_flags(const struct iris_screen *screen, 445 const struct pipe_resource *templ, 446 enum isl_aux_usage aux_usage) 447{ 448 if (templ->flags & IRIS_RESOURCE_FLAG_DEVICE_MEM) 449 return 0; 450 451 uint32_t flags = 0; 452 453 switch (templ->usage) { 454 case PIPE_USAGE_STAGING: 455 flags |= BO_ALLOC_SMEM | BO_ALLOC_COHERENT; 456 break; 457 case PIPE_USAGE_STREAM: 458 flags |= BO_ALLOC_SMEM; 459 break; 460 case PIPE_USAGE_DYNAMIC: 461 case PIPE_USAGE_DEFAULT: 462 case PIPE_USAGE_IMMUTABLE: 463 /* Use LMEM for these if possible */ 464 break; 465 } 466 467 if (templ->bind & PIPE_BIND_SCANOUT) 468 flags |= BO_ALLOC_SCANOUT; 469 470 if (templ->flags & (PIPE_RESOURCE_FLAG_MAP_COHERENT | 471 PIPE_RESOURCE_FLAG_MAP_PERSISTENT)) 472 flags |= BO_ALLOC_SMEM; 473 474 if (screen->devinfo.verx10 >= 125 && isl_aux_usage_has_ccs(aux_usage)) { 475 assert((flags & BO_ALLOC_SMEM) == 0); 476 flags |= BO_ALLOC_LMEM; 477 } 478 479 if ((templ->bind & PIPE_BIND_SHARED) || 480 util_format_get_num_planes(templ->format) > 1) 481 flags |= BO_ALLOC_NO_SUBALLOC; 482 483 return flags; 484} 485 486static void 487iris_resource_destroy(struct pipe_screen *screen, 488 struct pipe_resource *p_res) 489{ 490 struct iris_resource *res = (struct iris_resource *) p_res; 491 492 if (p_res->target == PIPE_BUFFER) 493 util_range_destroy(&res->valid_buffer_range); 494 495 iris_resource_disable_aux(res); 496 497 threaded_resource_deinit(p_res); 498 iris_bo_unreference(res->bo); 499 iris_pscreen_unref(res->orig_screen); 500 501 free(res); 502} 503 504static struct iris_resource * 505iris_alloc_resource(struct pipe_screen *pscreen, 506 const struct pipe_resource *templ) 507{ 508 struct iris_resource *res = calloc(1, sizeof(struct iris_resource)); 509 if (!res) 510 return NULL; 511 512 res->base.b = *templ; 513 res->base.b.screen = pscreen; 514 res->orig_screen = iris_pscreen_ref(pscreen); 515 pipe_reference_init(&res->base.b.reference, 1); 516 threaded_resource_init(&res->base.b, false); 517 518 if (templ->target == PIPE_BUFFER) 519 util_range_init(&res->valid_buffer_range); 520 521 return res; 522} 523 524unsigned 525iris_get_num_logical_layers(const struct iris_resource *res, unsigned level) 526{ 527 if (res->surf.dim == ISL_SURF_DIM_3D) 528 return u_minify(res->surf.logical_level0_px.depth, level); 529 else 530 return res->surf.logical_level0_px.array_len; 531} 532 533static enum isl_aux_state ** 534create_aux_state_map(struct iris_resource *res, enum isl_aux_state initial) 535{ 536 assert(res->aux.state == NULL); 537 538 uint32_t total_slices = 0; 539 for (uint32_t level = 0; level < res->surf.levels; level++) 540 total_slices += iris_get_num_logical_layers(res, level); 541 542 const size_t per_level_array_size = 543 res->surf.levels * sizeof(enum isl_aux_state *); 544 545 /* We're going to allocate a single chunk of data for both the per-level 546 * reference array and the arrays of aux_state. This makes cleanup 547 * significantly easier. 548 */ 549 const size_t total_size = 550 per_level_array_size + total_slices * sizeof(enum isl_aux_state); 551 552 void *data = malloc(total_size); 553 if (!data) 554 return NULL; 555 556 enum isl_aux_state **per_level_arr = data; 557 enum isl_aux_state *s = data + per_level_array_size; 558 for (uint32_t level = 0; level < res->surf.levels; level++) { 559 per_level_arr[level] = s; 560 const unsigned level_layers = iris_get_num_logical_layers(res, level); 561 for (uint32_t a = 0; a < level_layers; a++) 562 *(s++) = initial; 563 } 564 assert((void *)s == data + total_size); 565 566 return per_level_arr; 567} 568 569static unsigned 570iris_get_aux_clear_color_state_size(struct iris_screen *screen, 571 struct iris_resource *res) 572{ 573 if (!isl_aux_usage_has_fast_clears(res->aux.usage)) 574 return 0; 575 576 assert(!isl_surf_usage_is_stencil(res->surf.usage)); 577 578 /* Depth packets can't specify indirect clear values. The only time depth 579 * buffers can use indirect clear values is when they're accessed by the 580 * sampler via render surface state objects. 581 */ 582 if (isl_surf_usage_is_depth(res->surf.usage) && 583 !iris_sample_with_depth_aux(&screen->devinfo, res)) 584 return 0; 585 586 return screen->isl_dev.ss.clear_color_state_size; 587} 588 589static void 590map_aux_addresses(struct iris_screen *screen, struct iris_resource *res, 591 enum pipe_format pfmt, unsigned plane) 592{ 593 void *aux_map_ctx = iris_bufmgr_get_aux_map_context(screen->bufmgr); 594 if (!aux_map_ctx) 595 return; 596 597 if (isl_aux_usage_has_ccs(res->aux.usage)) { 598 const unsigned aux_offset = res->aux.extra_aux.surf.size_B > 0 ? 599 res->aux.extra_aux.offset : res->aux.offset; 600 const enum isl_format format = 601 iris_format_for_usage(&screen->devinfo, pfmt, res->surf.usage).fmt; 602 const uint64_t format_bits = 603 intel_aux_map_format_bits(res->surf.tiling, format, plane); 604 intel_aux_map_add_mapping(aux_map_ctx, res->bo->address + res->offset, 605 res->aux.bo->address + aux_offset, 606 res->surf.size_B, format_bits); 607 res->bo->aux_map_address = res->aux.bo->address; 608 } 609} 610 611static bool 612want_ccs_e_for_format(const struct intel_device_info *devinfo, 613 enum isl_format format) 614{ 615 if (!isl_format_supports_ccs_e(devinfo, format)) 616 return false; 617 618 const struct isl_format_layout *fmtl = isl_format_get_layout(format); 619 620 /* Prior to TGL, CCS_E seems to significantly hurt performance with 32-bit 621 * floating point formats. For example, Paraview's "Wavelet Volume" case 622 * uses both R32_FLOAT and R32G32B32A32_FLOAT, and enabling CCS_E for those 623 * formats causes a 62% FPS drop. 624 * 625 * However, many benchmarks seem to use 16-bit float with no issues. 626 */ 627 if (devinfo->ver <= 11 && 628 fmtl->channels.r.bits == 32 && fmtl->channels.r.type == ISL_SFLOAT) 629 return false; 630 631 return true; 632} 633 634static enum isl_surf_dim 635target_to_isl_surf_dim(enum pipe_texture_target target) 636{ 637 switch (target) { 638 case PIPE_BUFFER: 639 case PIPE_TEXTURE_1D: 640 case PIPE_TEXTURE_1D_ARRAY: 641 return ISL_SURF_DIM_1D; 642 case PIPE_TEXTURE_2D: 643 case PIPE_TEXTURE_CUBE: 644 case PIPE_TEXTURE_RECT: 645 case PIPE_TEXTURE_2D_ARRAY: 646 case PIPE_TEXTURE_CUBE_ARRAY: 647 return ISL_SURF_DIM_2D; 648 case PIPE_TEXTURE_3D: 649 return ISL_SURF_DIM_3D; 650 case PIPE_MAX_TEXTURE_TYPES: 651 break; 652 } 653 unreachable("invalid texture type"); 654} 655 656static bool 657iris_resource_configure_main(const struct iris_screen *screen, 658 struct iris_resource *res, 659 const struct pipe_resource *templ, 660 uint64_t modifier, uint32_t row_pitch_B) 661{ 662 res->mod_info = isl_drm_modifier_get_info(modifier); 663 664 if (modifier != DRM_FORMAT_MOD_INVALID && res->mod_info == NULL) 665 return false; 666 667 isl_tiling_flags_t tiling_flags = 0; 668 669 if (res->mod_info != NULL) { 670 tiling_flags = 1 << res->mod_info->tiling; 671 } else if (templ->usage == PIPE_USAGE_STAGING || 672 templ->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR)) { 673 tiling_flags = ISL_TILING_LINEAR_BIT; 674 } else if (!screen->devinfo.has_tiling_uapi && 675 (templ->bind & (PIPE_BIND_SCANOUT | PIPE_BIND_SHARED))) { 676 tiling_flags = ISL_TILING_LINEAR_BIT; 677 } else if (templ->bind & PIPE_BIND_SCANOUT) { 678 tiling_flags = ISL_TILING_X_BIT; 679 } else { 680 tiling_flags = ISL_TILING_ANY_MASK; 681 } 682 683 isl_surf_usage_flags_t usage = 0; 684 685 if (res->mod_info && res->mod_info->aux_usage == ISL_AUX_USAGE_NONE) 686 usage |= ISL_SURF_USAGE_DISABLE_AUX_BIT; 687 688 if (templ->usage == PIPE_USAGE_STAGING) 689 usage |= ISL_SURF_USAGE_STAGING_BIT; 690 691 if (templ->bind & PIPE_BIND_RENDER_TARGET) 692 usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT; 693 694 if (templ->bind & PIPE_BIND_SAMPLER_VIEW) 695 usage |= ISL_SURF_USAGE_TEXTURE_BIT; 696 697 if (templ->bind & PIPE_BIND_SHADER_IMAGE) 698 usage |= ISL_SURF_USAGE_STORAGE_BIT; 699 700 if (templ->bind & PIPE_BIND_SCANOUT) 701 usage |= ISL_SURF_USAGE_DISPLAY_BIT; 702 703 if (templ->target == PIPE_TEXTURE_CUBE || 704 templ->target == PIPE_TEXTURE_CUBE_ARRAY) { 705 usage |= ISL_SURF_USAGE_CUBE_BIT; 706 } 707 708 if (templ->usage != PIPE_USAGE_STAGING && 709 util_format_is_depth_or_stencil(templ->format)) { 710 711 /* Should be handled by u_transfer_helper */ 712 assert(!util_format_is_depth_and_stencil(templ->format)); 713 714 usage |= templ->format == PIPE_FORMAT_S8_UINT ? 715 ISL_SURF_USAGE_STENCIL_BIT : ISL_SURF_USAGE_DEPTH_BIT; 716 } 717 718 const enum isl_format format = 719 iris_format_for_usage(&screen->devinfo, templ->format, usage).fmt; 720 721 const struct isl_surf_init_info init_info = { 722 .dim = target_to_isl_surf_dim(templ->target), 723 .format = format, 724 .width = templ->width0, 725 .height = templ->height0, 726 .depth = templ->depth0, 727 .levels = templ->last_level + 1, 728 .array_len = templ->array_size, 729 .samples = MAX2(templ->nr_samples, 1), 730 .min_alignment_B = 0, 731 .row_pitch_B = row_pitch_B, 732 .usage = usage, 733 .tiling_flags = tiling_flags 734 }; 735 736 if (!isl_surf_init_s(&screen->isl_dev, &res->surf, &init_info)) 737 return false; 738 739 res->internal_format = templ->format; 740 741 return true; 742} 743 744static bool 745iris_get_ccs_surf_or_support(const struct isl_device *dev, 746 const struct isl_surf *surf, 747 struct isl_surf *aux_surf, 748 struct isl_surf *extra_aux_surf) 749{ 750 assert(extra_aux_surf->size_B == 0); 751 752 struct isl_surf *ccs_surf; 753 const struct isl_surf *hiz_or_mcs_surf; 754 if (aux_surf->size_B > 0) { 755 assert(aux_surf->usage & (ISL_SURF_USAGE_HIZ_BIT | 756 ISL_SURF_USAGE_MCS_BIT)); 757 hiz_or_mcs_surf = aux_surf; 758 ccs_surf = extra_aux_surf; 759 } else { 760 hiz_or_mcs_surf = NULL; 761 ccs_surf = aux_surf; 762 } 763 764 if (dev->info->verx10 >= 125) { 765 /* CCS doesn't require VMA on XeHP. So, instead of creating a separate 766 * surface, we can just return whether CCS is supported for the given 767 * input surfaces. 768 */ 769 return isl_surf_supports_ccs(dev, surf, hiz_or_mcs_surf); 770 } else { 771 return isl_surf_get_ccs_surf(dev, surf, hiz_or_mcs_surf, ccs_surf, 0); 772 } 773} 774 775/** 776 * Configure aux for the resource, but don't allocate it. For images which 777 * might be shared with modifiers, we must allocate the image and aux data in 778 * a single bo. 779 * 780 * Returns false on unexpected error (e.g. allocation failed, or invalid 781 * configuration result). 782 */ 783static bool 784iris_resource_configure_aux(struct iris_screen *screen, 785 struct iris_resource *res, bool imported) 786{ 787 const struct intel_device_info *devinfo = &screen->devinfo; 788 789 const bool has_mcs = 790 isl_surf_get_mcs_surf(&screen->isl_dev, &res->surf, &res->aux.surf); 791 792 const bool has_hiz = !INTEL_DEBUG(DEBUG_NO_HIZ) && 793 isl_surf_get_hiz_surf(&screen->isl_dev, &res->surf, &res->aux.surf); 794 795 const bool has_ccs = !INTEL_DEBUG(DEBUG_NO_CCS) && 796 iris_get_ccs_surf_or_support(&screen->isl_dev, &res->surf, 797 &res->aux.surf, &res->aux.extra_aux.surf); 798 799 if (has_mcs) { 800 assert(!res->mod_info); 801 assert(!has_hiz); 802 if (has_ccs) { 803 res->aux.usage = ISL_AUX_USAGE_MCS_CCS; 804 } else { 805 res->aux.usage = ISL_AUX_USAGE_MCS; 806 } 807 } else if (has_hiz) { 808 assert(!res->mod_info); 809 assert(!has_mcs); 810 if (!has_ccs) { 811 res->aux.usage = ISL_AUX_USAGE_HIZ; 812 } else if (res->surf.samples == 1 && 813 (res->surf.usage & ISL_SURF_USAGE_TEXTURE_BIT)) { 814 /* If this resource is single-sampled and will be used as a texture, 815 * put the HiZ surface in write-through mode so that we can sample 816 * from it. 817 */ 818 res->aux.usage = ISL_AUX_USAGE_HIZ_CCS_WT; 819 } else { 820 res->aux.usage = ISL_AUX_USAGE_HIZ_CCS; 821 } 822 } else if (has_ccs) { 823 if (res->mod_info) { 824 res->aux.usage = res->mod_info->aux_usage; 825 } else if (isl_surf_usage_is_stencil(res->surf.usage)) { 826 res->aux.usage = ISL_AUX_USAGE_STC_CCS; 827 } else if (want_ccs_e_for_format(devinfo, res->surf.format)) { 828 res->aux.usage = devinfo->ver < 12 ? 829 ISL_AUX_USAGE_CCS_E : ISL_AUX_USAGE_GFX12_CCS_E; 830 } else { 831 assert(isl_format_supports_ccs_d(devinfo, res->surf.format)); 832 res->aux.usage = ISL_AUX_USAGE_CCS_D; 833 } 834 } 835 836 enum isl_aux_state initial_state; 837 switch (res->aux.usage) { 838 case ISL_AUX_USAGE_NONE: 839 /* Having no aux buffer is only okay if there's no modifier with aux. */ 840 return !res->mod_info || res->mod_info->aux_usage == ISL_AUX_USAGE_NONE; 841 case ISL_AUX_USAGE_HIZ: 842 case ISL_AUX_USAGE_HIZ_CCS: 843 case ISL_AUX_USAGE_HIZ_CCS_WT: 844 initial_state = ISL_AUX_STATE_AUX_INVALID; 845 break; 846 case ISL_AUX_USAGE_MCS: 847 case ISL_AUX_USAGE_MCS_CCS: 848 /* The Ivybridge PRM, Vol 2 Part 1 p326 says: 849 * 850 * "When MCS buffer is enabled and bound to MSRT, it is required 851 * that it is cleared prior to any rendering." 852 * 853 * Since we only use the MCS buffer for rendering, we just clear it 854 * immediately on allocation. The clear value for MCS buffers is all 855 * 1's, so we simply memset it to 0xff. 856 */ 857 initial_state = ISL_AUX_STATE_CLEAR; 858 break; 859 case ISL_AUX_USAGE_CCS_D: 860 case ISL_AUX_USAGE_CCS_E: 861 case ISL_AUX_USAGE_GFX12_CCS_E: 862 case ISL_AUX_USAGE_STC_CCS: 863 case ISL_AUX_USAGE_MC: 864 if (imported) { 865 assert(res->aux.usage != ISL_AUX_USAGE_STC_CCS); 866 initial_state = 867 isl_drm_modifier_get_default_aux_state(res->mod_info->modifier); 868 } else if (devinfo->verx10 >= 125) { 869 assert(res->aux.surf.size_B == 0); 870 /* From Bspec 47709, "MCS/CCS Buffers for Render Target(s)": 871 * 872 * "CCS surface does not require initialization. Illegal CCS 873 * [values] are treated as uncompressed memory." 874 * 875 * Even if we wanted to, we can't initialize the CCS via CPU map. So, 876 * we choose an aux state which describes the current state and helps 877 * avoid ambiguating (something not currently supported for STC_CCS). 878 */ 879 assert(isl_aux_usage_has_compression(res->aux.usage)); 880 initial_state = isl_aux_usage_has_fast_clears(res->aux.usage) ? 881 ISL_AUX_STATE_COMPRESSED_CLEAR : 882 ISL_AUX_STATE_COMPRESSED_NO_CLEAR; 883 } else { 884 assert(res->aux.surf.size_B > 0); 885 /* When CCS is used, we need to ensure that it starts off in a valid 886 * state. From the Sky Lake PRM, "MCS Buffer for Render Target(s)": 887 * 888 * "If Software wants to enable Color Compression without Fast 889 * clear, Software needs to initialize MCS with zeros." 890 * 891 * A CCS surface initialized to zero is in the pass-through state. 892 * This state can avoid the need to ambiguate in some cases. We'll 893 * map and zero the CCS later on in iris_resource_init_aux_buf. 894 */ 895 initial_state = ISL_AUX_STATE_PASS_THROUGH; 896 } 897 break; 898 default: 899 unreachable("Unsupported aux mode"); 900 } 901 902 /* Create the aux_state for the auxiliary buffer. */ 903 res->aux.state = create_aux_state_map(res, initial_state); 904 if (!res->aux.state) 905 return false; 906 907 return true; 908} 909 910/** 911 * Initialize the aux buffer contents. 912 * 913 * Returns false on unexpected error (e.g. mapping a BO failed). 914 */ 915static bool 916iris_resource_init_aux_buf(struct iris_screen *screen, 917 struct iris_resource *res) 918{ 919 void *map = NULL; 920 921 if (iris_resource_get_aux_state(res, 0, 0) != ISL_AUX_STATE_AUX_INVALID && 922 res->aux.surf.size_B > 0) { 923 if (!map) 924 map = iris_bo_map(NULL, res->bo, MAP_WRITE | MAP_RAW); 925 if (!map) 926 return false; 927 928 /* See iris_resource_configure_aux for the memset_value rationale. */ 929 uint8_t memset_value = isl_aux_usage_has_mcs(res->aux.usage) ? 0xFF : 0; 930 memset((char*)map + res->aux.offset, memset_value, 931 res->aux.surf.size_B); 932 } 933 934 if (res->aux.extra_aux.surf.size_B > 0) { 935 if (!map) 936 map = iris_bo_map(NULL, res->bo, MAP_WRITE | MAP_RAW); 937 if (!map) 938 return false; 939 940 memset((char*)map + res->aux.extra_aux.offset, 941 0, res->aux.extra_aux.surf.size_B); 942 } 943 944 unsigned clear_color_size = iris_get_aux_clear_color_state_size(screen, res); 945 if (clear_color_size > 0) { 946 if (iris_bo_mmap_mode(res->bo) != IRIS_MMAP_NONE) { 947 if (!map) 948 map = iris_bo_map(NULL, res->bo, MAP_WRITE | MAP_RAW); 949 if (!map) 950 return false; 951 952 /* Zero the indirect clear color to match ::fast_clear_color. */ 953 memset((char *)map + res->aux.clear_color_offset, 0, clear_color_size); 954 } else { 955 res->aux.clear_color_unknown = true; 956 } 957 } 958 959 if (map) 960 iris_bo_unmap(res->bo); 961 962 if (res->aux.surf.size_B > 0) { 963 res->aux.bo = res->bo; 964 iris_bo_reference(res->aux.bo); 965 map_aux_addresses(screen, res, res->internal_format, 0); 966 } 967 968 if (clear_color_size > 0) { 969 res->aux.clear_color_bo = res->bo; 970 iris_bo_reference(res->aux.clear_color_bo); 971 } 972 973 return true; 974} 975 976static void 977import_aux_info(struct iris_resource *res, 978 const struct iris_resource *aux_res) 979{ 980 assert(aux_res->aux.surf.row_pitch_B && aux_res->aux.offset); 981 assert(res->bo == aux_res->aux.bo); 982 assert(res->aux.surf.row_pitch_B == aux_res->aux.surf.row_pitch_B); 983 assert(res->bo->size >= aux_res->aux.offset + res->aux.surf.size_B); 984 985 iris_bo_reference(aux_res->aux.bo); 986 res->aux.bo = aux_res->aux.bo; 987 res->aux.offset = aux_res->aux.offset; 988} 989 990static void 991iris_resource_finish_aux_import(struct pipe_screen *pscreen, 992 struct iris_resource *res) 993{ 994 struct iris_screen *screen = (struct iris_screen *)pscreen; 995 996 /* Create an array of resources. Combining main and aux planes is easier 997 * with indexing as opposed to scanning the linked list. 998 */ 999 struct iris_resource *r[4] = { NULL, }; 1000 unsigned num_planes = 0; 1001 unsigned num_main_planes = 0; 1002 for (struct pipe_resource *p_res = &res->base.b; p_res; p_res = p_res->next) { 1003 r[num_planes] = (struct iris_resource *)p_res; 1004 num_main_planes += r[num_planes++]->bo != NULL; 1005 } 1006 1007 /* Combine main and aux plane information. */ 1008 switch (res->mod_info->modifier) { 1009 case I915_FORMAT_MOD_Y_TILED_CCS: 1010 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS: 1011 assert(num_main_planes == 1 && num_planes == 2); 1012 import_aux_info(r[0], r[1]); 1013 map_aux_addresses(screen, r[0], res->external_format, 0); 1014 1015 /* Add on a clear color BO. 1016 * 1017 * Also add some padding to make sure the fast clear color state buffer 1018 * starts at a 4K alignment to avoid some unknown issues. See the 1019 * matching comment in iris_resource_create_with_modifiers(). 1020 */ 1021 if (iris_get_aux_clear_color_state_size(screen, res) > 0) { 1022 res->aux.clear_color_bo = 1023 iris_bo_alloc(screen->bufmgr, "clear color_buffer", 1024 iris_get_aux_clear_color_state_size(screen, res), 1025 4096, IRIS_MEMZONE_OTHER, BO_ALLOC_ZEROED); 1026 } 1027 break; 1028 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS: 1029 assert(num_main_planes == 1); 1030 assert(num_planes == 1); 1031 res->aux.clear_color_bo = 1032 iris_bo_alloc(screen->bufmgr, "clear color_buffer", 1033 iris_get_aux_clear_color_state_size(screen, res), 1034 4096, IRIS_MEMZONE_OTHER, BO_ALLOC_ZEROED); 1035 break; 1036 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: 1037 assert(num_main_planes == 1 && num_planes == 3); 1038 import_aux_info(r[0], r[1]); 1039 map_aux_addresses(screen, r[0], res->external_format, 0); 1040 1041 /* Import the clear color BO. */ 1042 iris_bo_reference(r[2]->aux.clear_color_bo); 1043 r[0]->aux.clear_color_bo = r[2]->aux.clear_color_bo; 1044 r[0]->aux.clear_color_offset = r[2]->aux.clear_color_offset; 1045 r[0]->aux.clear_color_unknown = true; 1046 break; 1047 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC: 1048 assert(num_main_planes == 1); 1049 assert(num_planes == 2); 1050 1051 /* Import the clear color BO. */ 1052 iris_bo_reference(r[1]->aux.clear_color_bo); 1053 r[0]->aux.clear_color_bo = r[1]->aux.clear_color_bo; 1054 r[0]->aux.clear_color_offset = r[1]->aux.clear_color_offset; 1055 r[0]->aux.clear_color_unknown = true; 1056 break; 1057 case I915_FORMAT_MOD_Y_TILED_GEN12_MC_CCS: 1058 if (num_main_planes == 1 && num_planes == 2) { 1059 import_aux_info(r[0], r[1]); 1060 map_aux_addresses(screen, r[0], res->external_format, 0); 1061 } else { 1062 assert(num_main_planes == 2 && num_planes == 4); 1063 import_aux_info(r[0], r[2]); 1064 import_aux_info(r[1], r[3]); 1065 map_aux_addresses(screen, r[0], res->external_format, 0); 1066 map_aux_addresses(screen, r[1], res->external_format, 1); 1067 } 1068 assert(!isl_aux_usage_has_fast_clears(res->mod_info->aux_usage)); 1069 break; 1070 case I915_FORMAT_MOD_4_TILED_DG2_MC_CCS: 1071 assert(!isl_aux_usage_has_fast_clears(res->mod_info->aux_usage)); 1072 break; 1073 default: 1074 assert(res->mod_info->aux_usage == ISL_AUX_USAGE_NONE); 1075 break; 1076 } 1077} 1078 1079static uint32_t 1080iris_buffer_alignment(uint64_t size) 1081{ 1082 /* Some buffer operations want some amount of alignment. The largest 1083 * buffer texture pixel size is 4 * 4 = 16B. OpenCL data is also supposed 1084 * to be aligned and largest OpenCL data type is a double16 which is 1085 * 8 * 16 = 128B. Align to the largest power of 2 which fits in the size, 1086 * up to 128B. 1087 */ 1088 uint32_t align = MAX2(4 * 4, 8 * 16); 1089 while (align > size) 1090 align >>= 1; 1091 1092 return align; 1093} 1094 1095static struct pipe_resource * 1096iris_resource_create_for_buffer(struct pipe_screen *pscreen, 1097 const struct pipe_resource *templ) 1098{ 1099 struct iris_screen *screen = (struct iris_screen *)pscreen; 1100 struct iris_resource *res = iris_alloc_resource(pscreen, templ); 1101 1102 assert(templ->target == PIPE_BUFFER); 1103 assert(templ->height0 <= 1); 1104 assert(templ->depth0 <= 1); 1105 assert(templ->format == PIPE_FORMAT_NONE || 1106 util_format_get_blocksize(templ->format) == 1); 1107 1108 res->internal_format = templ->format; 1109 res->surf.tiling = ISL_TILING_LINEAR; 1110 1111 enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER; 1112 const char *name = templ->target == PIPE_BUFFER ? "buffer" : "miptree"; 1113 if (templ->flags & IRIS_RESOURCE_FLAG_SHADER_MEMZONE) { 1114 memzone = IRIS_MEMZONE_SHADER; 1115 name = "shader kernels"; 1116 } else if (templ->flags & IRIS_RESOURCE_FLAG_SURFACE_MEMZONE) { 1117 memzone = IRIS_MEMZONE_SURFACE; 1118 name = "surface state"; 1119 } else if (templ->flags & IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE) { 1120 memzone = IRIS_MEMZONE_DYNAMIC; 1121 name = "dynamic state"; 1122 } else if (templ->flags & IRIS_RESOURCE_FLAG_BINDLESS_MEMZONE) { 1123 memzone = IRIS_MEMZONE_BINDLESS; 1124 name = "bindless surface state"; 1125 } 1126 1127 unsigned flags = iris_resource_alloc_flags(screen, templ, res->aux.usage); 1128 1129 res->bo = iris_bo_alloc(screen->bufmgr, name, templ->width0, 1130 iris_buffer_alignment(templ->width0), 1131 memzone, flags); 1132 1133 if (!res->bo) { 1134 iris_resource_destroy(pscreen, &res->base.b); 1135 return NULL; 1136 } 1137 1138 if (templ->bind & PIPE_BIND_SHARED) { 1139 iris_bo_mark_exported(res->bo); 1140 res->base.is_shared = true; 1141 } 1142 1143 return &res->base.b; 1144} 1145 1146static struct pipe_resource * 1147iris_resource_create_with_modifiers(struct pipe_screen *pscreen, 1148 const struct pipe_resource *templ, 1149 const uint64_t *modifiers, 1150 int modifiers_count) 1151{ 1152 struct iris_screen *screen = (struct iris_screen *)pscreen; 1153 struct intel_device_info *devinfo = &screen->devinfo; 1154 struct iris_resource *res = iris_alloc_resource(pscreen, templ); 1155 1156 if (!res) 1157 return NULL; 1158 1159 uint64_t modifier = 1160 select_best_modifier(devinfo, templ, modifiers, modifiers_count); 1161 1162 if (modifier == DRM_FORMAT_MOD_INVALID && modifiers_count > 0) { 1163 fprintf(stderr, "Unsupported modifier, resource creation failed.\n"); 1164 goto fail; 1165 } 1166 1167 UNUSED const bool isl_surf_created_successfully = 1168 iris_resource_configure_main(screen, res, templ, modifier, 0); 1169 assert(isl_surf_created_successfully); 1170 1171 if (!iris_resource_configure_aux(screen, res, false)) 1172 goto fail; 1173 1174 const char *name = "miptree"; 1175 enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER; 1176 1177 unsigned flags = iris_resource_alloc_flags(screen, templ, res->aux.usage); 1178 1179 /* These are for u_upload_mgr buffers only */ 1180 assert(!(templ->flags & (IRIS_RESOURCE_FLAG_SHADER_MEMZONE | 1181 IRIS_RESOURCE_FLAG_SURFACE_MEMZONE | 1182 IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE | 1183 IRIS_RESOURCE_FLAG_BINDLESS_MEMZONE))); 1184 1185 /* Modifiers require the aux data to be in the same buffer as the main 1186 * surface, but we combine them even when a modifier is not being used. 1187 */ 1188 uint64_t bo_size = res->surf.size_B; 1189 1190 /* Allocate space for the aux buffer. */ 1191 if (res->aux.surf.size_B > 0) { 1192 res->aux.offset = ALIGN(bo_size, res->aux.surf.alignment_B); 1193 bo_size = res->aux.offset + res->aux.surf.size_B; 1194 } 1195 1196 /* Allocate space for the extra aux buffer. */ 1197 if (res->aux.extra_aux.surf.size_B > 0) { 1198 res->aux.extra_aux.offset = 1199 ALIGN(bo_size, res->aux.extra_aux.surf.alignment_B); 1200 bo_size = res->aux.extra_aux.offset + res->aux.extra_aux.surf.size_B; 1201 } 1202 1203 /* Allocate space for the indirect clear color. 1204 * 1205 * Also add some padding to make sure the fast clear color state buffer 1206 * starts at a 4K alignment. We believe that 256B might be enough, but due 1207 * to lack of testing we will leave this as 4K for now. 1208 */ 1209 if (iris_get_aux_clear_color_state_size(screen, res) > 0) { 1210 res->aux.clear_color_offset = ALIGN(bo_size, 4096); 1211 bo_size = res->aux.clear_color_offset + 1212 iris_get_aux_clear_color_state_size(screen, res); 1213 } 1214 1215 uint32_t alignment = MAX2(4096, res->surf.alignment_B); 1216 res->bo = 1217 iris_bo_alloc(screen->bufmgr, name, bo_size, alignment, memzone, flags); 1218 1219 if (!res->bo) 1220 goto fail; 1221 1222 if (res->aux.usage != ISL_AUX_USAGE_NONE && 1223 !iris_resource_init_aux_buf(screen, res)) 1224 goto fail; 1225 1226 if (templ->bind & PIPE_BIND_SHARED) { 1227 iris_bo_mark_exported(res->bo); 1228 res->base.is_shared = true; 1229 } 1230 1231 return &res->base.b; 1232 1233fail: 1234 fprintf(stderr, "XXX: resource creation failed\n"); 1235 iris_resource_destroy(pscreen, &res->base.b); 1236 return NULL; 1237} 1238 1239static struct pipe_resource * 1240iris_resource_create(struct pipe_screen *pscreen, 1241 const struct pipe_resource *templ) 1242{ 1243 if (templ->target == PIPE_BUFFER) 1244 return iris_resource_create_for_buffer(pscreen, templ); 1245 else 1246 return iris_resource_create_with_modifiers(pscreen, templ, NULL, 0); 1247} 1248 1249static uint64_t 1250tiling_to_modifier(uint32_t tiling) 1251{ 1252 static const uint64_t map[] = { 1253 [I915_TILING_NONE] = DRM_FORMAT_MOD_LINEAR, 1254 [I915_TILING_X] = I915_FORMAT_MOD_X_TILED, 1255 [I915_TILING_Y] = I915_FORMAT_MOD_Y_TILED, 1256 }; 1257 1258 assert(tiling < ARRAY_SIZE(map)); 1259 1260 return map[tiling]; 1261} 1262 1263static struct pipe_resource * 1264iris_resource_from_user_memory(struct pipe_screen *pscreen, 1265 const struct pipe_resource *templ, 1266 void *user_memory) 1267{ 1268 struct iris_screen *screen = (struct iris_screen *)pscreen; 1269 struct iris_bufmgr *bufmgr = screen->bufmgr; 1270 struct iris_resource *res = iris_alloc_resource(pscreen, templ); 1271 if (!res) 1272 return NULL; 1273 1274 if (templ->target != PIPE_BUFFER && 1275 templ->target != PIPE_TEXTURE_1D && 1276 templ->target != PIPE_TEXTURE_2D) 1277 return NULL; 1278 1279 if (templ->array_size > 1) 1280 return NULL; 1281 1282 size_t res_size = templ->width0; 1283 if (templ->target != PIPE_BUFFER) { 1284 const uint32_t row_pitch_B = 1285 templ->width0 * util_format_get_blocksize(templ->format); 1286 res_size = templ->height0 * row_pitch_B; 1287 1288 if (!iris_resource_configure_main(screen, res, templ, 1289 DRM_FORMAT_MOD_LINEAR, 1290 row_pitch_B)) { 1291 iris_resource_destroy(pscreen, &res->base.b); 1292 return NULL; 1293 } 1294 assert(res->surf.size_B <= res_size); 1295 } 1296 1297 /* The userptr ioctl only works on whole pages. Because we know that 1298 * things will exist in memory at a page granularity, we can expand the 1299 * range given by the client into the whole number of pages and use an 1300 * offset on the resource to make it looks like it starts at the user's 1301 * pointer. 1302 */ 1303 size_t page_size = getpagesize(); 1304 assert(util_is_power_of_two_nonzero(page_size)); 1305 size_t offset = (uintptr_t)user_memory & (page_size - 1); 1306 void *mem_start = (char *)user_memory - offset; 1307 size_t mem_size = offset + res_size; 1308 mem_size = ALIGN_NPOT(mem_size, page_size); 1309 1310 res->internal_format = templ->format; 1311 res->base.is_user_ptr = true; 1312 res->bo = iris_bo_create_userptr(bufmgr, "user", mem_start, mem_size, 1313 IRIS_MEMZONE_OTHER); 1314 res->offset = offset; 1315 if (!res->bo) { 1316 iris_resource_destroy(pscreen, &res->base.b); 1317 return NULL; 1318 } 1319 1320 util_range_add(&res->base.b, &res->valid_buffer_range, 0, templ->width0); 1321 1322 return &res->base.b; 1323} 1324 1325static bool 1326mod_plane_is_clear_color(uint64_t modifier, uint32_t plane) 1327{ 1328 ASSERTED const struct isl_drm_modifier_info *mod_info = 1329 isl_drm_modifier_get_info(modifier); 1330 assert(mod_info); 1331 1332 switch (modifier) { 1333 case I915_FORMAT_MOD_Y_TILED_GEN12_RC_CCS_CC: 1334 assert(mod_info->supports_clear_color); 1335 return plane == 2; 1336 case I915_FORMAT_MOD_4_TILED_DG2_RC_CCS_CC: 1337 assert(mod_info->supports_clear_color); 1338 return plane == 1; 1339 default: 1340 assert(!mod_info->supports_clear_color); 1341 return false; 1342 } 1343} 1344 1345static unsigned 1346get_num_planes(const struct pipe_resource *resource) 1347{ 1348 unsigned count = 0; 1349 for (const struct pipe_resource *cur = resource; cur; cur = cur->next) 1350 count++; 1351 1352 return count; 1353} 1354 1355static struct pipe_resource * 1356iris_resource_from_handle(struct pipe_screen *pscreen, 1357 const struct pipe_resource *templ, 1358 struct winsys_handle *whandle, 1359 unsigned usage) 1360{ 1361 assert(templ->target != PIPE_BUFFER); 1362 1363 struct iris_screen *screen = (struct iris_screen *)pscreen; 1364 struct iris_bufmgr *bufmgr = screen->bufmgr; 1365 struct iris_resource *res = iris_alloc_resource(pscreen, templ); 1366 if (!res) 1367 return NULL; 1368 1369 switch (whandle->type) { 1370 case WINSYS_HANDLE_TYPE_FD: 1371 res->bo = iris_bo_import_dmabuf(bufmgr, whandle->handle); 1372 break; 1373 case WINSYS_HANDLE_TYPE_SHARED: 1374 res->bo = iris_bo_gem_create_from_name(bufmgr, "winsys image", 1375 whandle->handle); 1376 break; 1377 default: 1378 unreachable("invalid winsys handle type"); 1379 } 1380 if (!res->bo) 1381 goto fail; 1382 1383 res->offset = whandle->offset; 1384 res->external_format = whandle->format; 1385 1386 /* Create a surface for each plane specified by the external format. */ 1387 if (whandle->plane < util_format_get_num_planes(whandle->format)) { 1388 uint64_t modifier = whandle->modifier; 1389 1390 if (whandle->modifier == DRM_FORMAT_MOD_INVALID) { 1391 /* We don't have a modifier; match whatever GEM_GET_TILING says */ 1392 uint32_t tiling; 1393 iris_gem_get_tiling(res->bo, &tiling); 1394 modifier = tiling_to_modifier(tiling); 1395 } 1396 1397 UNUSED const bool isl_surf_created_successfully = 1398 iris_resource_configure_main(screen, res, templ, modifier, 1399 whandle->stride); 1400 assert(isl_surf_created_successfully); 1401 1402 UNUSED const bool ok = iris_resource_configure_aux(screen, res, true); 1403 assert(ok); 1404 /* The gallium dri layer will create a separate plane resource for the 1405 * aux image. iris_resource_finish_aux_import will merge the separate aux 1406 * parameters back into a single iris_resource. 1407 */ 1408 } else if (mod_plane_is_clear_color(whandle->modifier, whandle->plane)) { 1409 res->aux.clear_color_offset = whandle->offset; 1410 res->aux.clear_color_bo = res->bo; 1411 res->bo = NULL; 1412 } else { 1413 /* Save modifier import information to reconstruct later. After import, 1414 * this will be available under a second image accessible from the main 1415 * image with res->base.next. See iris_resource_finish_aux_import. 1416 */ 1417 res->aux.surf.row_pitch_B = whandle->stride; 1418 res->aux.offset = whandle->offset; 1419 res->aux.bo = res->bo; 1420 res->bo = NULL; 1421 } 1422 1423 if (get_num_planes(&res->base.b) == 1424 iris_get_dmabuf_modifier_planes(pscreen, whandle->modifier, 1425 whandle->format)) { 1426 iris_resource_finish_aux_import(pscreen, res); 1427 } 1428 1429 return &res->base.b; 1430 1431fail: 1432 iris_resource_destroy(pscreen, &res->base.b); 1433 return NULL; 1434} 1435 1436static struct pipe_resource * 1437iris_resource_from_memobj(struct pipe_screen *pscreen, 1438 const struct pipe_resource *templ, 1439 struct pipe_memory_object *pmemobj, 1440 uint64_t offset) 1441{ 1442 struct iris_screen *screen = (struct iris_screen *)pscreen; 1443 struct iris_memory_object *memobj = (struct iris_memory_object *)pmemobj; 1444 struct iris_resource *res = iris_alloc_resource(pscreen, templ); 1445 1446 if (!res) 1447 return NULL; 1448 1449 if (templ->flags & PIPE_RESOURCE_FLAG_TEXTURING_MORE_LIKELY) { 1450 UNUSED const bool isl_surf_created_successfully = 1451 iris_resource_configure_main(screen, res, templ, DRM_FORMAT_MOD_INVALID, 0); 1452 assert(isl_surf_created_successfully); 1453 } 1454 1455 res->bo = memobj->bo; 1456 res->offset = offset; 1457 res->external_format = memobj->format; 1458 res->internal_format = templ->format; 1459 1460 iris_bo_reference(memobj->bo); 1461 1462 return &res->base.b; 1463} 1464 1465/* Handle combined depth/stencil with memory objects. 1466 * 1467 * This function is modeled after u_transfer_helper_resource_create. 1468 */ 1469static struct pipe_resource * 1470iris_resource_from_memobj_wrapper(struct pipe_screen *pscreen, 1471 const struct pipe_resource *templ, 1472 struct pipe_memory_object *pmemobj, 1473 uint64_t offset) 1474{ 1475 enum pipe_format format = templ->format; 1476 1477 /* Normal case, no special handling: */ 1478 if (!(util_format_is_depth_and_stencil(format))) 1479 return iris_resource_from_memobj(pscreen, templ, pmemobj, offset); 1480 1481 struct pipe_resource t = *templ; 1482 t.format = util_format_get_depth_only(format); 1483 1484 struct pipe_resource *prsc = 1485 iris_resource_from_memobj(pscreen, &t, pmemobj, offset); 1486 if (!prsc) 1487 return NULL; 1488 1489 struct iris_resource *res = (struct iris_resource *) prsc; 1490 1491 /* Stencil offset in the buffer without aux. */ 1492 uint64_t s_offset = offset + 1493 ALIGN(res->surf.size_B, res->surf.alignment_B); 1494 1495 prsc->format = format; /* frob the format back to the "external" format */ 1496 1497 t.format = PIPE_FORMAT_S8_UINT; 1498 struct pipe_resource *stencil = 1499 iris_resource_from_memobj(pscreen, &t, pmemobj, s_offset); 1500 if (!stencil) { 1501 iris_resource_destroy(pscreen, prsc); 1502 return NULL; 1503 } 1504 1505 iris_resource_set_separate_stencil(prsc, stencil); 1506 return prsc; 1507} 1508 1509static void 1510iris_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource) 1511{ 1512 struct iris_context *ice = (struct iris_context *)ctx; 1513 struct iris_resource *res = (void *) resource; 1514 const struct isl_drm_modifier_info *mod = res->mod_info; 1515 1516 iris_resource_prepare_access(ice, res, 1517 0, INTEL_REMAINING_LEVELS, 1518 0, INTEL_REMAINING_LAYERS, 1519 mod ? mod->aux_usage : ISL_AUX_USAGE_NONE, 1520 mod ? mod->supports_clear_color : false); 1521 1522 if (!res->mod_info && res->aux.usage != ISL_AUX_USAGE_NONE) { 1523 /* flush_resource may be used to prepare an image for sharing external 1524 * to the driver (e.g. via eglCreateImage). To account for this, make 1525 * sure to get rid of any compression that a consumer wouldn't know how 1526 * to handle. 1527 */ 1528 iris_foreach_batch(ice, batch) { 1529 if (iris_batch_references(batch, res->bo)) 1530 iris_batch_flush(batch); 1531 } 1532 1533 iris_resource_disable_aux(res); 1534 } 1535} 1536 1537/** 1538 * Reallocate a (non-external) resource into new storage, copying the data 1539 * and modifying the original resource to point at the new storage. 1540 * 1541 * This is useful for e.g. moving a suballocated internal resource to a 1542 * dedicated allocation that can be exported by itself. 1543 */ 1544static void 1545iris_reallocate_resource_inplace(struct iris_context *ice, 1546 struct iris_resource *old_res, 1547 unsigned new_bind_flag) 1548{ 1549 struct pipe_screen *pscreen = ice->ctx.screen; 1550 1551 if (iris_bo_is_external(old_res->bo)) 1552 return; 1553 1554 assert(old_res->mod_info == NULL); 1555 assert(old_res->bo == old_res->aux.bo || old_res->aux.bo == NULL); 1556 assert(old_res->bo == old_res->aux.clear_color_bo || 1557 old_res->aux.clear_color_bo == NULL); 1558 assert(old_res->external_format == PIPE_FORMAT_NONE); 1559 1560 struct pipe_resource templ = old_res->base.b; 1561 templ.bind |= new_bind_flag; 1562 1563 struct iris_resource *new_res = 1564 (void *) pscreen->resource_create(pscreen, &templ); 1565 1566 assert(iris_bo_is_real(new_res->bo)); 1567 1568 struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER]; 1569 1570 if (old_res->base.b.target == PIPE_BUFFER) { 1571 struct pipe_box box = (struct pipe_box) { 1572 .width = old_res->base.b.width0, 1573 .height = 1, 1574 }; 1575 1576 iris_copy_region(&ice->blorp, batch, &new_res->base.b, 0, 0, 0, 0, 1577 &old_res->base.b, 0, &box); 1578 } else { 1579 for (unsigned l = 0; l <= templ.last_level; l++) { 1580 struct pipe_box box = (struct pipe_box) { 1581 .width = u_minify(templ.width0, l), 1582 .height = u_minify(templ.height0, l), 1583 .depth = util_num_layers(&templ, l), 1584 }; 1585 1586 iris_copy_region(&ice->blorp, batch, &new_res->base.b, l, 0, 0, 0, 1587 &old_res->base.b, l, &box); 1588 } 1589 } 1590 1591 iris_flush_resource(&ice->ctx, &new_res->base.b); 1592 1593 struct iris_bo *old_bo = old_res->bo; 1594 struct iris_bo *old_aux_bo = old_res->aux.bo; 1595 struct iris_bo *old_clear_color_bo = old_res->aux.clear_color_bo; 1596 1597 /* Replace the structure fields with the new ones */ 1598 old_res->base.b.bind = templ.bind; 1599 old_res->bo = new_res->bo; 1600 old_res->aux.surf = new_res->aux.surf; 1601 old_res->aux.bo = new_res->aux.bo; 1602 old_res->aux.offset = new_res->aux.offset; 1603 old_res->aux.extra_aux.surf = new_res->aux.extra_aux.surf; 1604 old_res->aux.extra_aux.offset = new_res->aux.extra_aux.offset; 1605 old_res->aux.clear_color_bo = new_res->aux.clear_color_bo; 1606 old_res->aux.clear_color_offset = new_res->aux.clear_color_offset; 1607 old_res->aux.usage = new_res->aux.usage; 1608 1609 if (new_res->aux.state) { 1610 assert(old_res->aux.state); 1611 for (unsigned l = 0; l <= templ.last_level; l++) { 1612 unsigned layers = util_num_layers(&templ, l); 1613 for (unsigned z = 0; z < layers; z++) { 1614 enum isl_aux_state aux = 1615 iris_resource_get_aux_state(new_res, l, z); 1616 iris_resource_set_aux_state(ice, old_res, l, z, 1, aux); 1617 } 1618 } 1619 } 1620 1621 /* old_res now points at the new BOs, make new_res point at the old ones 1622 * so they'll be freed when we unreference the resource below. 1623 */ 1624 new_res->bo = old_bo; 1625 new_res->aux.bo = old_aux_bo; 1626 new_res->aux.clear_color_bo = old_clear_color_bo; 1627 1628 pipe_resource_reference((struct pipe_resource **)&new_res, NULL); 1629} 1630 1631static void 1632iris_resource_disable_suballoc_on_first_query(struct pipe_screen *pscreen, 1633 struct pipe_context *ctx, 1634 struct iris_resource *res) 1635{ 1636 if (iris_bo_is_real(res->bo)) 1637 return; 1638 1639 assert(!(res->base.b.bind & PIPE_BIND_SHARED)); 1640 1641 bool destroy_context; 1642 if (ctx) { 1643 ctx = threaded_context_unwrap_sync(ctx); 1644 destroy_context = false; 1645 } else { 1646 /* We need to execute a blit on some GPU context, but the DRI layer 1647 * often doesn't give us one. So we have to invent a temporary one. 1648 * 1649 * We can't store a permanent context in the screen, as it would cause 1650 * circular refcounting where screens reference contexts that reference 1651 * resources, while resources reference screens...causing nothing to be 1652 * freed. So we just create and destroy a temporary one here. 1653 */ 1654 ctx = iris_create_context(pscreen, NULL, 0); 1655 destroy_context = true; 1656 } 1657 1658 struct iris_context *ice = (struct iris_context *)ctx; 1659 1660 iris_reallocate_resource_inplace(ice, res, PIPE_BIND_SHARED); 1661 assert(res->base.b.bind & PIPE_BIND_SHARED); 1662 1663 if (destroy_context) 1664 iris_destroy_context(ctx); 1665} 1666 1667 1668static void 1669iris_resource_disable_aux_on_first_query(struct pipe_resource *resource, 1670 unsigned usage) 1671{ 1672 struct iris_resource *res = (struct iris_resource *)resource; 1673 bool mod_with_aux = 1674 res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE; 1675 1676 /* Disable aux usage if explicit flush not set and this is the first time 1677 * we are dealing with this resource and the resource was not created with 1678 * a modifier with aux. 1679 */ 1680 if (!mod_with_aux && 1681 (!(usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH) && res->aux.usage != 0) && 1682 p_atomic_read(&resource->reference.count) == 1) { 1683 iris_resource_disable_aux(res); 1684 } 1685} 1686 1687static bool 1688iris_resource_get_param(struct pipe_screen *pscreen, 1689 struct pipe_context *ctx, 1690 struct pipe_resource *resource, 1691 unsigned plane, 1692 unsigned layer, 1693 unsigned level, 1694 enum pipe_resource_param param, 1695 unsigned handle_usage, 1696 uint64_t *value) 1697{ 1698 struct iris_screen *screen = (struct iris_screen *)pscreen; 1699 struct iris_resource *res = (struct iris_resource *)resource; 1700 bool mod_with_aux = 1701 res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE; 1702 bool wants_aux = mod_with_aux && plane > 0; 1703 bool wants_cc = mod_with_aux && 1704 mod_plane_is_clear_color(res->mod_info->modifier, plane); 1705 bool result; 1706 unsigned handle; 1707 1708 iris_resource_disable_aux_on_first_query(resource, handle_usage); 1709 iris_resource_disable_suballoc_on_first_query(pscreen, ctx, res); 1710 1711 struct iris_bo *bo = wants_cc ? res->aux.clear_color_bo : 1712 wants_aux ? res->aux.bo : res->bo; 1713 1714 assert(iris_bo_is_real(bo)); 1715 1716 switch (param) { 1717 case PIPE_RESOURCE_PARAM_NPLANES: 1718 if (mod_with_aux) { 1719 *value = iris_get_dmabuf_modifier_planes(pscreen, 1720 res->mod_info->modifier, 1721 res->external_format); 1722 } else { 1723 *value = get_num_planes(&res->base.b); 1724 } 1725 return true; 1726 case PIPE_RESOURCE_PARAM_STRIDE: 1727 *value = wants_cc ? 64 : 1728 wants_aux ? res->aux.surf.row_pitch_B : res->surf.row_pitch_B; 1729 1730 /* Mesa's implementation of eglCreateImage rejects strides of zero (see 1731 * dri2_check_dma_buf_attribs). Ensure we return a non-zero stride as 1732 * this value may be queried from GBM and passed into EGL. 1733 * 1734 * Also, although modifiers which use a clear color plane specify that 1735 * the plane's pitch should be ignored, some kernels have been found to 1736 * require 64-byte alignment. 1737 */ 1738 assert(*value != 0 && (!wants_cc || *value % 64 == 0)); 1739 1740 return true; 1741 case PIPE_RESOURCE_PARAM_OFFSET: 1742 *value = wants_cc ? res->aux.clear_color_offset : 1743 wants_aux ? res->aux.offset : 0; 1744 return true; 1745 case PIPE_RESOURCE_PARAM_MODIFIER: 1746 *value = res->mod_info ? res->mod_info->modifier : 1747 tiling_to_modifier(isl_tiling_to_i915_tiling(res->surf.tiling)); 1748 return true; 1749 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_SHARED: 1750 if (!wants_aux) 1751 iris_gem_set_tiling(bo, &res->surf); 1752 1753 result = iris_bo_flink(bo, &handle) == 0; 1754 if (result) 1755 *value = handle; 1756 return result; 1757 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_KMS: { 1758 if (!wants_aux) 1759 iris_gem_set_tiling(bo, &res->surf); 1760 1761 /* Because we share the same drm file across multiple iris_screen, when 1762 * we export a GEM handle we must make sure it is valid in the DRM file 1763 * descriptor the caller is using (this is the FD given at screen 1764 * creation). 1765 */ 1766 uint32_t handle; 1767 if (iris_bo_export_gem_handle_for_device(bo, screen->winsys_fd, &handle)) 1768 return false; 1769 *value = handle; 1770 return true; 1771 } 1772 1773 case PIPE_RESOURCE_PARAM_HANDLE_TYPE_FD: 1774 if (!wants_aux) 1775 iris_gem_set_tiling(bo, &res->surf); 1776 1777 result = iris_bo_export_dmabuf(bo, (int *) &handle) == 0; 1778 if (result) 1779 *value = handle; 1780 return result; 1781 default: 1782 return false; 1783 } 1784} 1785 1786static bool 1787iris_resource_get_handle(struct pipe_screen *pscreen, 1788 struct pipe_context *ctx, 1789 struct pipe_resource *resource, 1790 struct winsys_handle *whandle, 1791 unsigned usage) 1792{ 1793 struct iris_screen *screen = (struct iris_screen *) pscreen; 1794 struct iris_resource *res = (struct iris_resource *)resource; 1795 bool mod_with_aux = 1796 res->mod_info && res->mod_info->aux_usage != ISL_AUX_USAGE_NONE; 1797 1798 iris_resource_disable_aux_on_first_query(resource, usage); 1799 iris_resource_disable_suballoc_on_first_query(pscreen, ctx, res); 1800 1801 assert(iris_bo_is_real(res->bo)); 1802 1803 struct iris_bo *bo; 1804 if (res->mod_info && 1805 mod_plane_is_clear_color(res->mod_info->modifier, whandle->plane)) { 1806 bo = res->aux.clear_color_bo; 1807 whandle->offset = res->aux.clear_color_offset; 1808 } else if (mod_with_aux && whandle->plane > 0) { 1809 bo = res->aux.bo; 1810 whandle->stride = res->aux.surf.row_pitch_B; 1811 whandle->offset = res->aux.offset; 1812 } else { 1813 /* If this is a buffer, stride should be 0 - no need to special case */ 1814 whandle->stride = res->surf.row_pitch_B; 1815 bo = res->bo; 1816 } 1817 1818 whandle->format = res->external_format; 1819 whandle->modifier = 1820 res->mod_info ? res->mod_info->modifier 1821 : tiling_to_modifier(isl_tiling_to_i915_tiling(res->surf.tiling)); 1822 1823#ifndef NDEBUG 1824 enum isl_aux_usage allowed_usage = 1825 usage & PIPE_HANDLE_USAGE_EXPLICIT_FLUSH ? res->aux.usage : 1826 res->mod_info ? res->mod_info->aux_usage : ISL_AUX_USAGE_NONE; 1827 1828 if (res->aux.usage != allowed_usage) { 1829 enum isl_aux_state aux_state = iris_resource_get_aux_state(res, 0, 0); 1830 assert(aux_state == ISL_AUX_STATE_RESOLVED || 1831 aux_state == ISL_AUX_STATE_PASS_THROUGH); 1832 } 1833#endif 1834 1835 switch (whandle->type) { 1836 case WINSYS_HANDLE_TYPE_SHARED: 1837 iris_gem_set_tiling(bo, &res->surf); 1838 return iris_bo_flink(bo, &whandle->handle) == 0; 1839 case WINSYS_HANDLE_TYPE_KMS: { 1840 iris_gem_set_tiling(bo, &res->surf); 1841 1842 /* Because we share the same drm file across multiple iris_screen, when 1843 * we export a GEM handle we must make sure it is valid in the DRM file 1844 * descriptor the caller is using (this is the FD given at screen 1845 * creation). 1846 */ 1847 uint32_t handle; 1848 if (iris_bo_export_gem_handle_for_device(bo, screen->winsys_fd, &handle)) 1849 return false; 1850 whandle->handle = handle; 1851 return true; 1852 } 1853 case WINSYS_HANDLE_TYPE_FD: 1854 iris_gem_set_tiling(bo, &res->surf); 1855 return iris_bo_export_dmabuf(bo, (int *) &whandle->handle) == 0; 1856 } 1857 1858 return false; 1859} 1860 1861static bool 1862resource_is_busy(struct iris_context *ice, 1863 struct iris_resource *res) 1864{ 1865 bool busy = iris_bo_busy(res->bo); 1866 1867 iris_foreach_batch(ice, batch) 1868 busy |= iris_batch_references(batch, res->bo); 1869 1870 return busy; 1871} 1872 1873void 1874iris_replace_buffer_storage(struct pipe_context *ctx, 1875 struct pipe_resource *p_dst, 1876 struct pipe_resource *p_src, 1877 unsigned num_rebinds, 1878 uint32_t rebind_mask, 1879 uint32_t delete_buffer_id) 1880{ 1881 struct iris_screen *screen = (void *) ctx->screen; 1882 struct iris_context *ice = (void *) ctx; 1883 struct iris_resource *dst = (void *) p_dst; 1884 struct iris_resource *src = (void *) p_src; 1885 1886 assert(memcmp(&dst->surf, &src->surf, sizeof(dst->surf)) == 0); 1887 1888 struct iris_bo *old_bo = dst->bo; 1889 1890 /* Swap out the backing storage */ 1891 iris_bo_reference(src->bo); 1892 dst->bo = src->bo; 1893 1894 /* Rebind the buffer, replacing any state referring to the old BO's 1895 * address, and marking state dirty so it's reemitted. 1896 */ 1897 screen->vtbl.rebind_buffer(ice, dst); 1898 1899 iris_bo_unreference(old_bo); 1900} 1901 1902static void 1903iris_invalidate_resource(struct pipe_context *ctx, 1904 struct pipe_resource *resource) 1905{ 1906 struct iris_screen *screen = (void *) ctx->screen; 1907 struct iris_context *ice = (void *) ctx; 1908 struct iris_resource *res = (void *) resource; 1909 1910 if (resource->target != PIPE_BUFFER) 1911 return; 1912 1913 /* If it's already invalidated, don't bother doing anything. */ 1914 if (res->valid_buffer_range.start > res->valid_buffer_range.end) 1915 return; 1916 1917 if (!resource_is_busy(ice, res)) { 1918 /* The resource is idle, so just mark that it contains no data and 1919 * keep using the same underlying buffer object. 1920 */ 1921 util_range_set_empty(&res->valid_buffer_range); 1922 return; 1923 } 1924 1925 /* Otherwise, try and replace the backing storage with a new BO. */ 1926 1927 /* We can't reallocate memory we didn't allocate in the first place. */ 1928 if (res->bo->gem_handle && res->bo->real.userptr) 1929 return; 1930 1931 struct iris_bo *old_bo = res->bo; 1932 struct iris_bo *new_bo = 1933 iris_bo_alloc(screen->bufmgr, res->bo->name, resource->width0, 1934 iris_buffer_alignment(resource->width0), 1935 iris_memzone_for_address(old_bo->address), 0); 1936 if (!new_bo) 1937 return; 1938 1939 /* Swap out the backing storage */ 1940 res->bo = new_bo; 1941 1942 /* Rebind the buffer, replacing any state referring to the old BO's 1943 * address, and marking state dirty so it's reemitted. 1944 */ 1945 screen->vtbl.rebind_buffer(ice, res); 1946 1947 util_range_set_empty(&res->valid_buffer_range); 1948 1949 iris_bo_unreference(old_bo); 1950} 1951 1952static void 1953iris_flush_staging_region(struct pipe_transfer *xfer, 1954 const struct pipe_box *flush_box) 1955{ 1956 if (!(xfer->usage & PIPE_MAP_WRITE)) 1957 return; 1958 1959 struct iris_transfer *map = (void *) xfer; 1960 1961 struct pipe_box src_box = *flush_box; 1962 1963 /* Account for extra alignment padding in staging buffer */ 1964 if (xfer->resource->target == PIPE_BUFFER) 1965 src_box.x += xfer->box.x % IRIS_MAP_BUFFER_ALIGNMENT; 1966 1967 struct pipe_box dst_box = (struct pipe_box) { 1968 .x = xfer->box.x + flush_box->x, 1969 .y = xfer->box.y + flush_box->y, 1970 .z = xfer->box.z + flush_box->z, 1971 .width = flush_box->width, 1972 .height = flush_box->height, 1973 .depth = flush_box->depth, 1974 }; 1975 1976 iris_copy_region(map->blorp, map->batch, xfer->resource, xfer->level, 1977 dst_box.x, dst_box.y, dst_box.z, map->staging, 0, 1978 &src_box); 1979} 1980 1981static void 1982iris_unmap_copy_region(struct iris_transfer *map) 1983{ 1984 iris_resource_destroy(map->staging->screen, map->staging); 1985 1986 map->ptr = NULL; 1987} 1988 1989static void 1990iris_map_copy_region(struct iris_transfer *map) 1991{ 1992 struct pipe_screen *pscreen = &map->batch->screen->base; 1993 struct pipe_transfer *xfer = &map->base.b; 1994 struct pipe_box *box = &xfer->box; 1995 struct iris_resource *res = (void *) xfer->resource; 1996 1997 unsigned extra = xfer->resource->target == PIPE_BUFFER ? 1998 box->x % IRIS_MAP_BUFFER_ALIGNMENT : 0; 1999 2000 struct pipe_resource templ = (struct pipe_resource) { 2001 .usage = PIPE_USAGE_STAGING, 2002 .width0 = box->width + extra, 2003 .height0 = box->height, 2004 .depth0 = 1, 2005 .nr_samples = xfer->resource->nr_samples, 2006 .nr_storage_samples = xfer->resource->nr_storage_samples, 2007 .array_size = box->depth, 2008 .format = res->internal_format, 2009 }; 2010 2011 if (xfer->resource->target == PIPE_BUFFER) 2012 templ.target = PIPE_BUFFER; 2013 else if (templ.array_size > 1) 2014 templ.target = PIPE_TEXTURE_2D_ARRAY; 2015 else 2016 templ.target = PIPE_TEXTURE_2D; 2017 2018 map->staging = iris_resource_create(pscreen, &templ); 2019 assert(map->staging); 2020 2021 if (templ.target != PIPE_BUFFER) { 2022 struct isl_surf *surf = &((struct iris_resource *) map->staging)->surf; 2023 xfer->stride = isl_surf_get_row_pitch_B(surf); 2024 xfer->layer_stride = isl_surf_get_array_pitch(surf); 2025 } 2026 2027 if (!(xfer->usage & PIPE_MAP_DISCARD_RANGE)) { 2028 iris_copy_region(map->blorp, map->batch, map->staging, 0, extra, 0, 0, 2029 xfer->resource, xfer->level, box); 2030 /* Ensure writes to the staging BO land before we map it below. */ 2031 iris_emit_pipe_control_flush(map->batch, 2032 "transfer read: flush before mapping", 2033 PIPE_CONTROL_RENDER_TARGET_FLUSH | 2034 PIPE_CONTROL_TILE_CACHE_FLUSH | 2035 PIPE_CONTROL_CS_STALL); 2036 } 2037 2038 struct iris_bo *staging_bo = iris_resource_bo(map->staging); 2039 2040 if (iris_batch_references(map->batch, staging_bo)) 2041 iris_batch_flush(map->batch); 2042 2043 assert(((struct iris_resource *)map->staging)->offset == 0); 2044 map->ptr = 2045 iris_bo_map(map->dbg, staging_bo, xfer->usage & MAP_FLAGS) + extra; 2046 2047 map->unmap = iris_unmap_copy_region; 2048} 2049 2050static void 2051get_image_offset_el(const struct isl_surf *surf, unsigned level, unsigned z, 2052 unsigned *out_x0_el, unsigned *out_y0_el) 2053{ 2054 ASSERTED uint32_t z0_el, a0_el; 2055 if (surf->dim == ISL_SURF_DIM_3D) { 2056 isl_surf_get_image_offset_el(surf, level, 0, z, 2057 out_x0_el, out_y0_el, &z0_el, &a0_el); 2058 } else { 2059 isl_surf_get_image_offset_el(surf, level, z, 0, 2060 out_x0_el, out_y0_el, &z0_el, &a0_el); 2061 } 2062 assert(z0_el == 0 && a0_el == 0); 2063} 2064 2065/** 2066 * Get pointer offset into stencil buffer. 2067 * 2068 * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we 2069 * must decode the tile's layout in software. 2070 * 2071 * See 2072 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile 2073 * Format. 2074 * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm 2075 * 2076 * Even though the returned offset is always positive, the return type is 2077 * signed due to 2078 * commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137 2079 * mesa: Fix return type of _mesa_get_format_bytes() (#37351) 2080 */ 2081static intptr_t 2082s8_offset(uint32_t stride, uint32_t x, uint32_t y) 2083{ 2084 uint32_t tile_size = 4096; 2085 uint32_t tile_width = 64; 2086 uint32_t tile_height = 64; 2087 uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */ 2088 2089 uint32_t tile_x = x / tile_width; 2090 uint32_t tile_y = y / tile_height; 2091 2092 /* The byte's address relative to the tile's base addres. */ 2093 uint32_t byte_x = x % tile_width; 2094 uint32_t byte_y = y % tile_height; 2095 2096 uintptr_t u = tile_y * row_size 2097 + tile_x * tile_size 2098 + 512 * (byte_x / 8) 2099 + 64 * (byte_y / 8) 2100 + 32 * ((byte_y / 4) % 2) 2101 + 16 * ((byte_x / 4) % 2) 2102 + 8 * ((byte_y / 2) % 2) 2103 + 4 * ((byte_x / 2) % 2) 2104 + 2 * (byte_y % 2) 2105 + 1 * (byte_x % 2); 2106 2107 return u; 2108} 2109 2110static void 2111iris_unmap_s8(struct iris_transfer *map) 2112{ 2113 struct pipe_transfer *xfer = &map->base.b; 2114 const struct pipe_box *box = &xfer->box; 2115 struct iris_resource *res = (struct iris_resource *) xfer->resource; 2116 struct isl_surf *surf = &res->surf; 2117 2118 if (xfer->usage & PIPE_MAP_WRITE) { 2119 uint8_t *untiled_s8_map = map->ptr; 2120 uint8_t *tiled_s8_map = res->offset + 2121 iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS); 2122 2123 for (int s = 0; s < box->depth; s++) { 2124 unsigned x0_el, y0_el; 2125 get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el); 2126 2127 for (uint32_t y = 0; y < box->height; y++) { 2128 for (uint32_t x = 0; x < box->width; x++) { 2129 ptrdiff_t offset = s8_offset(surf->row_pitch_B, 2130 x0_el + box->x + x, 2131 y0_el + box->y + y); 2132 tiled_s8_map[offset] = 2133 untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x]; 2134 } 2135 } 2136 } 2137 } 2138 2139 free(map->buffer); 2140} 2141 2142static void 2143iris_map_s8(struct iris_transfer *map) 2144{ 2145 struct pipe_transfer *xfer = &map->base.b; 2146 const struct pipe_box *box = &xfer->box; 2147 struct iris_resource *res = (struct iris_resource *) xfer->resource; 2148 struct isl_surf *surf = &res->surf; 2149 2150 xfer->stride = surf->row_pitch_B; 2151 xfer->layer_stride = xfer->stride * box->height; 2152 2153 /* The tiling and detiling functions require that the linear buffer has 2154 * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we 2155 * over-allocate the linear buffer to get the proper alignment. 2156 */ 2157 map->buffer = map->ptr = malloc(xfer->layer_stride * box->depth); 2158 assert(map->buffer); 2159 2160 /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no 2161 * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless 2162 * invalidate is set, since we'll be writing the whole rectangle from our 2163 * temporary buffer back out. 2164 */ 2165 if (!(xfer->usage & PIPE_MAP_DISCARD_RANGE)) { 2166 uint8_t *untiled_s8_map = map->ptr; 2167 uint8_t *tiled_s8_map = res->offset + 2168 iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS); 2169 2170 for (int s = 0; s < box->depth; s++) { 2171 unsigned x0_el, y0_el; 2172 get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el); 2173 2174 for (uint32_t y = 0; y < box->height; y++) { 2175 for (uint32_t x = 0; x < box->width; x++) { 2176 ptrdiff_t offset = s8_offset(surf->row_pitch_B, 2177 x0_el + box->x + x, 2178 y0_el + box->y + y); 2179 untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x] = 2180 tiled_s8_map[offset]; 2181 } 2182 } 2183 } 2184 } 2185 2186 map->unmap = iris_unmap_s8; 2187} 2188 2189/* Compute extent parameters for use with tiled_memcpy functions. 2190 * xs are in units of bytes and ys are in units of strides. 2191 */ 2192static inline void 2193tile_extents(const struct isl_surf *surf, 2194 const struct pipe_box *box, 2195 unsigned level, int z, 2196 unsigned *x1_B, unsigned *x2_B, 2197 unsigned *y1_el, unsigned *y2_el) 2198{ 2199 const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format); 2200 const unsigned cpp = fmtl->bpb / 8; 2201 2202 assert(box->x % fmtl->bw == 0); 2203 assert(box->y % fmtl->bh == 0); 2204 2205 unsigned x0_el, y0_el; 2206 get_image_offset_el(surf, level, box->z + z, &x0_el, &y0_el); 2207 2208 *x1_B = (box->x / fmtl->bw + x0_el) * cpp; 2209 *y1_el = box->y / fmtl->bh + y0_el; 2210 *x2_B = (DIV_ROUND_UP(box->x + box->width, fmtl->bw) + x0_el) * cpp; 2211 *y2_el = DIV_ROUND_UP(box->y + box->height, fmtl->bh) + y0_el; 2212} 2213 2214static void 2215iris_unmap_tiled_memcpy(struct iris_transfer *map) 2216{ 2217 struct pipe_transfer *xfer = &map->base.b; 2218 const struct pipe_box *box = &xfer->box; 2219 struct iris_resource *res = (struct iris_resource *) xfer->resource; 2220 struct isl_surf *surf = &res->surf; 2221 2222 const bool has_swizzling = false; 2223 2224 if (xfer->usage & PIPE_MAP_WRITE) { 2225 char *dst = res->offset + 2226 iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS); 2227 2228 for (int s = 0; s < box->depth; s++) { 2229 unsigned x1, x2, y1, y2; 2230 tile_extents(surf, box, xfer->level, s, &x1, &x2, &y1, &y2); 2231 2232 void *ptr = map->ptr + s * xfer->layer_stride; 2233 2234 isl_memcpy_linear_to_tiled(x1, x2, y1, y2, dst, ptr, 2235 surf->row_pitch_B, xfer->stride, 2236 has_swizzling, surf->tiling, ISL_MEMCPY); 2237 } 2238 } 2239 os_free_aligned(map->buffer); 2240 map->buffer = map->ptr = NULL; 2241} 2242 2243static void 2244iris_map_tiled_memcpy(struct iris_transfer *map) 2245{ 2246 struct pipe_transfer *xfer = &map->base.b; 2247 const struct pipe_box *box = &xfer->box; 2248 struct iris_resource *res = (struct iris_resource *) xfer->resource; 2249 struct isl_surf *surf = &res->surf; 2250 2251 xfer->stride = ALIGN(surf->row_pitch_B, 16); 2252 xfer->layer_stride = xfer->stride * box->height; 2253 2254 unsigned x1, x2, y1, y2; 2255 tile_extents(surf, box, xfer->level, 0, &x1, &x2, &y1, &y2); 2256 2257 /* The tiling and detiling functions require that the linear buffer has 2258 * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we 2259 * over-allocate the linear buffer to get the proper alignment. 2260 */ 2261 map->buffer = 2262 os_malloc_aligned(xfer->layer_stride * box->depth, 16); 2263 assert(map->buffer); 2264 map->ptr = (char *)map->buffer + (x1 & 0xf); 2265 2266 const bool has_swizzling = false; 2267 2268 if (!(xfer->usage & PIPE_MAP_DISCARD_RANGE)) { 2269 char *src = res->offset + 2270 iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS); 2271 2272 for (int s = 0; s < box->depth; s++) { 2273 unsigned x1, x2, y1, y2; 2274 tile_extents(surf, box, xfer->level, s, &x1, &x2, &y1, &y2); 2275 2276 /* Use 's' rather than 'box->z' to rebase the first slice to 0. */ 2277 void *ptr = map->ptr + s * xfer->layer_stride; 2278 2279 isl_memcpy_tiled_to_linear(x1, x2, y1, y2, ptr, src, xfer->stride, 2280 surf->row_pitch_B, has_swizzling, 2281 surf->tiling, ISL_MEMCPY_STREAMING_LOAD); 2282 } 2283 } 2284 2285 map->unmap = iris_unmap_tiled_memcpy; 2286} 2287 2288static void 2289iris_map_direct(struct iris_transfer *map) 2290{ 2291 struct pipe_transfer *xfer = &map->base.b; 2292 struct pipe_box *box = &xfer->box; 2293 struct iris_resource *res = (struct iris_resource *) xfer->resource; 2294 2295 void *ptr = res->offset + 2296 iris_bo_map(map->dbg, res->bo, xfer->usage & MAP_FLAGS); 2297 2298 if (res->base.b.target == PIPE_BUFFER) { 2299 xfer->stride = 0; 2300 xfer->layer_stride = 0; 2301 2302 map->ptr = ptr + box->x; 2303 } else { 2304 struct isl_surf *surf = &res->surf; 2305 const struct isl_format_layout *fmtl = 2306 isl_format_get_layout(surf->format); 2307 const unsigned cpp = fmtl->bpb / 8; 2308 unsigned x0_el, y0_el; 2309 2310 assert(box->x % fmtl->bw == 0); 2311 assert(box->y % fmtl->bh == 0); 2312 get_image_offset_el(surf, xfer->level, box->z, &x0_el, &y0_el); 2313 2314 x0_el += box->x / fmtl->bw; 2315 y0_el += box->y / fmtl->bh; 2316 2317 xfer->stride = isl_surf_get_row_pitch_B(surf); 2318 xfer->layer_stride = isl_surf_get_array_pitch(surf); 2319 2320 map->ptr = ptr + y0_el * xfer->stride + x0_el * cpp; 2321 } 2322} 2323 2324static bool 2325can_promote_to_async(const struct iris_resource *res, 2326 const struct pipe_box *box, 2327 enum pipe_map_flags usage) 2328{ 2329 /* If we're writing to a section of the buffer that hasn't even been 2330 * initialized with useful data, then we can safely promote this write 2331 * to be unsynchronized. This helps the common pattern of appending data. 2332 */ 2333 return res->base.b.target == PIPE_BUFFER && (usage & PIPE_MAP_WRITE) && 2334 !(usage & TC_TRANSFER_MAP_NO_INFER_UNSYNCHRONIZED) && 2335 !util_ranges_intersect(&res->valid_buffer_range, box->x, 2336 box->x + box->width); 2337} 2338 2339static void * 2340iris_transfer_map(struct pipe_context *ctx, 2341 struct pipe_resource *resource, 2342 unsigned level, 2343 enum pipe_map_flags usage, 2344 const struct pipe_box *box, 2345 struct pipe_transfer **ptransfer) 2346{ 2347 struct iris_context *ice = (struct iris_context *)ctx; 2348 struct iris_resource *res = (struct iris_resource *)resource; 2349 struct isl_surf *surf = &res->surf; 2350 2351 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) { 2352 /* Replace the backing storage with a fresh buffer for non-async maps */ 2353 if (!(usage & (PIPE_MAP_UNSYNCHRONIZED | 2354 TC_TRANSFER_MAP_NO_INVALIDATE))) 2355 iris_invalidate_resource(ctx, resource); 2356 2357 /* If we can discard the whole resource, we can discard the range. */ 2358 usage |= PIPE_MAP_DISCARD_RANGE; 2359 } 2360 2361 if (!(usage & PIPE_MAP_UNSYNCHRONIZED) && 2362 can_promote_to_async(res, box, usage)) { 2363 usage |= PIPE_MAP_UNSYNCHRONIZED; 2364 } 2365 2366 /* Avoid using GPU copies for persistent/coherent buffers, as the idea 2367 * there is to access them simultaneously on the CPU & GPU. This also 2368 * avoids trying to use GPU copies for our u_upload_mgr buffers which 2369 * contain state we're constructing for a GPU draw call, which would 2370 * kill us with infinite stack recursion. 2371 */ 2372 if (usage & (PIPE_MAP_PERSISTENT | PIPE_MAP_COHERENT)) 2373 usage |= PIPE_MAP_DIRECTLY; 2374 2375 /* We cannot provide a direct mapping of tiled resources, and we 2376 * may not be able to mmap imported BOs since they may come from 2377 * other devices that I915_GEM_MMAP cannot work with. 2378 */ 2379 if ((usage & PIPE_MAP_DIRECTLY) && 2380 (surf->tiling != ISL_TILING_LINEAR || iris_bo_is_imported(res->bo))) 2381 return NULL; 2382 2383 bool map_would_stall = false; 2384 2385 if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) { 2386 map_would_stall = 2387 resource_is_busy(ice, res) || 2388 iris_has_invalid_primary(res, level, 1, box->z, box->depth); 2389 2390 if (map_would_stall && (usage & PIPE_MAP_DONTBLOCK) && 2391 (usage & PIPE_MAP_DIRECTLY)) 2392 return NULL; 2393 } 2394 2395 struct iris_transfer *map; 2396 2397 if (usage & TC_TRANSFER_MAP_THREADED_UNSYNC) 2398 map = slab_zalloc(&ice->transfer_pool_unsync); 2399 else 2400 map = slab_zalloc(&ice->transfer_pool); 2401 2402 if (!map) 2403 return NULL; 2404 2405 struct pipe_transfer *xfer = &map->base.b; 2406 2407 map->dbg = &ice->dbg; 2408 2409 pipe_resource_reference(&xfer->resource, resource); 2410 xfer->level = level; 2411 xfer->usage = usage; 2412 xfer->box = *box; 2413 *ptransfer = xfer; 2414 2415 map->dest_had_defined_contents = 2416 util_ranges_intersect(&res->valid_buffer_range, box->x, 2417 box->x + box->width); 2418 2419 if (usage & PIPE_MAP_WRITE) 2420 util_range_add(&res->base.b, &res->valid_buffer_range, box->x, box->x + box->width); 2421 2422 if (iris_bo_mmap_mode(res->bo) != IRIS_MMAP_NONE) { 2423 /* GPU copies are not useful for buffer reads. Instead of stalling to 2424 * read from the original buffer, we'd simply copy it to a temporary... 2425 * then stall (a bit longer) to read from that buffer. 2426 * 2427 * Images are less clear-cut. Resolves can be destructive, removing 2428 * some of the underlying compression, so we'd rather blit the data to 2429 * a linear temporary and map that, to avoid the resolve. 2430 */ 2431 if (!(usage & PIPE_MAP_DISCARD_RANGE) && 2432 !iris_has_invalid_primary(res, level, 1, box->z, box->depth)) { 2433 usage |= PIPE_MAP_DIRECTLY; 2434 } 2435 2436 /* We can map directly if it wouldn't stall, there's no compression, 2437 * and we aren't doing an uncached read. 2438 */ 2439 if (!map_would_stall && 2440 !isl_aux_usage_has_compression(res->aux.usage) && 2441 !((usage & PIPE_MAP_READ) && 2442 iris_bo_mmap_mode(res->bo) != IRIS_MMAP_WB)) { 2443 usage |= PIPE_MAP_DIRECTLY; 2444 } 2445 } 2446 2447 /* TODO: Teach iris_map_tiled_memcpy about Tile4... */ 2448 if (res->surf.tiling == ISL_TILING_4) 2449 usage &= ~PIPE_MAP_DIRECTLY; 2450 2451 if (!(usage & PIPE_MAP_DIRECTLY)) { 2452 /* If we need a synchronous mapping and the resource is busy, or needs 2453 * resolving, we copy to/from a linear temporary buffer using the GPU. 2454 */ 2455 map->batch = &ice->batches[IRIS_BATCH_RENDER]; 2456 map->blorp = &ice->blorp; 2457 iris_map_copy_region(map); 2458 } else { 2459 /* Otherwise we're free to map on the CPU. */ 2460 2461 if (resource->target != PIPE_BUFFER) { 2462 iris_resource_access_raw(ice, res, level, box->z, box->depth, 2463 usage & PIPE_MAP_WRITE); 2464 } 2465 2466 if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) { 2467 iris_foreach_batch(ice, batch) { 2468 if (iris_batch_references(batch, res->bo)) 2469 iris_batch_flush(batch); 2470 } 2471 } 2472 2473 if (surf->tiling == ISL_TILING_W) { 2474 /* TODO: Teach iris_map_tiled_memcpy about W-tiling... */ 2475 iris_map_s8(map); 2476 } else if (surf->tiling != ISL_TILING_LINEAR) { 2477 iris_map_tiled_memcpy(map); 2478 } else { 2479 iris_map_direct(map); 2480 } 2481 } 2482 2483 return map->ptr; 2484} 2485 2486static void 2487iris_transfer_flush_region(struct pipe_context *ctx, 2488 struct pipe_transfer *xfer, 2489 const struct pipe_box *box) 2490{ 2491 struct iris_context *ice = (struct iris_context *)ctx; 2492 struct iris_resource *res = (struct iris_resource *) xfer->resource; 2493 struct iris_transfer *map = (void *) xfer; 2494 2495 if (map->staging) 2496 iris_flush_staging_region(xfer, box); 2497 2498 if (res->base.b.target == PIPE_BUFFER) { 2499 util_range_add(&res->base.b, &res->valid_buffer_range, box->x, box->x + box->width); 2500 } 2501 2502 /* Make sure we flag constants dirty even if there's no need to emit 2503 * any PIPE_CONTROLs to a batch. 2504 */ 2505 iris_dirty_for_history(ice, res); 2506} 2507 2508static void 2509iris_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *xfer) 2510{ 2511 struct iris_context *ice = (struct iris_context *)ctx; 2512 struct iris_transfer *map = (void *) xfer; 2513 2514 if (!(xfer->usage & (PIPE_MAP_FLUSH_EXPLICIT | 2515 PIPE_MAP_COHERENT))) { 2516 struct pipe_box flush_box = { 2517 .x = 0, .y = 0, .z = 0, 2518 .width = xfer->box.width, 2519 .height = xfer->box.height, 2520 .depth = xfer->box.depth, 2521 }; 2522 iris_transfer_flush_region(ctx, xfer, &flush_box); 2523 } 2524 2525 if (map->unmap) 2526 map->unmap(map); 2527 2528 pipe_resource_reference(&xfer->resource, NULL); 2529 2530 /* transfer_unmap is always called from the driver thread, so we have to 2531 * use transfer_pool, not transfer_pool_unsync. Freeing an object into a 2532 * different pool is allowed, however. 2533 */ 2534 slab_free(&ice->transfer_pool, map); 2535} 2536 2537/** 2538 * The pipe->texture_subdata() driver hook. 2539 * 2540 * Mesa's state tracker takes this path whenever possible, even with 2541 * PIPE_CAP_TEXTURE_TRANSFER_MODES set. 2542 */ 2543static void 2544iris_texture_subdata(struct pipe_context *ctx, 2545 struct pipe_resource *resource, 2546 unsigned level, 2547 unsigned usage, 2548 const struct pipe_box *box, 2549 const void *data, 2550 unsigned stride, 2551 unsigned layer_stride) 2552{ 2553 struct iris_context *ice = (struct iris_context *)ctx; 2554 struct iris_resource *res = (struct iris_resource *)resource; 2555 const struct isl_surf *surf = &res->surf; 2556 2557 assert(resource->target != PIPE_BUFFER); 2558 2559 /* Just use the transfer-based path for linear buffers - it will already 2560 * do a direct mapping, or a simple linear staging buffer. 2561 * 2562 * Linear staging buffers appear to be better than tiled ones, too, so 2563 * take that path if we need the GPU to perform color compression, or 2564 * stall-avoidance blits. 2565 * 2566 * TODO: Teach isl_memcpy_linear_to_tiled about Tile4... 2567 */ 2568 if (surf->tiling == ISL_TILING_LINEAR || 2569 surf->tiling == ISL_TILING_4 || 2570 isl_aux_usage_has_compression(res->aux.usage) || 2571 resource_is_busy(ice, res) || 2572 iris_bo_mmap_mode(res->bo) == IRIS_MMAP_NONE) { 2573 return u_default_texture_subdata(ctx, resource, level, usage, box, 2574 data, stride, layer_stride); 2575 } 2576 2577 /* No state trackers pass any flags other than PIPE_MAP_WRITE */ 2578 2579 iris_resource_access_raw(ice, res, level, box->z, box->depth, true); 2580 2581 iris_foreach_batch(ice, batch) { 2582 if (iris_batch_references(batch, res->bo)) 2583 iris_batch_flush(batch); 2584 } 2585 2586 uint8_t *dst = iris_bo_map(&ice->dbg, res->bo, MAP_WRITE | MAP_RAW); 2587 2588 for (int s = 0; s < box->depth; s++) { 2589 const uint8_t *src = data + s * layer_stride; 2590 2591 if (surf->tiling == ISL_TILING_W) { 2592 unsigned x0_el, y0_el; 2593 get_image_offset_el(surf, level, box->z + s, &x0_el, &y0_el); 2594 2595 for (unsigned y = 0; y < box->height; y++) { 2596 for (unsigned x = 0; x < box->width; x++) { 2597 ptrdiff_t offset = s8_offset(surf->row_pitch_B, 2598 x0_el + box->x + x, 2599 y0_el + box->y + y); 2600 dst[offset] = src[y * stride + x]; 2601 } 2602 } 2603 } else { 2604 unsigned x1, x2, y1, y2; 2605 2606 tile_extents(surf, box, level, s, &x1, &x2, &y1, &y2); 2607 2608 isl_memcpy_linear_to_tiled(x1, x2, y1, y2, 2609 (void *)dst, (void *)src, 2610 surf->row_pitch_B, stride, 2611 false, surf->tiling, ISL_MEMCPY); 2612 } 2613 } 2614} 2615 2616/** 2617 * Mark state dirty that needs to be re-emitted when a resource is written. 2618 */ 2619void 2620iris_dirty_for_history(struct iris_context *ice, 2621 struct iris_resource *res) 2622{ 2623 const uint64_t stages = res->bind_stages; 2624 uint64_t dirty = 0ull; 2625 uint64_t stage_dirty = 0ull; 2626 2627 if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) { 2628 for (unsigned stage = 0; stage < MESA_SHADER_STAGES; stage++) { 2629 if (stages & (1u << stage)) { 2630 struct iris_shader_state *shs = &ice->state.shaders[stage]; 2631 shs->dirty_cbufs |= ~0u; 2632 } 2633 } 2634 dirty |= IRIS_DIRTY_RENDER_MISC_BUFFER_FLUSHES | 2635 IRIS_DIRTY_COMPUTE_MISC_BUFFER_FLUSHES; 2636 stage_dirty |= (stages << IRIS_SHIFT_FOR_STAGE_DIRTY_CONSTANTS); 2637 } 2638 2639 if (res->bind_history & (PIPE_BIND_SAMPLER_VIEW | 2640 PIPE_BIND_SHADER_IMAGE)) { 2641 dirty |= IRIS_DIRTY_RENDER_RESOLVES_AND_FLUSHES | 2642 IRIS_DIRTY_COMPUTE_RESOLVES_AND_FLUSHES; 2643 stage_dirty |= (stages << IRIS_SHIFT_FOR_STAGE_DIRTY_BINDINGS); 2644 } 2645 2646 if (res->bind_history & PIPE_BIND_SHADER_BUFFER) { 2647 dirty |= IRIS_DIRTY_RENDER_MISC_BUFFER_FLUSHES | 2648 IRIS_DIRTY_COMPUTE_MISC_BUFFER_FLUSHES; 2649 stage_dirty |= (stages << IRIS_SHIFT_FOR_STAGE_DIRTY_BINDINGS); 2650 } 2651 2652 if (res->bind_history & PIPE_BIND_VERTEX_BUFFER) 2653 dirty |= IRIS_DIRTY_VERTEX_BUFFER_FLUSHES; 2654 2655 if (ice->state.streamout_active && (res->bind_history & PIPE_BIND_STREAM_OUTPUT)) 2656 dirty |= IRIS_DIRTY_SO_BUFFERS; 2657 2658 ice->state.dirty |= dirty; 2659 ice->state.stage_dirty |= stage_dirty; 2660} 2661 2662bool 2663iris_resource_set_clear_color(struct iris_context *ice, 2664 struct iris_resource *res, 2665 union isl_color_value color) 2666{ 2667 if (res->aux.clear_color_unknown || 2668 memcmp(&res->aux.clear_color, &color, sizeof(color)) != 0) { 2669 res->aux.clear_color = color; 2670 res->aux.clear_color_unknown = false; 2671 return true; 2672 } 2673 2674 return false; 2675} 2676 2677static enum pipe_format 2678iris_resource_get_internal_format(struct pipe_resource *p_res) 2679{ 2680 struct iris_resource *res = (void *) p_res; 2681 return res->internal_format; 2682} 2683 2684static const struct u_transfer_vtbl transfer_vtbl = { 2685 .resource_create = iris_resource_create, 2686 .resource_destroy = iris_resource_destroy, 2687 .transfer_map = iris_transfer_map, 2688 .transfer_unmap = iris_transfer_unmap, 2689 .transfer_flush_region = iris_transfer_flush_region, 2690 .get_internal_format = iris_resource_get_internal_format, 2691 .set_stencil = iris_resource_set_separate_stencil, 2692 .get_stencil = iris_resource_get_separate_stencil, 2693}; 2694 2695void 2696iris_init_screen_resource_functions(struct pipe_screen *pscreen) 2697{ 2698 pscreen->query_dmabuf_modifiers = iris_query_dmabuf_modifiers; 2699 pscreen->is_dmabuf_modifier_supported = iris_is_dmabuf_modifier_supported; 2700 pscreen->get_dmabuf_modifier_planes = iris_get_dmabuf_modifier_planes; 2701 pscreen->resource_create_with_modifiers = 2702 iris_resource_create_with_modifiers; 2703 pscreen->resource_create = u_transfer_helper_resource_create; 2704 pscreen->resource_from_user_memory = iris_resource_from_user_memory; 2705 pscreen->resource_from_handle = iris_resource_from_handle; 2706 pscreen->resource_from_memobj = iris_resource_from_memobj_wrapper; 2707 pscreen->resource_get_handle = iris_resource_get_handle; 2708 pscreen->resource_get_param = iris_resource_get_param; 2709 pscreen->resource_destroy = u_transfer_helper_resource_destroy; 2710 pscreen->memobj_create_from_handle = iris_memobj_create_from_handle; 2711 pscreen->memobj_destroy = iris_memobj_destroy; 2712 pscreen->transfer_helper = 2713 u_transfer_helper_create(&transfer_vtbl, true, true, false, true, false); 2714} 2715 2716void 2717iris_init_resource_functions(struct pipe_context *ctx) 2718{ 2719 ctx->flush_resource = iris_flush_resource; 2720 ctx->invalidate_resource = iris_invalidate_resource; 2721 ctx->buffer_map = u_transfer_helper_transfer_map; 2722 ctx->texture_map = u_transfer_helper_transfer_map; 2723 ctx->transfer_flush_region = u_transfer_helper_transfer_flush_region; 2724 ctx->buffer_unmap = u_transfer_helper_transfer_unmap; 2725 ctx->texture_unmap = u_transfer_helper_transfer_unmap; 2726 ctx->buffer_subdata = u_default_buffer_subdata; 2727 ctx->clear_buffer = u_default_clear_buffer; 2728 ctx->texture_subdata = iris_texture_subdata; 2729} 2730