1/* 2 * Copyright © 2016 Red Hat 3 * based on intel anv code: 4 * Copyright © 2015 Intel Corporation 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the "Software"), 8 * to deal in the Software without restriction, including without limitation 9 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 10 * and/or sell copies of the Software, and to permit persons to whom the 11 * Software is furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice (including the next 14 * paragraph) shall be included in all copies or substantial portions of the 15 * Software. 16 * 17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 * IN THE SOFTWARE. 24 */ 25 26#include "radv_meta.h" 27 28#include "vk_util.h" 29 30#include <fcntl.h> 31#include <limits.h> 32#ifndef _WIN32 33#include <pwd.h> 34#endif 35#include <sys/stat.h> 36 37static void 38radv_suspend_queries(struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer) 39{ 40 /* Pipeline statistics queries. */ 41 if (cmd_buffer->state.active_pipeline_queries > 0) { 42 cmd_buffer->state.flush_bits &= ~RADV_CMD_FLAG_START_PIPELINE_STATS; 43 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_STOP_PIPELINE_STATS; 44 } 45 46 /* Occlusion queries. */ 47 if (cmd_buffer->state.active_occlusion_queries > 0) { 48 radv_set_db_count_control(cmd_buffer, false); 49 } 50 51 /* Primitives generated queries. */ 52 if (cmd_buffer->state.prims_gen_query_enabled) { 53 cmd_buffer->state.suspend_streamout = true; 54 radv_emit_streamout_enable(cmd_buffer); 55 56 /* Save the number of active GDS queries and reset it to make sure internal operations won't 57 * increment the counters via GDS. 58 */ 59 state->active_pipeline_gds_queries = cmd_buffer->state.active_pipeline_gds_queries; 60 cmd_buffer->state.active_pipeline_gds_queries = 0; 61 } 62} 63 64static void 65radv_resume_queries(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer) 66{ 67 /* Pipeline statistics queries. */ 68 if (cmd_buffer->state.active_pipeline_queries > 0) { 69 cmd_buffer->state.flush_bits &= ~RADV_CMD_FLAG_STOP_PIPELINE_STATS; 70 cmd_buffer->state.flush_bits |= RADV_CMD_FLAG_START_PIPELINE_STATS; 71 } 72 73 /* Occlusion queries. */ 74 if (cmd_buffer->state.active_occlusion_queries > 0) { 75 radv_set_db_count_control(cmd_buffer, true); 76 } 77 78 /* Primitives generated queries. */ 79 if (cmd_buffer->state.prims_gen_query_enabled) { 80 cmd_buffer->state.suspend_streamout = false; 81 radv_emit_streamout_enable(cmd_buffer); 82 83 /* Restore the number of active GDS queries to resume counting. */ 84 cmd_buffer->state.active_pipeline_gds_queries = state->active_pipeline_gds_queries; 85 } 86} 87 88void 89radv_meta_save(struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer, 90 uint32_t flags) 91{ 92 VkPipelineBindPoint bind_point = flags & RADV_META_SAVE_GRAPHICS_PIPELINE 93 ? VK_PIPELINE_BIND_POINT_GRAPHICS 94 : VK_PIPELINE_BIND_POINT_COMPUTE; 95 struct radv_descriptor_state *descriptors_state = 96 radv_get_descriptors_state(cmd_buffer, bind_point); 97 98 assert(flags & (RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_COMPUTE_PIPELINE)); 99 100 state->flags = flags; 101 102 if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) { 103 assert(!(state->flags & RADV_META_SAVE_COMPUTE_PIPELINE)); 104 105 state->old_graphics_pipeline = cmd_buffer->state.graphics_pipeline; 106 107 /* Save all viewports. */ 108 state->dynamic.viewport.count = cmd_buffer->state.dynamic.viewport.count; 109 typed_memcpy(state->dynamic.viewport.viewports, cmd_buffer->state.dynamic.viewport.viewports, 110 MAX_VIEWPORTS); 111 typed_memcpy(state->dynamic.viewport.xform, cmd_buffer->state.dynamic.viewport.xform, 112 MAX_VIEWPORTS); 113 114 /* Save all scissors. */ 115 state->dynamic.scissor.count = cmd_buffer->state.dynamic.scissor.count; 116 typed_memcpy(state->dynamic.scissor.scissors, cmd_buffer->state.dynamic.scissor.scissors, 117 MAX_SCISSORS); 118 119 state->dynamic.line_stipple.factor = cmd_buffer->state.dynamic.line_stipple.factor; 120 state->dynamic.line_stipple.pattern = cmd_buffer->state.dynamic.line_stipple.pattern; 121 122 state->dynamic.cull_mode = cmd_buffer->state.dynamic.cull_mode; 123 state->dynamic.front_face = cmd_buffer->state.dynamic.front_face; 124 125 state->dynamic.primitive_topology = cmd_buffer->state.dynamic.primitive_topology; 126 127 state->dynamic.depth_test_enable = cmd_buffer->state.dynamic.depth_test_enable; 128 state->dynamic.depth_write_enable = cmd_buffer->state.dynamic.depth_write_enable; 129 state->dynamic.depth_compare_op = cmd_buffer->state.dynamic.depth_compare_op; 130 state->dynamic.depth_bounds_test_enable = cmd_buffer->state.dynamic.depth_bounds_test_enable; 131 state->dynamic.stencil_test_enable = cmd_buffer->state.dynamic.stencil_test_enable; 132 133 state->dynamic.stencil_op.front.compare_op = cmd_buffer->state.dynamic.stencil_op.front.compare_op; 134 state->dynamic.stencil_op.front.fail_op = cmd_buffer->state.dynamic.stencil_op.front.fail_op; 135 state->dynamic.stencil_op.front.pass_op = cmd_buffer->state.dynamic.stencil_op.front.pass_op; 136 state->dynamic.stencil_op.front.depth_fail_op = 137 cmd_buffer->state.dynamic.stencil_op.front.depth_fail_op; 138 139 state->dynamic.stencil_op.back.compare_op = cmd_buffer->state.dynamic.stencil_op.back.compare_op; 140 state->dynamic.stencil_op.back.fail_op = cmd_buffer->state.dynamic.stencil_op.back.fail_op; 141 state->dynamic.stencil_op.back.pass_op = cmd_buffer->state.dynamic.stencil_op.back.pass_op; 142 state->dynamic.stencil_op.back.depth_fail_op = 143 cmd_buffer->state.dynamic.stencil_op.back.depth_fail_op; 144 145 state->dynamic.line_width = cmd_buffer->state.dynamic.line_width; 146 147 state->dynamic.depth_bias.bias = cmd_buffer->state.dynamic.depth_bias.bias; 148 state->dynamic.depth_bias.clamp = cmd_buffer->state.dynamic.depth_bias.clamp; 149 state->dynamic.depth_bias.slope = cmd_buffer->state.dynamic.depth_bias.slope; 150 151 memcpy(state->dynamic.blend_constants, cmd_buffer->state.dynamic.blend_constants, 152 sizeof(state->dynamic.blend_constants)); 153 154 state->dynamic.depth_bounds.min = cmd_buffer->state.dynamic.depth_bounds.min; 155 state->dynamic.depth_bounds.max = cmd_buffer->state.dynamic.depth_bounds.max; 156 157 state->dynamic.stencil_compare_mask.front = cmd_buffer->state.dynamic.stencil_compare_mask.front; 158 state->dynamic.stencil_compare_mask.back = cmd_buffer->state.dynamic.stencil_compare_mask.back; 159 160 state->dynamic.stencil_write_mask.front = cmd_buffer->state.dynamic.stencil_write_mask.front; 161 state->dynamic.stencil_write_mask.back = cmd_buffer->state.dynamic.stencil_write_mask.back; 162 163 state->dynamic.stencil_reference.front = cmd_buffer->state.dynamic.stencil_reference.front; 164 state->dynamic.stencil_reference.back = cmd_buffer->state.dynamic.stencil_reference.back; 165 166 state->dynamic.fragment_shading_rate.size = cmd_buffer->state.dynamic.fragment_shading_rate.size; 167 state->dynamic.fragment_shading_rate.combiner_ops[0] = 168 cmd_buffer->state.dynamic.fragment_shading_rate.combiner_ops[0]; 169 state->dynamic.fragment_shading_rate.combiner_ops[1] = 170 cmd_buffer->state.dynamic.fragment_shading_rate.combiner_ops[1]; 171 172 state->dynamic.depth_bias_enable = cmd_buffer->state.dynamic.depth_bias_enable; 173 174 state->dynamic.primitive_restart_enable = cmd_buffer->state.dynamic.primitive_restart_enable; 175 176 state->dynamic.rasterizer_discard_enable = cmd_buffer->state.dynamic.rasterizer_discard_enable; 177 178 state->dynamic.logic_op = cmd_buffer->state.dynamic.logic_op; 179 180 state->dynamic.color_write_enable = cmd_buffer->state.dynamic.color_write_enable; 181 182 state->dynamic.discard_rectangle.count = cmd_buffer->state.dynamic.discard_rectangle.count; 183 typed_memcpy(state->dynamic.discard_rectangle.rectangles, 184 cmd_buffer->state.dynamic.discard_rectangle.rectangles, 185 MAX_DISCARD_RECTANGLES); 186 } 187 188 if (state->flags & RADV_META_SAVE_SAMPLE_LOCATIONS) { 189 typed_memcpy(&state->dynamic.sample_location, &cmd_buffer->state.dynamic.sample_location, 1); 190 } 191 192 if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) { 193 assert(!(state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE)); 194 195 state->old_compute_pipeline = cmd_buffer->state.compute_pipeline; 196 } 197 198 if (state->flags & RADV_META_SAVE_DESCRIPTORS) { 199 state->old_descriptor_set0 = descriptors_state->sets[0]; 200 if (!(descriptors_state->valid & 1) || !state->old_descriptor_set0) 201 state->flags &= ~RADV_META_SAVE_DESCRIPTORS; 202 } 203 204 if (state->flags & RADV_META_SAVE_CONSTANTS) { 205 memcpy(state->push_constants, cmd_buffer->push_constants, MAX_PUSH_CONSTANTS_SIZE); 206 } 207 208 if (state->flags & RADV_META_SAVE_PASS) { 209 state->pass = cmd_buffer->state.pass; 210 state->subpass = cmd_buffer->state.subpass; 211 state->framebuffer = cmd_buffer->state.framebuffer; 212 state->attachments = cmd_buffer->state.attachments; 213 state->render_area = cmd_buffer->state.render_area; 214 } 215 216 if (state->flags & RADV_META_SUSPEND_PREDICATING) { 217 state->predicating = cmd_buffer->state.predicating; 218 cmd_buffer->state.predicating = false; 219 } 220 221 radv_suspend_queries(state, cmd_buffer); 222} 223 224void 225radv_meta_restore(const struct radv_meta_saved_state *state, struct radv_cmd_buffer *cmd_buffer) 226{ 227 VkPipelineBindPoint bind_point = state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE 228 ? VK_PIPELINE_BIND_POINT_GRAPHICS 229 : VK_PIPELINE_BIND_POINT_COMPUTE; 230 231 if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) { 232 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_GRAPHICS, 233 radv_pipeline_to_handle(&state->old_graphics_pipeline->base)); 234 235 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_PIPELINE; 236 237 /* Restore all viewports. */ 238 cmd_buffer->state.dynamic.viewport.count = state->dynamic.viewport.count; 239 typed_memcpy(cmd_buffer->state.dynamic.viewport.viewports, state->dynamic.viewport.viewports, 240 MAX_VIEWPORTS); 241 typed_memcpy(cmd_buffer->state.dynamic.viewport.xform, state->dynamic.viewport.xform, 242 MAX_VIEWPORTS); 243 244 /* Restore all scissors. */ 245 cmd_buffer->state.dynamic.scissor.count = state->dynamic.scissor.count; 246 typed_memcpy(cmd_buffer->state.dynamic.scissor.scissors, state->dynamic.scissor.scissors, 247 MAX_SCISSORS); 248 249 cmd_buffer->state.dynamic.line_stipple.factor = state->dynamic.line_stipple.factor; 250 cmd_buffer->state.dynamic.line_stipple.pattern = state->dynamic.line_stipple.pattern; 251 252 cmd_buffer->state.dynamic.cull_mode = state->dynamic.cull_mode; 253 cmd_buffer->state.dynamic.front_face = state->dynamic.front_face; 254 255 cmd_buffer->state.dynamic.primitive_topology = state->dynamic.primitive_topology; 256 257 cmd_buffer->state.dynamic.depth_test_enable = state->dynamic.depth_test_enable; 258 cmd_buffer->state.dynamic.depth_write_enable = state->dynamic.depth_write_enable; 259 cmd_buffer->state.dynamic.depth_compare_op = state->dynamic.depth_compare_op; 260 cmd_buffer->state.dynamic.depth_bounds_test_enable = state->dynamic.depth_bounds_test_enable; 261 cmd_buffer->state.dynamic.stencil_test_enable = state->dynamic.stencil_test_enable; 262 263 cmd_buffer->state.dynamic.stencil_op.front.compare_op = state->dynamic.stencil_op.front.compare_op; 264 cmd_buffer->state.dynamic.stencil_op.front.fail_op = state->dynamic.stencil_op.front.fail_op; 265 cmd_buffer->state.dynamic.stencil_op.front.pass_op = state->dynamic.stencil_op.front.pass_op; 266 cmd_buffer->state.dynamic.stencil_op.front.depth_fail_op = 267 state->dynamic.stencil_op.front.depth_fail_op; 268 269 cmd_buffer->state.dynamic.stencil_op.back.compare_op = state->dynamic.stencil_op.back.compare_op; 270 cmd_buffer->state.dynamic.stencil_op.back.fail_op = state->dynamic.stencil_op.back.fail_op; 271 cmd_buffer->state.dynamic.stencil_op.back.pass_op = state->dynamic.stencil_op.back.pass_op; 272 cmd_buffer->state.dynamic.stencil_op.back.depth_fail_op = 273 state->dynamic.stencil_op.back.depth_fail_op; 274 275 cmd_buffer->state.dynamic.line_width = state->dynamic.line_width; 276 277 cmd_buffer->state.dynamic.depth_bias.bias = state->dynamic.depth_bias.bias; 278 cmd_buffer->state.dynamic.depth_bias.clamp = state->dynamic.depth_bias.clamp; 279 cmd_buffer->state.dynamic.depth_bias.slope = state->dynamic.depth_bias.slope; 280 281 memcpy(cmd_buffer->state.dynamic.blend_constants, state->dynamic.blend_constants, 282 sizeof(state->dynamic.blend_constants)); 283 284 cmd_buffer->state.dynamic.depth_bounds.min = state->dynamic.depth_bounds.min; 285 cmd_buffer->state.dynamic.depth_bounds.max = state->dynamic.depth_bounds.max; 286 287 cmd_buffer->state.dynamic.stencil_compare_mask.front = state->dynamic.stencil_compare_mask.front; 288 cmd_buffer->state.dynamic.stencil_compare_mask.back = state->dynamic.stencil_compare_mask.back; 289 290 cmd_buffer->state.dynamic.stencil_write_mask.front = state->dynamic.stencil_write_mask.front; 291 cmd_buffer->state.dynamic.stencil_write_mask.back = state->dynamic.stencil_write_mask.back; 292 293 cmd_buffer->state.dynamic.stencil_reference.front = state->dynamic.stencil_reference.front; 294 cmd_buffer->state.dynamic.stencil_reference.back = state->dynamic.stencil_reference.back; 295 296 cmd_buffer->state.dynamic.fragment_shading_rate.size = state->dynamic.fragment_shading_rate.size; 297 cmd_buffer->state.dynamic.fragment_shading_rate.combiner_ops[0] = 298 state->dynamic.fragment_shading_rate.combiner_ops[0]; 299 cmd_buffer->state.dynamic.fragment_shading_rate.combiner_ops[1] = 300 state->dynamic.fragment_shading_rate.combiner_ops[1]; 301 302 cmd_buffer->state.dynamic.depth_bias_enable = state->dynamic.depth_bias_enable; 303 304 cmd_buffer->state.dynamic.primitive_restart_enable = state->dynamic.primitive_restart_enable; 305 306 cmd_buffer->state.dynamic.rasterizer_discard_enable = state->dynamic.rasterizer_discard_enable; 307 308 cmd_buffer->state.dynamic.logic_op = state->dynamic.logic_op; 309 310 cmd_buffer->state.dynamic.color_write_enable = state->dynamic.color_write_enable; 311 312 cmd_buffer->state.dynamic.discard_rectangle.count = state->dynamic.discard_rectangle.count; 313 typed_memcpy(cmd_buffer->state.dynamic.discard_rectangle.rectangles, 314 state->dynamic.discard_rectangle.rectangles, 315 MAX_DISCARD_RECTANGLES); 316 317 cmd_buffer->state.dirty |= 318 RADV_CMD_DIRTY_DYNAMIC_VIEWPORT | RADV_CMD_DIRTY_DYNAMIC_SCISSOR | 319 RADV_CMD_DIRTY_DYNAMIC_CULL_MODE | RADV_CMD_DIRTY_DYNAMIC_FRONT_FACE | 320 RADV_CMD_DIRTY_DYNAMIC_PRIMITIVE_TOPOLOGY | RADV_CMD_DIRTY_DYNAMIC_DEPTH_TEST_ENABLE | 321 RADV_CMD_DIRTY_DYNAMIC_DEPTH_WRITE_ENABLE | RADV_CMD_DIRTY_DYNAMIC_DEPTH_COMPARE_OP | 322 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS_TEST_ENABLE | 323 RADV_CMD_DIRTY_DYNAMIC_STENCIL_TEST_ENABLE | RADV_CMD_DIRTY_DYNAMIC_STENCIL_OP | 324 RADV_CMD_DIRTY_DYNAMIC_STENCIL_WRITE_MASK | RADV_CMD_DIRTY_DYNAMIC_STENCIL_REFERENCE | 325 RADV_CMD_DIRTY_DYNAMIC_FRAGMENT_SHADING_RATE | RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS_ENABLE | 326 RADV_CMD_DIRTY_DYNAMIC_PRIMITIVE_RESTART_ENABLE | 327 RADV_CMD_DIRTY_DYNAMIC_RASTERIZER_DISCARD_ENABLE | RADV_CMD_DIRTY_DYNAMIC_LOGIC_OP | 328 RADV_CMD_DIRTY_DYNAMIC_COLOR_WRITE_ENABLE | RADV_CMD_DIRTY_DYNAMIC_LINE_STIPPLE | 329 RADV_CMD_DIRTY_DYNAMIC_STENCIL_COMPARE_MASK | RADV_CMD_DIRTY_DYNAMIC_DEPTH_BOUNDS | 330 RADV_CMD_DIRTY_DYNAMIC_BLEND_CONSTANTS | RADV_CMD_DIRTY_DYNAMIC_LINE_WIDTH | 331 RADV_CMD_DIRTY_DYNAMIC_DEPTH_BIAS | RADV_CMD_DIRTY_DYNAMIC_DISCARD_RECTANGLE; 332 } 333 334 if (state->flags & RADV_META_SAVE_SAMPLE_LOCATIONS) { 335 typed_memcpy(&cmd_buffer->state.dynamic.sample_location.locations, 336 &state->dynamic.sample_location.locations, 1); 337 338 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_DYNAMIC_SAMPLE_LOCATIONS; 339 } 340 341 if (state->flags & RADV_META_SAVE_COMPUTE_PIPELINE) { 342 if (state->old_compute_pipeline) { 343 radv_CmdBindPipeline(radv_cmd_buffer_to_handle(cmd_buffer), VK_PIPELINE_BIND_POINT_COMPUTE, 344 radv_pipeline_to_handle(&state->old_compute_pipeline->base)); 345 } 346 } 347 348 if (state->flags & RADV_META_SAVE_DESCRIPTORS) { 349 radv_set_descriptor_set(cmd_buffer, bind_point, state->old_descriptor_set0, 0); 350 } 351 352 if (state->flags & RADV_META_SAVE_CONSTANTS) { 353 VkShaderStageFlags stages = VK_SHADER_STAGE_COMPUTE_BIT; 354 355 if (state->flags & RADV_META_SAVE_GRAPHICS_PIPELINE) 356 stages |= VK_SHADER_STAGE_ALL_GRAPHICS; 357 358 radv_CmdPushConstants(radv_cmd_buffer_to_handle(cmd_buffer), VK_NULL_HANDLE, stages, 0, 359 MAX_PUSH_CONSTANTS_SIZE, state->push_constants); 360 } 361 362 if (state->flags & RADV_META_SAVE_PASS) { 363 cmd_buffer->state.pass = state->pass; 364 cmd_buffer->state.subpass = state->subpass; 365 cmd_buffer->state.framebuffer = state->framebuffer; 366 cmd_buffer->state.attachments = state->attachments; 367 cmd_buffer->state.render_area = state->render_area; 368 if (state->subpass) 369 cmd_buffer->state.dirty |= RADV_CMD_DIRTY_FRAMEBUFFER; 370 } 371 372 if (state->flags & RADV_META_SUSPEND_PREDICATING) 373 cmd_buffer->state.predicating = state->predicating; 374 375 radv_resume_queries(state, cmd_buffer); 376} 377 378VkImageViewType 379radv_meta_get_view_type(const struct radv_image *image) 380{ 381 switch (image->vk.image_type) { 382 case VK_IMAGE_TYPE_1D: 383 return VK_IMAGE_VIEW_TYPE_1D; 384 case VK_IMAGE_TYPE_2D: 385 return VK_IMAGE_VIEW_TYPE_2D; 386 case VK_IMAGE_TYPE_3D: 387 return VK_IMAGE_VIEW_TYPE_3D; 388 default: 389 unreachable("bad VkImageViewType"); 390 } 391} 392 393/** 394 * When creating a destination VkImageView, this function provides the needed 395 * VkImageViewCreateInfo::subresourceRange::baseArrayLayer. 396 */ 397uint32_t 398radv_meta_get_iview_layer(const struct radv_image *dest_image, 399 const VkImageSubresourceLayers *dest_subresource, 400 const VkOffset3D *dest_offset) 401{ 402 switch (dest_image->vk.image_type) { 403 case VK_IMAGE_TYPE_1D: 404 case VK_IMAGE_TYPE_2D: 405 return dest_subresource->baseArrayLayer; 406 case VK_IMAGE_TYPE_3D: 407 /* HACK: Vulkan does not allow attaching a 3D image to a framebuffer, 408 * but meta does it anyway. When doing so, we translate the 409 * destination's z offset into an array offset. 410 */ 411 return dest_offset->z; 412 default: 413 assert(!"bad VkImageType"); 414 return 0; 415 } 416} 417 418static VKAPI_ATTR void * VKAPI_CALL 419meta_alloc(void *_device, size_t size, size_t alignment, VkSystemAllocationScope allocationScope) 420{ 421 struct radv_device *device = _device; 422 return device->vk.alloc.pfnAllocation(device->vk.alloc.pUserData, size, alignment, 423 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); 424} 425 426static VKAPI_ATTR void * VKAPI_CALL 427meta_realloc(void *_device, void *original, size_t size, size_t alignment, 428 VkSystemAllocationScope allocationScope) 429{ 430 struct radv_device *device = _device; 431 return device->vk.alloc.pfnReallocation(device->vk.alloc.pUserData, original, size, alignment, 432 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE); 433} 434 435static VKAPI_ATTR void VKAPI_CALL 436meta_free(void *_device, void *data) 437{ 438 struct radv_device *device = _device; 439 device->vk.alloc.pfnFree(device->vk.alloc.pUserData, data); 440} 441 442#ifndef _WIN32 443static bool 444radv_builtin_cache_path(char *path) 445{ 446 char *xdg_cache_home = getenv("XDG_CACHE_HOME"); 447 const char *suffix = "/radv_builtin_shaders"; 448 const char *suffix2 = "/.cache/radv_builtin_shaders"; 449 struct passwd pwd, *result; 450 char path2[PATH_MAX + 1]; /* PATH_MAX is not a real max,but suffices here. */ 451 int ret; 452 453 if (xdg_cache_home) { 454 ret = snprintf(path, PATH_MAX + 1, "%s%s%zd", xdg_cache_home, suffix, sizeof(void *) * 8); 455 return ret > 0 && ret < PATH_MAX + 1; 456 } 457 458 getpwuid_r(getuid(), &pwd, path2, PATH_MAX - strlen(suffix2), &result); 459 if (!result) 460 return false; 461 462 strcpy(path, pwd.pw_dir); 463 strcat(path, "/.cache"); 464 if (mkdir(path, 0755) && errno != EEXIST) 465 return false; 466 467 ret = snprintf(path, PATH_MAX + 1, "%s%s%zd", pwd.pw_dir, suffix2, sizeof(void *) * 8); 468 return ret > 0 && ret < PATH_MAX + 1; 469} 470#endif 471 472static bool 473radv_load_meta_pipeline(struct radv_device *device) 474{ 475#ifdef _WIN32 476 return false; 477#else 478 char path[PATH_MAX + 1]; 479 struct stat st; 480 void *data = NULL; 481 bool ret = false; 482 483 if (!radv_builtin_cache_path(path)) 484 return false; 485 486 int fd = open(path, O_RDONLY); 487 if (fd < 0) 488 return false; 489 if (fstat(fd, &st)) 490 goto fail; 491 data = malloc(st.st_size); 492 if (!data) 493 goto fail; 494 if (read(fd, data, st.st_size) == -1) 495 goto fail; 496 497 ret = radv_pipeline_cache_load(&device->meta_state.cache, data, st.st_size); 498fail: 499 free(data); 500 close(fd); 501 return ret; 502#endif 503} 504 505static void 506radv_store_meta_pipeline(struct radv_device *device) 507{ 508#ifndef _WIN32 509 char path[PATH_MAX + 1], path2[PATH_MAX + 7]; 510 size_t size; 511 void *data = NULL; 512 513 if (!device->meta_state.cache.modified) 514 return; 515 516 if (radv_GetPipelineCacheData(radv_device_to_handle(device), 517 radv_pipeline_cache_to_handle(&device->meta_state.cache), &size, 518 NULL)) 519 return; 520 521 if (!radv_builtin_cache_path(path)) 522 return; 523 524 strcpy(path2, path); 525 strcat(path2, "XXXXXX"); 526 int fd = mkstemp(path2); // open(path, O_WRONLY | O_CREAT, 0600); 527 if (fd < 0) 528 return; 529 data = malloc(size); 530 if (!data) 531 goto fail; 532 533 if (radv_GetPipelineCacheData(radv_device_to_handle(device), 534 radv_pipeline_cache_to_handle(&device->meta_state.cache), &size, 535 data)) 536 goto fail; 537 if (write(fd, data, size) == -1) 538 goto fail; 539 540 rename(path2, path); 541fail: 542 free(data); 543 close(fd); 544 unlink(path2); 545#endif 546} 547 548VkResult 549radv_device_init_meta(struct radv_device *device) 550{ 551 VkResult result; 552 553 memset(&device->meta_state, 0, sizeof(device->meta_state)); 554 555 device->meta_state.alloc = (VkAllocationCallbacks){ 556 .pUserData = device, 557 .pfnAllocation = meta_alloc, 558 .pfnReallocation = meta_realloc, 559 .pfnFree = meta_free, 560 }; 561 562 device->meta_state.cache.alloc = device->meta_state.alloc; 563 radv_pipeline_cache_init(&device->meta_state.cache, device); 564 bool loaded_cache = radv_load_meta_pipeline(device); 565 bool on_demand = !loaded_cache; 566 567 mtx_init(&device->meta_state.mtx, mtx_plain); 568 569 device->app_shaders_internal = true; 570 571 result = radv_device_init_meta_clear_state(device, on_demand); 572 if (result != VK_SUCCESS) 573 goto fail_clear; 574 575 result = radv_device_init_meta_resolve_state(device, on_demand); 576 if (result != VK_SUCCESS) 577 goto fail_resolve; 578 579 result = radv_device_init_meta_blit_state(device, on_demand); 580 if (result != VK_SUCCESS) 581 goto fail_blit; 582 583 result = radv_device_init_meta_blit2d_state(device, on_demand); 584 if (result != VK_SUCCESS) 585 goto fail_blit2d; 586 587 result = radv_device_init_meta_bufimage_state(device); 588 if (result != VK_SUCCESS) 589 goto fail_bufimage; 590 591 result = radv_device_init_meta_depth_decomp_state(device, on_demand); 592 if (result != VK_SUCCESS) 593 goto fail_depth_decomp; 594 595 result = radv_device_init_meta_buffer_state(device); 596 if (result != VK_SUCCESS) 597 goto fail_buffer; 598 599 result = radv_device_init_meta_query_state(device, on_demand); 600 if (result != VK_SUCCESS) 601 goto fail_query; 602 603 result = radv_device_init_meta_fast_clear_flush_state(device, on_demand); 604 if (result != VK_SUCCESS) 605 goto fail_fast_clear; 606 607 result = radv_device_init_meta_resolve_compute_state(device, on_demand); 608 if (result != VK_SUCCESS) 609 goto fail_resolve_compute; 610 611 result = radv_device_init_meta_resolve_fragment_state(device, on_demand); 612 if (result != VK_SUCCESS) 613 goto fail_resolve_fragment; 614 615 result = radv_device_init_meta_fmask_expand_state(device); 616 if (result != VK_SUCCESS) 617 goto fail_fmask_expand; 618 619 if (radv_enable_rt(device->physical_device, false)) { 620 result = radv_device_init_accel_struct_build_state(device); 621 if (result != VK_SUCCESS) 622 goto fail_accel_struct_build; 623 } 624 625 result = radv_device_init_meta_fmask_copy_state(device); 626 if (result != VK_SUCCESS) 627 goto fail_fmask_copy; 628 629 result = radv_device_init_meta_etc_decode_state(device, on_demand); 630 if (result != VK_SUCCESS) 631 goto fail_etc_decode; 632 633 if (device->uses_device_generated_commands) { 634 result = radv_device_init_dgc_prepare_state(device); 635 if (result != VK_SUCCESS) 636 goto fail_dgc; 637 } 638 639 device->app_shaders_internal = false; 640 641 return VK_SUCCESS; 642 643fail_dgc: 644 radv_device_finish_dgc_prepare_state(device); 645fail_etc_decode: 646 radv_device_finish_meta_etc_decode_state(device); 647fail_fmask_copy: 648 radv_device_finish_meta_fmask_copy_state(device); 649fail_accel_struct_build: 650 radv_device_finish_accel_struct_build_state(device); 651fail_fmask_expand: 652 radv_device_finish_meta_fmask_expand_state(device); 653fail_resolve_fragment: 654 radv_device_finish_meta_resolve_fragment_state(device); 655fail_resolve_compute: 656 radv_device_finish_meta_resolve_compute_state(device); 657fail_fast_clear: 658 radv_device_finish_meta_fast_clear_flush_state(device); 659fail_query: 660 radv_device_finish_meta_query_state(device); 661fail_buffer: 662 radv_device_finish_meta_buffer_state(device); 663fail_depth_decomp: 664 radv_device_finish_meta_depth_decomp_state(device); 665fail_bufimage: 666 radv_device_finish_meta_bufimage_state(device); 667fail_blit2d: 668 radv_device_finish_meta_blit2d_state(device); 669fail_blit: 670 radv_device_finish_meta_blit_state(device); 671fail_resolve: 672 radv_device_finish_meta_resolve_state(device); 673fail_clear: 674 radv_device_finish_meta_clear_state(device); 675 676 mtx_destroy(&device->meta_state.mtx); 677 radv_pipeline_cache_finish(&device->meta_state.cache); 678 return result; 679} 680 681void 682radv_device_finish_meta(struct radv_device *device) 683{ 684 radv_device_finish_dgc_prepare_state(device); 685 radv_device_finish_meta_etc_decode_state(device); 686 radv_device_finish_accel_struct_build_state(device); 687 radv_device_finish_meta_clear_state(device); 688 radv_device_finish_meta_resolve_state(device); 689 radv_device_finish_meta_blit_state(device); 690 radv_device_finish_meta_blit2d_state(device); 691 radv_device_finish_meta_bufimage_state(device); 692 radv_device_finish_meta_depth_decomp_state(device); 693 radv_device_finish_meta_query_state(device); 694 radv_device_finish_meta_buffer_state(device); 695 radv_device_finish_meta_fast_clear_flush_state(device); 696 radv_device_finish_meta_resolve_compute_state(device); 697 radv_device_finish_meta_resolve_fragment_state(device); 698 radv_device_finish_meta_fmask_expand_state(device); 699 radv_device_finish_meta_dcc_retile_state(device); 700 radv_device_finish_meta_copy_vrs_htile_state(device); 701 radv_device_finish_meta_fmask_copy_state(device); 702 703 radv_store_meta_pipeline(device); 704 radv_pipeline_cache_finish(&device->meta_state.cache); 705 mtx_destroy(&device->meta_state.mtx); 706} 707 708nir_builder PRINTFLIKE(3, 4) 709 radv_meta_init_shader(struct radv_device *dev, gl_shader_stage stage, const char *name, ...) 710{ 711 nir_builder b = nir_builder_init_simple_shader(stage, NULL, NULL); 712 if (name) { 713 va_list args; 714 va_start(args, name); 715 b.shader->info.name = ralloc_vasprintf(b.shader, name, args); 716 va_end(args); 717 } 718 719 b.shader->options = &dev->physical_device->nir_options[stage]; 720 b.shader->info.workgroup_size[0] = 1; 721 b.shader->info.workgroup_size[1] = 1; 722 b.shader->info.workgroup_size[2] = 1; 723 724 return b; 725} 726 727/* vertex shader that generates vertices */ 728nir_shader * 729radv_meta_build_nir_vs_generate_vertices(struct radv_device *dev) 730{ 731 const struct glsl_type *vec4 = glsl_vec4_type(); 732 733 nir_variable *v_position; 734 735 nir_builder b = radv_meta_init_shader(dev, MESA_SHADER_VERTEX, "meta_vs_gen_verts"); 736 737 nir_ssa_def *outvec = nir_gen_rect_vertices(&b, NULL, NULL); 738 739 v_position = nir_variable_create(b.shader, nir_var_shader_out, vec4, "gl_Position"); 740 v_position->data.location = VARYING_SLOT_POS; 741 742 nir_store_var(&b, v_position, outvec, 0xf); 743 744 return b.shader; 745} 746 747nir_shader * 748radv_meta_build_nir_fs_noop(struct radv_device *dev) 749{ 750 return radv_meta_init_shader(dev, MESA_SHADER_FRAGMENT, "meta_noop_fs").shader; 751} 752 753void 754radv_meta_build_resolve_shader_core(nir_builder *b, bool is_integer, int samples, 755 nir_variable *input_img, nir_variable *color, 756 nir_ssa_def *img_coord) 757{ 758 /* do a txf_ms on each sample */ 759 nir_ssa_def *tmp; 760 bool inserted_if = false; 761 762 nir_ssa_def *input_img_deref = &nir_build_deref_var(b, input_img)->dest.ssa; 763 764 nir_tex_instr *tex = nir_tex_instr_create(b->shader, 3); 765 tex->sampler_dim = GLSL_SAMPLER_DIM_MS; 766 tex->op = nir_texop_txf_ms; 767 tex->src[0].src_type = nir_tex_src_coord; 768 tex->src[0].src = nir_src_for_ssa(img_coord); 769 tex->src[1].src_type = nir_tex_src_ms_index; 770 tex->src[1].src = nir_src_for_ssa(nir_imm_int(b, 0)); 771 tex->src[2].src_type = nir_tex_src_texture_deref; 772 tex->src[2].src = nir_src_for_ssa(input_img_deref); 773 tex->dest_type = nir_get_nir_type_for_glsl_base_type(glsl_get_sampler_result_type(input_img->type)); 774 tex->is_array = false; 775 tex->coord_components = 2; 776 777 nir_ssa_dest_init(&tex->instr, &tex->dest, 4, 32, "tex"); 778 nir_builder_instr_insert(b, &tex->instr); 779 780 tmp = &tex->dest.ssa; 781 782 if (!is_integer && samples > 1) { 783 nir_tex_instr *tex_all_same = nir_tex_instr_create(b->shader, 2); 784 tex_all_same->sampler_dim = GLSL_SAMPLER_DIM_MS; 785 tex_all_same->op = nir_texop_samples_identical; 786 tex_all_same->src[0].src_type = nir_tex_src_coord; 787 tex_all_same->src[0].src = nir_src_for_ssa(img_coord); 788 tex_all_same->src[1].src_type = nir_tex_src_texture_deref; 789 tex_all_same->src[1].src = nir_src_for_ssa(input_img_deref); 790 tex_all_same->dest_type = nir_type_bool1; 791 tex_all_same->is_array = false; 792 tex_all_same->coord_components = 2; 793 794 nir_ssa_dest_init(&tex_all_same->instr, &tex_all_same->dest, 1, 1, "tex"); 795 nir_builder_instr_insert(b, &tex_all_same->instr); 796 797 nir_ssa_def *not_all_same = nir_inot(b, &tex_all_same->dest.ssa); 798 nir_push_if(b, not_all_same); 799 for (int i = 1; i < samples; i++) { 800 nir_tex_instr *tex_add = nir_tex_instr_create(b->shader, 3); 801 tex_add->sampler_dim = GLSL_SAMPLER_DIM_MS; 802 tex_add->op = nir_texop_txf_ms; 803 tex_add->src[0].src_type = nir_tex_src_coord; 804 tex_add->src[0].src = nir_src_for_ssa(img_coord); 805 tex_add->src[1].src_type = nir_tex_src_ms_index; 806 tex_add->src[1].src = nir_src_for_ssa(nir_imm_int(b, i)); 807 tex_add->src[2].src_type = nir_tex_src_texture_deref; 808 tex_add->src[2].src = nir_src_for_ssa(input_img_deref); 809 tex_add->dest_type = nir_type_float32; 810 tex_add->is_array = false; 811 tex_add->coord_components = 2; 812 813 nir_ssa_dest_init(&tex_add->instr, &tex_add->dest, 4, 32, "tex"); 814 nir_builder_instr_insert(b, &tex_add->instr); 815 816 tmp = nir_fadd(b, tmp, &tex_add->dest.ssa); 817 } 818 819 tmp = nir_fdiv(b, tmp, nir_imm_float(b, samples)); 820 nir_store_var(b, color, tmp, 0xf); 821 nir_push_else(b, NULL); 822 inserted_if = true; 823 } 824 nir_store_var(b, color, &tex->dest.ssa, 0xf); 825 826 if (inserted_if) 827 nir_pop_if(b, NULL); 828} 829 830nir_ssa_def * 831radv_meta_load_descriptor(nir_builder *b, unsigned desc_set, unsigned binding) 832{ 833 nir_ssa_def *rsrc = nir_vulkan_resource_index(b, 3, 32, nir_imm_int(b, 0), .desc_set = desc_set, 834 .binding = binding); 835 return nir_channels(b, rsrc, 0x3); 836} 837 838nir_ssa_def * 839get_global_ids(nir_builder *b, unsigned num_components) 840{ 841 unsigned mask = BITFIELD_MASK(num_components); 842 843 nir_ssa_def *local_ids = nir_channels(b, nir_load_local_invocation_id(b), mask); 844 nir_ssa_def *block_ids = nir_channels(b, nir_load_workgroup_id(b, 32), mask); 845 nir_ssa_def *block_size = nir_channels( 846 b, 847 nir_imm_ivec4(b, b->shader->info.workgroup_size[0], b->shader->info.workgroup_size[1], 848 b->shader->info.workgroup_size[2], 0), 849 mask); 850 851 return nir_iadd(b, nir_imul(b, block_ids, block_size), local_ids); 852} 853 854void 855radv_break_on_count(nir_builder *b, nir_variable *var, nir_ssa_def *count) 856{ 857 nir_ssa_def *counter = nir_load_var(b, var); 858 859 nir_push_if(b, nir_uge(b, counter, count)); 860 nir_jump(b, nir_jump_break); 861 nir_pop_if(b, NULL); 862 863 counter = nir_iadd_imm(b, counter, 1); 864 nir_store_var(b, var, counter, 0x1); 865} 866