Lines Matching refs:pipeline
174 anv_pipeline_init(struct anv_pipeline *pipeline,
182 memset(pipeline, 0, sizeof(*pipeline));
184 vk_object_base_init(&device->vk, &pipeline->base,
186 pipeline->device = device;
191 pipeline->batch.alloc = pAllocator ? pAllocator : &device->vk.alloc;
192 pipeline->batch.relocs = &pipeline->batch_relocs;
193 pipeline->batch.status = VK_SUCCESS;
195 result = anv_reloc_list_init(&pipeline->batch_relocs,
196 pipeline->batch.alloc);
200 pipeline->mem_ctx = ralloc_context(NULL);
202 pipeline->type = type;
203 pipeline->flags = flags;
205 util_dynarray_init(&pipeline->executables, pipeline->mem_ctx);
211 anv_pipeline_finish(struct anv_pipeline *pipeline,
215 anv_reloc_list_finish(&pipeline->batch_relocs,
217 ralloc_free(pipeline->mem_ctx);
218 vk_object_base_finish(&pipeline->base);
227 ANV_FROM_HANDLE(anv_pipeline, pipeline, _pipeline);
229 if (!pipeline)
232 switch (pipeline->type) {
235 anv_pipeline_to_graphics(pipeline);
246 anv_pipeline_to_compute(pipeline);
256 anv_pipeline_to_ray_tracing(pipeline);
266 unreachable("invalid pipeline type");
269 anv_pipeline_finish(pipeline, device, pAllocator);
270 vk_free2(&device->vk.alloc, pAllocator, pipeline);
363 pipeline_has_coarse_pixel(const struct anv_graphics_pipeline *pipeline,
378 * "Sample shading is enabled for a graphics pipeline:
381 * graphics pipeline includes an input variable decorated with
387 * creating the graphics pipeline is set to VK_TRUE. In this case
401 /* Not dynamic & pipeline has a 1x1 fragment shading rate with no
402 * possibility for element of the pipeline to change the value.
435 populate_wm_prog_key(const struct anv_graphics_pipeline *pipeline,
443 const struct anv_device *device = pipeline->base.device;
491 pipeline_has_coarse_pixel(pipeline, dynamic, ms, fsr);
548 anv_pipeline_hash_graphics(struct anv_graphics_pipeline *pipeline,
556 _mesa_sha1_update(&ctx, &pipeline->view_mask,
557 sizeof(pipeline->view_mask));
562 const bool rba = pipeline->base.device->robust_buffer_access;
577 anv_pipeline_hash_compute(struct anv_compute_pipeline *pipeline,
588 const struct anv_device *device = pipeline->base.device;
604 anv_pipeline_hash_ray_tracing_shader(struct anv_ray_tracing_pipeline *pipeline,
615 const bool rba = pipeline->base.device->robust_buffer_access;
625 anv_pipeline_hash_ray_tracing_combined_shader(struct anv_ray_tracing_pipeline *pipeline,
637 const bool rba = pipeline->base.device->robust_buffer_access;
649 anv_pipeline_stage_get_nir(struct anv_pipeline *pipeline,
655 pipeline->device->physical->compiler;
660 nir = anv_device_search_for_nir(pipeline->device, cache,
669 nir = anv_shader_stage_to_nir(pipeline->device, stage->info, mem_ctx);
671 anv_device_upload_nir(pipeline->device, cache, nir, stage->shader_sha1);
691 anv_pipeline_lower_nir(struct anv_pipeline *pipeline,
696 const struct anv_physical_device *pdevice = pipeline->device->physical;
713 if (pipeline->type == ANV_PIPELINE_GRAPHICS) {
715 anv_pipeline_to_graphics(pipeline));
729 /* Apply the actual pipeline layout to UBOs, SSBOs, and textures */
731 pdevice, pipeline->device->robust_buffer_access,
736 pipeline->device->robust_buffer_access));
739 pipeline->device->robust_buffer_access));
761 pdevice, pipeline->device->robust_buffer_access,
808 struct anv_graphics_pipeline *pipeline,
814 uint32_t pos_slots = pipeline->use_primitive_replication ?
815 MAX2(1, util_bitcount(pipeline->view_mask)) : 1;
830 .log_data = pipeline->base.device,
893 * pipeline cache. So it should be safe.
1161 anv_pipeline_add_executable(struct anv_pipeline *pipeline,
1168 (pipeline->flags &
1170 nir = nir_shader_as_str(stage->nir, pipeline->mem_ctx);
1175 (pipeline->flags &
1234 intel_disassemble(&pipeline->device->physical->compiler->isa,
1240 disasm = ralloc_size(pipeline->mem_ctx, stream_size + 1);
1253 util_dynarray_append(&pipeline->executables,
1258 anv_pipeline_add_executables(struct anv_pipeline *pipeline,
1272 anv_pipeline_add_executable(pipeline, stage, stats++, 0);
1276 anv_pipeline_add_executable(pipeline, stage, stats++,
1281 anv_pipeline_add_executable(pipeline, stage, stats++,
1285 anv_pipeline_add_executable(pipeline, stage, bin->stats, 0);
1288 pipeline->ray_queries = MAX2(pipeline->ray_queries, bin->prog_data->ray_queries);
1292 anv_pipeline_init_from_cached_graphics(struct anv_graphics_pipeline *pipeline)
1294 /* TODO: Cache this pipeline-wide information. */
1296 if (anv_pipeline_is_primitive(pipeline)) {
1301 assert(pipeline->active_stages & VK_SHADER_STAGE_VERTEX_BIT);
1304 (const void *) pipeline->shaders[MESA_SHADER_VERTEX]->prog_data;
1310 pipeline->use_primitive_replication = pos_slots > 1;
1315 anv_graphics_pipeline_init_keys(struct anv_graphics_pipeline *pipeline,
1327 const struct anv_device *device = pipeline->base.device;
1331 pipeline->base.device->robust_buffer_access,
1336 pipeline->base.device->robust_buffer_access,
1342 pipeline->base.device->robust_buffer_access,
1347 pipeline->base.device->robust_buffer_access,
1351 populate_wm_prog_key(pipeline,
1352 pipeline->base.device->robust_buffer_access,
1359 pipeline->base.device->robust_buffer_access,
1364 pipeline->base.device->robust_buffer_access,
1375 assert(pipeline->active_stages & VK_SHADER_STAGE_VERTEX_BIT ||
1376 pipeline->active_stages & VK_SHADER_STAGE_MESH_BIT_NV);
1380 anv_graphics_pipeline_load_cached_shaders(struct anv_graphics_pipeline *pipeline,
1395 anv_device_search_for_kernel(pipeline->base.device, cache,
1400 pipeline->shaders[s] = bin;
1411 if (found == __builtin_popcount(pipeline->active_stages)) {
1417 for (unsigned s = 0; s < ARRAY_SIZE(pipeline->shaders); s++) {
1421 anv_pipeline_add_executables(&pipeline->base, &stages[s],
1422 pipeline->shaders[s]);
1424 anv_pipeline_init_from_cached_graphics(pipeline);
1428 * of the time but it can if we have a partially populated pipeline
1431 assert(found < __builtin_popcount(pipeline->active_stages));
1434 &pipeline->base.device->vk.base),
1435 "Found a partial pipeline in the cache. This is "
1436 "most likely caused by an incomplete pipeline cache "
1443 for (unsigned s = 0; s < ARRAY_SIZE(pipeline->shaders); s++) {
1445 if (pipeline->shaders[s]) {
1446 anv_shader_bin_unref(pipeline->base.device, pipeline->shaders[s]);
1447 pipeline->shaders[s] = NULL;
1468 anv_graphics_pipeline_load_nir(struct anv_graphics_pipeline *pipeline,
1481 assert(pipeline->shaders[s] == NULL);
1488 stages[s].nir = anv_pipeline_stage_get_nir(&pipeline->base, cache,
1492 return vk_error(pipeline, VK_ERROR_UNKNOWN);
1502 anv_graphics_pipeline_compile(struct anv_graphics_pipeline *pipeline,
1515 const struct brw_compiler *compiler = pipeline->base.device->physical->compiler;
1523 anv_graphics_pipeline_init_keys(pipeline, state, stages);
1526 anv_pipeline_hash_graphics(pipeline, layout, stages, sha1);
1537 (pipeline->base.flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR);
1540 anv_graphics_pipeline_load_cached_shaders(pipeline, cache, stages,
1551 result = anv_graphics_pipeline_load_nir(pipeline, cache, stages,
1592 if (pipeline->base.device->info.ver >= 12 &&
1593 pipeline->view_mask != 0) {
1603 pipeline->use_primitive_replication =
1604 anv_check_for_primitive_replication(shaders, pipeline);
1606 pipeline->use_primitive_replication = false;
1619 anv_pipeline_lower_nir(&pipeline->base, stage_ctx, &stages[s], layout);
1645 const struct intel_device_info *devinfo = &pipeline->base.device->info;
1681 anv_pipeline_compile_vs(compiler, stage_ctx, pipeline,
1685 anv_pipeline_compile_tcs(compiler, stage_ctx, pipeline->base.device,
1689 anv_pipeline_compile_tes(compiler, stage_ctx, pipeline->base.device,
1693 anv_pipeline_compile_gs(compiler, stage_ctx, pipeline->base.device,
1697 anv_pipeline_compile_task(compiler, stage_ctx, pipeline->base.device,
1701 anv_pipeline_compile_mesh(compiler, stage_ctx, pipeline->base.device,
1705 anv_pipeline_compile_fs(compiler, stage_ctx, pipeline->base.device,
1713 result = vk_error(pipeline->base.device, VK_ERROR_OUT_OF_HOST_MEMORY);
1721 anv_device_upload_kernel(pipeline->base.device, cache, s,
1733 result = vk_error(pipeline, VK_ERROR_OUT_OF_HOST_MEMORY);
1737 anv_pipeline_add_executables(&pipeline->base, &stages[s], bin);
1739 pipeline->shaders[s] = bin;
1771 for (unsigned s = 0; s < ARRAY_SIZE(pipeline->shaders); s++) {
1772 if (pipeline->shaders[s])
1773 anv_shader_bin_unref(pipeline->base.device, pipeline->shaders[s]);
1780 anv_pipeline_compile_cs(struct anv_compute_pipeline *pipeline,
1792 struct anv_device *device = pipeline->base.device;
1814 (pipeline->base.flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR);
1816 anv_pipeline_hash_compute(pipeline, layout, &stage, stage.cache_key.sha1);
1845 stage.nir = anv_pipeline_stage_get_nir(&pipeline->base, cache, mem_ctx, &stage);
1848 return vk_error(pipeline, VK_ERROR_UNKNOWN);
1853 anv_pipeline_lower_nir(&pipeline->base, mem_ctx, &stage, layout);
1892 return vk_error(pipeline, VK_ERROR_OUT_OF_HOST_MEMORY);
1914 return vk_error(pipeline, VK_ERROR_OUT_OF_HOST_MEMORY);
1920 anv_pipeline_add_executables(&pipeline->base, &stage, bin);
1943 pipeline->cs = bin;
1955 struct anv_compute_pipeline *pipeline;
1960 pipeline = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*pipeline), 8,
1962 if (pipeline == NULL)
1965 result = anv_pipeline_init(&pipeline->base, device,
1969 vk_free2(&device->vk.alloc, pAllocator, pipeline);
1973 anv_batch_set_storage(&pipeline->base.batch, ANV_NULL_ADDRESS,
1974 pipeline->batch_data, sizeof(pipeline->batch_data));
1976 result = anv_pipeline_compile_cs(pipeline, cache, pCreateInfo);
1978 anv_pipeline_finish(&pipeline->base, device, pAllocator);
1979 vk_free2(&device->vk.alloc, pAllocator, pipeline);
1983 anv_genX(&device->info, compute_pipeline_emit)(pipeline);
1985 *pPipeline = anv_pipeline_to_handle(&pipeline->base);
1987 return pipeline->base.batch.status;
2033 * pipeline. For now this simply returns the conservative defaults calculated
2035 * more statistics from the pipeline state (e.g. guess of expected URB usage
2039 anv_pipeline_setup_l3_config(struct anv_pipeline *pipeline, bool needs_slm)
2041 const struct intel_device_info *devinfo = &pipeline->device->info;
2046 pipeline->l3_config = intel_get_l3_config(devinfo, w);
2050 anv_graphics_pipeline_init(struct anv_graphics_pipeline *pipeline,
2059 result = anv_pipeline_init(&pipeline->base, device,
2065 anv_batch_set_storage(&pipeline->base.batch, ANV_NULL_ADDRESS,
2066 pipeline->batch_data, sizeof(pipeline->batch_data));
2068 pipeline->active_stages = 0;
2070 pipeline->active_stages |= pCreateInfo->pStages[i].stage;
2072 if (pipeline->active_stages & VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT)
2073 pipeline->active_stages |= VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT;
2075 if (anv_pipeline_is_mesh(pipeline))
2078 pipeline->dynamic_state.ms.sample_locations = &pipeline->sample_locations;
2079 vk_dynamic_graphics_state_fill(&pipeline->dynamic_state, state);
2081 pipeline->depth_clamp_enable = state->rs->depth_clamp_enable;
2082 pipeline->depth_clip_enable = state->rs->depth_clip_enable;
2083 pipeline->view_mask = state->rp->view_mask;
2085 result = anv_graphics_pipeline_compile(pipeline, cache, pCreateInfo, state);
2087 anv_pipeline_finish(&pipeline->base, device, alloc);
2091 anv_pipeline_setup_l3_config(&pipeline->base, false);
2093 if (anv_pipeline_is_primitive(pipeline)) {
2094 const uint64_t inputs_read = get_vs_prog_data(pipeline)->inputs_read;
2098 pipeline->vb_used |= BITFIELD64_BIT(state->vi->attributes[a].binding);
2102 pipeline->vb[b].stride = state->vi->bindings[b].stride;
2103 pipeline->vb[b].instanced = state->vi->bindings[b].input_rate ==
2105 pipeline->vb[b].instance_divisor = state->vi->bindings[b].divisor;
2113 pipeline->instance_multiplier = 1;
2114 if (pipeline->view_mask && !pipeline->use_primitive_replication)
2115 pipeline->instance_multiplier = util_bitcount(pipeline->view_mask);
2117 assert(anv_pipeline_is_mesh(pipeline));
2121 pipeline->negative_one_to_one =
2127 pipeline->polygon_mode = state->rs->polygon_mode;
2128 pipeline->rasterization_samples =
2130 pipeline->line_mode = state->rs->line.mode;
2131 if (pipeline->line_mode == VK_LINE_RASTERIZATION_MODE_DEFAULT_EXT) {
2132 if (pipeline->rasterization_samples > 1) {
2133 pipeline->line_mode = VK_LINE_RASTERIZATION_MODE_RECTANGULAR_EXT;
2135 pipeline->line_mode = VK_LINE_RASTERIZATION_MODE_BRESENHAM_EXT;
2138 pipeline->patch_control_points =
2146 pipeline->color_comp_writes[i] = state->cb->attachments[i].write_mask;
2159 struct anv_graphics_pipeline *pipeline;
2164 pipeline = vk_zalloc2(&device->vk.alloc, pAllocator, sizeof(*pipeline), 8,
2166 if (pipeline == NULL)
2175 vk_free2(&device->vk.alloc, pAllocator, pipeline);
2179 result = anv_graphics_pipeline_init(pipeline, device, cache,
2182 vk_free2(&device->vk.alloc, pAllocator, pipeline);
2186 anv_genX(&device->info, graphics_pipeline_emit)(pipeline, &state);
2188 *pPipeline = anv_pipeline_to_handle(&pipeline->base);
2190 return pipeline->base.batch.status;
2236 compile_upload_rt_shader(struct anv_ray_tracing_pipeline *pipeline,
2244 pipeline->base.device->physical->compiler;
2271 .log_data = pipeline->base.device,
2276 return vk_error(pipeline, VK_ERROR_OUT_OF_HOST_MEMORY);
2283 anv_device_upload_kernel(pipeline->base.device,
2293 return vk_error(pipeline, VK_ERROR_OUT_OF_HOST_MEMORY);
2296 anv_pipeline_add_executables(&pipeline->base, stage, bin);
2297 util_dynarray_append(&pipeline->shaders, struct anv_shader_bin *, bin);
2320 anv_pipeline_compute_ray_tracing_stacks(struct anv_ray_tracing_pipeline *pipeline,
2325 pipeline->stack_size = 0; /* 0 means dynamic */
2330 * pipeline is:
2340 pipeline->stack_size =
2357 if (pipeline->stack_size == 0)
2358 pipeline->stack_size = 1;
2363 anv_pipeline_init_ray_tracing_stages(struct anv_ray_tracing_pipeline *pipeline,
2393 populate_bs_prog_key(pipeline->base.device,
2394 pipeline->base.device->robust_buffer_access,
2400 anv_pipeline_hash_ray_tracing_shader(pipeline, layout, &stages[i],
2421 anv_pipeline_hash_ray_tracing_combined_shader(pipeline,
2427 anv_pipeline_hash_ray_tracing_shader(pipeline, layout,
2439 anv_pipeline_load_cached_shaders(struct anv_ray_tracing_pipeline *pipeline,
2455 stages[i].bin = anv_device_search_for_kernel(pipeline->base.device, cache,
2466 anv_pipeline_add_executables(&pipeline->base, &stages[i], stages[i].bin);
2467 util_dynarray_append(&pipeline->shaders, struct anv_shader_bin *, stages[i].bin);
2482 anv_pipeline_compile_ray_tracing(struct anv_ray_tracing_pipeline *pipeline,
2486 const struct intel_device_info *devinfo = &pipeline->base.device->info;
2497 anv_pipeline_init_ray_tracing_stages(pipeline, info, pipeline_ctx);
2502 (pipeline->base.flags & VK_PIPELINE_CREATE_CAPTURE_INTERNAL_REPRESENTATIONS_BIT_KHR);
2507 anv_pipeline_load_cached_shaders(pipeline, cache, info, stages, stack_max)) {
2524 stages[i].nir = anv_pipeline_stage_get_nir(&pipeline->base, cache,
2528 return vk_error(pipeline, VK_ERROR_OUT_OF_HOST_MEMORY);
2531 anv_pipeline_lower_nir(&pipeline->base, pipeline_ctx, &stages[i], layout);
2581 result = compile_upload_rt_shader(pipeline, cache, nir, &stages[i],
2599 struct anv_rt_shader_group *group = &pipeline->groups[i];
2637 result = compile_upload_rt_shader(pipeline, cache,
2665 anv_pipeline_compute_ray_tracing_stacks(pipeline, info, stack_max);
2814 anv_ray_tracing_pipeline_init(struct anv_ray_tracing_pipeline *pipeline,
2822 util_dynarray_init(&pipeline->shaders, pipeline->base.mem_ctx);
2824 result = anv_pipeline_compile_ray_tracing(pipeline, cache, pCreateInfo);
2828 anv_pipeline_setup_l3_config(&pipeline->base, /* needs_slm */ false);
2833 util_dynarray_foreach(&pipeline->shaders,
2867 VK_MULTIALLOC_DECL(&ma, struct anv_ray_tracing_pipeline, pipeline, 1);
2873 result = anv_pipeline_init(&pipeline->base, device,
2877 vk_free2(&device->vk.alloc, pAllocator, pipeline);
2881 pipeline->group_count = pCreateInfo->groupCount;
2882 pipeline->groups = groups;
2930 result = anv_ray_tracing_pipeline_init(pipeline, device, cache,
2933 anv_pipeline_finish(&pipeline->base, device, pAllocator);
2934 vk_free2(&device->vk.alloc, pAllocator, pipeline);
2938 anv_genX(&device->info, ray_tracing_pipeline_emit)(pipeline);
2940 *pPipeline = anv_pipeline_to_handle(&pipeline->base);
2942 return pipeline->base.batch.status;
2998 ANV_FROM_HANDLE(anv_pipeline, pipeline, pPipelineInfo->pipeline);
3002 util_dynarray_foreach (&pipeline->executables, struct anv_pipeline_executable, exe) {
3032 anv_pipeline_get_executable(struct anv_pipeline *pipeline, uint32_t index)
3034 assert(index < util_dynarray_num_elements(&pipeline->executables,
3037 &pipeline->executables, struct anv_pipeline_executable, index);
3046 ANV_FROM_HANDLE(anv_pipeline, pipeline, pExecutableInfo->pipeline);
3051 anv_pipeline_get_executable(pipeline, pExecutableInfo->executableIndex);
3054 switch (pipeline->type) {
3056 prog_data = anv_pipeline_to_graphics(pipeline)->shaders[exe->stage]->prog_data;
3060 prog_data = anv_pipeline_to_compute(pipeline)->cs->prog_data;
3065 util_dynarray_element(&anv_pipeline_to_ray_tracing(pipeline)->shaders,
3072 unreachable("invalid pipeline type");
3187 ANV_FROM_HANDLE(anv_pipeline, pipeline, pExecutableInfo->pipeline);
3193 anv_pipeline_get_executable(pipeline, pExecutableInfo->executableIndex);
3230 ANV_FROM_HANDLE(anv_pipeline, pipeline, _pipeline);
3232 if (pipeline->type != ANV_PIPELINE_RAY_TRACING)
3236 anv_pipeline_to_ray_tracing(pipeline);
3250 VkPipeline pipeline,
3268 ANV_FROM_HANDLE(anv_pipeline, pipeline, _pipeline);
3269 assert(pipeline->type == ANV_PIPELINE_RAY_TRACING);
3272 anv_pipeline_to_ray_tracing(pipeline);