Lines Matching defs:nir
28 #include "nir/nir.h"
29 #include "nir/nir_builder.h"
2274 bool has_tess = !!stages[MESA_SHADER_TESS_CTRL].nir;
2405 if (stages[MESA_SHADER_GEOMETRY].nir) {
2406 nir_shader *gs = stages[MESA_SHADER_GEOMETRY].nir;
2411 if (stages[MESA_SHADER_TESS_CTRL].nir) {
2412 nir_shader *tes = stages[MESA_SHADER_TESS_EVAL].nir;
2462 nir_shader *ms = stage->nir;
2491 stages[MESA_SHADER_TESS_CTRL].nir ? &gs_info->tes.es_info : &gs_info->vs.es_info;
2492 unsigned gs_type = stages[MESA_SHADER_GEOMETRY].nir ? MESA_SHADER_GEOMETRY : MESA_SHADER_VERTEX;
2495 unsigned gs_num_invocations = stages[MESA_SHADER_GEOMETRY].nir ? MAX2(gs_info->gs.invocations, 1) : 1;
2559 struct radv_streamout_info *so_info = stages[MESA_SHADER_TESS_CTRL].nir
2570 if (!stages[MESA_SHADER_TESS_CTRL].nir && stages[MESA_SHADER_VERTEX].info.vs.outinfo.export_prim_id)
2614 wavesize = stages[MESA_SHADER_TESS_CTRL].nir ? stages[MESA_SHADER_TESS_EVAL].info.wave_size
2706 gl_shader_stage es_stage = stages[MESA_SHADER_TESS_CTRL].nir ? MESA_SHADER_TESS_EVAL : MESA_SHADER_VERTEX;
2786 radv_lower_viewport_to_zero(nir_shader *nir)
2788 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
2828 find_layer_out_var(nir_shader *nir)
2830 nir_variable *var = nir_find_variable_with_location(nir, nir_var_shader_out, VARYING_SLOT_LAYER);
2834 var = nir_variable_create(nir, nir_var_shader_out, glsl_int_type(), "layer id");
2842 radv_lower_multiview(nir_shader *nir)
2848 if (nir->info.stage == MESA_SHADER_MESH)
2851 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
2867 if (nir->info.stage == MESA_SHADER_GEOMETRY) {
2886 layer = find_layer_out_var(nir);
2891 nir->info.outputs_written |= BITFIELD64_BIT(VARYING_SLOT_LAYER);
2894 if (nir->info.stage == MESA_SHADER_VERTEX)
2897 if (nir->info.stage == MESA_SHADER_VERTEX && progress)
2910 radv_export_implicit_primitive_id(nir_shader *nir)
2912 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
2918 nir_variable *var = nir_variable_create(nir, nir_var_shader_out, glsl_int_type(), NULL);
2925 nir->info.outputs_written |= BITFIELD64_BIT(VARYING_SLOT_PRIMITIVE_ID);
2943 if (stages[MESA_SHADER_FRAGMENT].nir) {
2944 ordered_shaders[shader_count++] = stages[MESA_SHADER_FRAGMENT].nir;
2946 if (stages[MESA_SHADER_GEOMETRY].nir) {
2947 ordered_shaders[shader_count++] = stages[MESA_SHADER_GEOMETRY].nir;
2949 if (stages[MESA_SHADER_TESS_EVAL].nir) {
2950 ordered_shaders[shader_count++] = stages[MESA_SHADER_TESS_EVAL].nir;
2952 if (stages[MESA_SHADER_TESS_CTRL].nir) {
2953 ordered_shaders[shader_count++] = stages[MESA_SHADER_TESS_CTRL].nir;
2955 if (stages[MESA_SHADER_VERTEX].nir) {
2956 ordered_shaders[shader_count++] = stages[MESA_SHADER_VERTEX].nir;
2958 if (stages[MESA_SHADER_MESH].nir) {
2959 ordered_shaders[shader_count++] = stages[MESA_SHADER_MESH].nir;
2961 if (stages[MESA_SHADER_TASK].nir) {
2962 ordered_shaders[shader_count++] = stages[MESA_SHADER_TASK].nir;
2964 if (stages[MESA_SHADER_COMPUTE].nir) {
2965 ordered_shaders[shader_count++] = stages[MESA_SHADER_COMPUTE].nir;
2968 if (stages[MESA_SHADER_MESH].nir && stages[MESA_SHADER_FRAGMENT].nir) {
2969 nir_shader *ps = stages[MESA_SHADER_FRAGMENT].nir;
2980 bool has_geom_tess = stages[MESA_SHADER_GEOMETRY].nir || stages[MESA_SHADER_TESS_CTRL].nir;
2981 bool merged_gs = stages[MESA_SHADER_GEOMETRY].nir && pdevice->rad_info.gfx_level >= GFX9;
3043 if (stages[MESA_SHADER_FRAGMENT].nir &&
3044 (stages[MESA_SHADER_FRAGMENT].nir->info.inputs_read & VARYING_BIT_PRIMITIVE_ID) &&
3045 !(stages[last_vgt_api_stage].nir->info.outputs_written & VARYING_BIT_PRIMITIVE_ID) &&
3048 radv_export_implicit_primitive_id(stages[last_vgt_api_stage].nir);
3053 stages[last_vgt_api_stage].nir->xfb_info;
3122 if (stages[MESA_SHADER_FRAGMENT].nir &&
3123 (stages[MESA_SHADER_FRAGMENT].nir->info.inputs_read & VARYING_BIT_VIEWPORT) &&
3124 !(stages[last_vgt_api_stage].nir->info.outputs_written & VARYING_BIT_VIEWPORT)) {
3125 NIR_PASS(_, stages[MESA_SHADER_FRAGMENT].nir, radv_lower_viewport_to_zero);
3130 !(stages[last_vgt_api_stage].nir->info.outputs_written &
3132 nir_shader *last_vgt_shader = stages[last_vgt_api_stage].nir;
3199 if (stages[MESA_SHADER_FRAGMENT].nir) {
3200 nir_foreach_shader_out_variable(var, stages[MESA_SHADER_FRAGMENT].nir)
3206 if (stages[MESA_SHADER_MESH].nir) {
3210 nir_foreach_shader_out_variable(var, stages[MESA_SHADER_MESH].nir) {
3216 if (!stages[MESA_SHADER_VERTEX].nir)
3219 bool has_tess = stages[MESA_SHADER_TESS_CTRL].nir;
3220 bool has_gs = stages[MESA_SHADER_GEOMETRY].nir;
3232 nir_foreach_shader_in_variable (var, stages[MESA_SHADER_VERTEX].nir) {
3238 stages[MESA_SHADER_VERTEX].nir, stages[MESA_SHADER_TESS_CTRL].nir);
3240 stages[MESA_SHADER_TESS_CTRL].nir, stages[MESA_SHADER_TESS_EVAL].nir);
3256 stages[MESA_SHADER_TESS_EVAL].nir, stages[MESA_SHADER_GEOMETRY].nir);
3266 stages[MESA_SHADER_VERTEX].nir, stages[MESA_SHADER_GEOMETRY].nir);
3276 nir_foreach_shader_out_variable(var, stages[last_vgt_api_stage].nir)
3427 stages[MESA_SHADER_FRAGMENT].nir ? stages[MESA_SHADER_FRAGMENT].nir->info.inputs_read : 0;
3432 num_vertices_per_prim = stages[es_stage].nir->info.tess.point_mode ? 1
3433 : stages[es_stage].nir->info.tess._primitive_mode == TESS_PRIMITIVE_ISOLINES ? 2
3437 pdevice, stages[es_stage].nir, ps_inputs_read, num_vertices_per_prim, &stages[es_stage].info) &&
3440 nir_function_impl *impl = nir_shader_get_entrypoint(stages[es_stage].nir);
3472 if (stages[MESA_SHADER_TESS_CTRL].nir) {
3474 } else if (stages[MESA_SHADER_VERTEX].nir) {
3476 } else if (stages[MESA_SHADER_MESH].nir) {
3480 if (stages[MESA_SHADER_TESS_CTRL].nir && stages[MESA_SHADER_GEOMETRY].nir &&
3481 stages[MESA_SHADER_GEOMETRY].nir->info.gs.invocations *
3482 stages[MESA_SHADER_GEOMETRY].nir->info.gs.vertices_out >
3498 if (stages[i].nir)
3502 bool uses_xfb = stages[last_xfb_stage].nir &&
3503 stages[last_xfb_stage].nir->xfb_info;
3509 if (stages[MESA_SHADER_TESS_CTRL].nir)
3520 if (!stages[MESA_SHADER_GEOMETRY].nir && !uses_xfb) {
3521 if (stages[MESA_SHADER_TESS_CTRL].nir && stages[MESA_SHADER_TESS_EVAL].info.is_ngg) {
3523 } else if (stages[MESA_SHADER_VERTEX].nir && stages[MESA_SHADER_VERTEX].info.is_ngg) {
3542 if (stages[i].nir)
3546 if (stages[MESA_SHADER_TESS_CTRL].nir) {
3550 if (stages[MESA_SHADER_GEOMETRY].nir) {
3551 if (stages[MESA_SHADER_TESS_CTRL].nir)
3557 if (stages[MESA_SHADER_FRAGMENT].nir) {
3559 radv_nir_shader_info_pass(device, stages[MESA_SHADER_FRAGMENT].nir, pipeline_layout,
3589 stages[MESA_SHADER_TESS_CTRL].nir) {
3590 struct nir_shader *combined_nir[] = {stages[MESA_SHADER_VERTEX].nir, stages[MESA_SHADER_TESS_CTRL].nir};
3607 stages[MESA_SHADER_GEOMETRY].nir) {
3609 stages[MESA_SHADER_TESS_EVAL].nir ? MESA_SHADER_TESS_EVAL : MESA_SHADER_VERTEX;
3610 struct nir_shader *combined_nir[] = {stages[pre_stage].nir, stages[MESA_SHADER_GEOMETRY].nir};
3636 radv_nir_shader_info_pass(device, stages[i].nir, pipeline_layout, pipeline_key,
3640 if (stages[MESA_SHADER_COMPUTE].nir) {
3648 unsigned local_size = stages[MESA_SHADER_COMPUTE].nir->info.workgroup_size[0] *
3649 stages[MESA_SHADER_COMPUTE].nir->info.workgroup_size[1] *
3650 stages[MESA_SHADER_COMPUTE].nir->info.workgroup_size[2];
3656 stages[MESA_SHADER_COMPUTE].nir->info.cs.uses_wide_subgroup_intrinsics && !req_subgroup_size &&
3669 if (stages[i].nir) {
3676 if (stages[MESA_SHADER_FRAGMENT].nir)
3679 if (stages[MESA_SHADER_COMPUTE].nir) {
3681 assert(!stages[MESA_SHADER_COMPUTE].nir->info.workgroup_size_variable);
3685 stages[MESA_SHADER_COMPUTE].nir->info.workgroup_size, false, UINT32_MAX);
3688 if (stages[MESA_SHADER_TASK].nir) {
3695 stages[MESA_SHADER_TASK].nir->info.workgroup_size, false, UINT32_MAX);
3707 if (stages[i].nir)
3718 if (gfx_level >= GFX9 && stages[MESA_SHADER_TESS_CTRL].nir) {
3731 if (gfx_level >= GFX9 && stages[MESA_SHADER_GEOMETRY].nir) {
3733 stages[MESA_SHADER_TESS_EVAL].nir ? MESA_SHADER_TESS_EVAL : MESA_SHADER_VERTEX;
3802 merge_tess_info(&stages[MESA_SHADER_TESS_EVAL].nir->info,
3803 &stages[MESA_SHADER_TESS_CTRL].nir->info);
3806 unsigned tess_out_patch_size = stages[MESA_SHADER_TESS_CTRL].nir->info.tess.tcs_vertices_out;
3827 !!(stages[MESA_SHADER_TESS_EVAL].nir->info.inputs_read &
3829 stages[MESA_SHADER_TESS_CTRL].info.tcs.tes_inputs_read = stages[MESA_SHADER_TESS_EVAL].nir->info.inputs_read;
3831 stages[MESA_SHADER_TESS_EVAL].nir->info.patch_inputs_read;
3852 stages[MESA_SHADER_VERTEX].nir->info.float_controls_execution_mode ==
3853 stages[MESA_SHADER_TESS_CTRL].nir->info.float_controls_execution_mode;
3857 stages[MESA_SHADER_TESS_CTRL].nir->info.inputs_read &
3858 stages[MESA_SHADER_VERTEX].nir->info.outputs_written &
3859 ~stages[MESA_SHADER_TESS_CTRL].nir->info.tess.tcs_cross_invocation_inputs_read &
3860 ~stages[MESA_SHADER_TESS_CTRL].nir->info.inputs_read_indirectly &
3861 ~stages[MESA_SHADER_VERTEX].nir->info.outputs_accessed_indirectly;
4137 nir_shader *last_vgt_shader = stages[last_vgt_api_stage].nir;
4146 nir_shader *fs_shader = stages[MESA_SHADER_FRAGMENT].nir;
4184 radv_lower_vs_input(nir_shader *nir, const struct radv_pipeline_key *pipeline_key)
4186 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
4274 radv_lower_fs_output(nir_shader *nir, const struct radv_pipeline_key *pipeline_key)
4276 nir_function_impl *impl = nir_shader_get_entrypoint(nir);
4318 if (enable_mrt_output_nan_fixup && !nir->info.internal && !is_16bit) {
4464 if (module->nir)
4465 out_stage->internal_nir = module->nir;
4488 radv_nir_shader_info_pass(device, stages[MESA_SHADER_GEOMETRY].nir, pipeline_layout, pipeline_key,
4502 return radv_create_gs_copy_shader(device, stages[MESA_SHADER_GEOMETRY].nir, &info, &gs_copy_args,
4520 if (stages[i].nir)
4527 if (stages[MESA_SHADER_GEOMETRY].nir && !pipeline_has_ngg) {
4538 nir_shader *shaders[2] = { stages[s].nir, NULL };
4546 if (s == MESA_SHADER_GEOMETRY && stages[MESA_SHADER_TESS_EVAL].nir) {
4552 shaders[0] = stages[pre_stage].nir;
4553 shaders[1] = stages[s].nir;
4678 stages[s].nir = radv_shader_spirv_to_nir(device, &stages[s], pipeline_key);
4688 nir_shader *last_vgt_shader = stages[*last_vgt_api_stage].nir;
4697 bool pipeline_has_ngg = (stages[MESA_SHADER_VERTEX].nir && stages[MESA_SHADER_VERTEX].info.is_ngg) ||
4698 (stages[MESA_SHADER_TESS_EVAL].nir && stages[MESA_SHADER_TESS_EVAL].info.is_ngg) ||
4699 (stages[MESA_SHADER_MESH].nir && stages[MESA_SHADER_MESH].info.is_ngg);
4701 if (stages[MESA_SHADER_GEOMETRY].nir) {
4710 NIR_PASS(_, stages[MESA_SHADER_GEOMETRY].nir, nir_lower_gs_intrinsics, nir_gs_flags);
4717 if (stages[i].nir) {
4720 radv_optimize_nir(stages[i].nir, optimize_conservatively, false);
4723 nir_shader_gather_info(stages[i].nir, nir_shader_get_entrypoint(stages[i].nir));
4724 radv_lower_io(device, stages[i].nir, stages[MESA_SHADER_MESH].nir);
4730 if (stages[MESA_SHADER_TESS_CTRL].nir) {
4731 nir_lower_patch_vertices(stages[MESA_SHADER_TESS_EVAL].nir,
4732 stages[MESA_SHADER_TESS_CTRL].nir->info.tess.tcs_vertices_out, NULL);
4736 if (stages[MESA_SHADER_VERTEX].nir) {
4737 NIR_PASS(_, stages[MESA_SHADER_VERTEX].nir, radv_lower_vs_input, pipeline_key);
4740 if (stages[MESA_SHADER_FRAGMENT].nir && !radv_use_llvm_for_stage(device, MESA_SHADER_FRAGMENT)) {
4742 NIR_PASS(_, stages[MESA_SHADER_FRAGMENT].nir, radv_lower_fs_output, pipeline_key);
4750 if (stages[MESA_SHADER_GEOMETRY].nir)
4752 else if (stages[MESA_SHADER_TESS_CTRL].nir)
4754 else if (stages[MESA_SHADER_VERTEX].nir)
4756 else if (stages[MESA_SHADER_MESH].nir)
4765 } else if (stages[MESA_SHADER_GEOMETRY].nir) {
4771 stages[MESA_SHADER_TESS_EVAL].nir ? MESA_SHADER_TESS_EVAL : MESA_SHADER_VERTEX;
4779 if (stages[MESA_SHADER_FRAGMENT].nir) {
4780 NIR_PASS(_, stages[MESA_SHADER_FRAGMENT].nir, radv_lower_fs_intrinsics,
4785 if (stages[i].nir) {
4798 NIR_PASS(_, stages[i].nir, nir_lower_non_uniform_access, &options);
4800 NIR_PASS(_, stages[i].nir, nir_lower_memory_model);
4819 NIR_PASS(progress, stages[i].nir, nir_opt_load_store_vectorize, &vectorize_opts);
4821 NIR_PASS(_, stages[i].nir, nir_copy_prop);
4822 NIR_PASS(_, stages[i].nir, nir_opt_shrink_stores,
4826 nir_shader_gather_info(stages[i].nir, nir_shader_get_entrypoint(stages[i].nir));
4831 if (i == MESA_SHADER_VERTEX && stages[MESA_SHADER_TESS_CTRL].nir)
4833 else if (i == MESA_SHADER_VERTEX && stages[MESA_SHADER_GEOMETRY].nir)
4835 else if (i == MESA_SHADER_TESS_EVAL && stages[MESA_SHADER_GEOMETRY].nir)
4838 NIR_PASS(_, stages[i].nir, radv_nir_lower_ycbcr_textures, pipeline_layout);
4839 NIR_PASS_V(stages[i].nir, radv_nir_apply_pipeline_layout, device, pipeline_layout, info,
4842 NIR_PASS(_, stages[i].nir, nir_opt_shrink_vectors);
4844 NIR_PASS(_, stages[i].nir, nir_lower_alu_width, opt_vectorize_callback, device);
4847 NIR_PASS(_, stages[i].nir, nir_lower_int64);
4849 NIR_PASS(_, stages[i].nir, nir_opt_idiv_const, 8);
4851 NIR_PASS(_, stages[i].nir, nir_lower_idiv,
4861 NIR_PASS(_, stages[i].nir, nir_opt_sink, sink_opts);
4862 NIR_PASS(_, stages[i].nir, nir_opt_move,
4871 NIR_PASS(_, stages[i].nir, ac_nir_lower_global_access);
4872 NIR_PASS_V(stages[i].nir, radv_nir_lower_abi, device->physical_device->rad_info.gfx_level,
4876 stages[i].nir, io_to_mem || lowered_ngg || i == MESA_SHADER_COMPUTE || i == MESA_SHADER_TASK);
4878 if (stages[i].nir->info.bit_sizes_int & (8 | 16)) {
4880 NIR_PASS(_, stages[i].nir, nir_convert_to_lcssa, true, true);
4881 nir_divergence_analysis(stages[i].nir);
4884 if (nir_lower_bit_size(stages[i].nir, lower_bit_size_callback, device)) {
4885 NIR_PASS(_, stages[i].nir, nir_opt_constant_folding);
4889 NIR_PASS(_, stages[i].nir, nir_opt_remove_phis); /* cleanup LCSSA phis */
4891 if (((stages[i].nir->info.bit_sizes_int | stages[i].nir->info.bit_sizes_float) & 16) &&
4915 NIR_PASS(_, stages[i].nir, nir_fold_16bit_tex_image, &fold_16bit_options);
4917 NIR_PASS(_, stages[i].nir, nir_opt_vectorize, opt_vectorize_callback, device);
4921 NIR_PASS(_, stages[i].nir, nir_lower_alu_width, opt_vectorize_callback, device);
4922 NIR_PASS(_, stages[i].nir, nir_lower_load_const_to_scalar);
4923 NIR_PASS(_, stages[i].nir, nir_copy_prop);
4924 NIR_PASS(_, stages[i].nir, nir_opt_dce);
4927 NIR_PASS(_, stages[i].nir, nir_opt_sink, sink_opts);
4931 NIR_PASS(_, stages[i].nir, nir_opt_move, move_opts);
4938 if (stages[i].nir) {
4939 if (radv_can_dump_shader(device, stages[i].nir, false))
4940 nir_print_shader(stages[i].nir, stderr);
4987 if (stages[i].nir) {
4988 if (radv_can_dump_shader_stats(device, stages[i].nir) && pipeline->shaders[i]) {
4992 ralloc_free(stages[i].nir);