Lines Matching refs:ctx
73 create_empty_block(compiler_context *ctx)
75 midgard_block *blk = rzalloc(ctx, midgard_block);
81 blk->base.name = ctx->block_source_count++;
87 schedule_barrier(compiler_context *ctx)
89 midgard_block *temp = ctx->after_block;
90 ctx->after_block = create_empty_block(ctx);
91 ctx->block_count++;
92 list_addtail(&ctx->after_block->base.link, &ctx->blocks);
93 list_inithead(&ctx->after_block->base.instructions);
94 pan_block_add_successor(&ctx->current_block->base, &ctx->after_block->base);
95 ctx->current_block = ctx->after_block;
96 ctx->after_block = temp;
102 #define EMIT(op, ...) emit_mir_instruction(ctx, v_##op(__VA_ARGS__));
205 attach_constants(compiler_context *ctx, midgard_instruction *ins, void *constants, int name)
473 emit_load_const(compiler_context *ctx, nir_load_const_instr *instr)
477 midgard_constants *consts = rzalloc(ctx, midgard_constants);
503 _mesa_hash_table_u64_insert(ctx->ssa_constants, (def.index << 1) + 1, consts);
510 emit_explicit_constant(compiler_context *ctx, unsigned node, unsigned to)
512 void *constant_value = _mesa_hash_table_u64_search(ctx->ssa_constants, node + 1);
516 attach_constants(ctx, &ins, constant_value, node + 1);
517 emit_mir_instruction(ctx, ins);
534 #define ATOMIC_CASE_IMPL(ctx, instr, nir, op, is_shared) \
536 emit_atomic(ctx, instr, is_shared, midgard_op_##op, ~0); \
539 #define ATOMIC_CASE(ctx, instr, nir, op) \
540 ATOMIC_CASE_IMPL(ctx, instr, shared_atomic_##nir, atomic_##op, true); \
541 ATOMIC_CASE_IMPL(ctx, instr, global_atomic_##nir, atomic_##op, false);
543 #define IMAGE_ATOMIC_CASE(ctx, instr, nir, op) \
545 midgard_instruction ins = emit_image_op(ctx, instr, true); \
546 emit_atomic(ctx, instr, false, midgard_op_atomic_##op, ins.dest); \
597 mir_accept_dest_mod(compiler_context *ctx, nir_dest **dest, nir_op op)
601 BITSET_SET(ctx->already_emitted, (*dest)->ssa.index);
622 mir_determine_float_outmod(compiler_context *ctx, nir_dest **dest, unsigned prior_outmod)
624 bool clamp_0_inf = mir_accept_dest_mod(ctx, dest, nir_op_fclamp_pos_mali);
625 bool clamp_0_1 = mir_accept_dest_mod(ctx, dest, nir_op_fsat);
626 bool clamp_m1_1 = mir_accept_dest_mod(ctx, dest, nir_op_fsat_signed_mali);
730 emit_alu(compiler_context *ctx, nir_alu_instr *instr)
734 if (dest->is_ssa && BITSET_TEST(ctx->already_emitted, dest->ssa.index))
741 midgard_emit_derivatives(ctx, instr);
989 outmod = mir_determine_float_outmod(ctx, &dest, outmod);
1156 emit_mir_instruction(ctx, ins_split[i]);
1159 emit_mir_instruction(ctx, ins);
1194 compiler_context *ctx,
1230 ins.src[2] = nir_src_index(ctx, indirect_offset);
1247 return emit_mir_instruction(ctx, ins);
1255 compiler_context *ctx,
1329 mir_set_offset(ctx, &ins, offset, seg);
1340 emit_mir_instruction(ctx, ins);
1349 compiler_context *ctx,
1363 unsigned val = nir_src_index(ctx, &instr->src[val_src]);
1365 emit_explicit_constant(ctx, val, val);
1380 unsigned xchg_val = nir_src_index(ctx, &instr->src[xchg_val_src]);
1381 emit_explicit_constant(ctx, xchg_val, xchg_val);
1396 nir_src_index(ctx, src_offset);
1410 mir_set_offset(ctx, &ins, src_offset, is_shared ? LDST_SHARED : LDST_GLOBAL);
1414 emit_mir_instruction(ctx, ins);
1419 compiler_context *ctx,
1448 ins.src[2] = nir_src_index(ctx, indirect_offset);
1476 emit_mir_instruction(ctx, ins);
1483 emit_image_op(compiler_context *ctx, nir_intrinsic_instr *instr, bool is_atomic)
1486 unsigned nr_attr = ctx->stage == MESA_SHADER_VERTEX ?
1487 util_bitcount64(ctx->nir->info.inputs_read) : 0;
1495 unsigned coord_reg = nir_src_index(ctx, &instr->src[1]);
1496 emit_explicit_constant(ctx, coord_reg, coord_reg);
1508 unsigned val = nir_src_index(ctx, &instr->src[3]);
1509 emit_explicit_constant(ctx, val, val);
1516 unsigned dest = make_compiler_temp_reg(ctx);
1535 ins.src[2] = nir_src_index(ctx, index);
1540 emit_mir_instruction(ctx, ins);
1547 compiler_context *ctx,
1573 emit_mir_instruction(ctx, ins);
1577 emit_sysval_read(compiler_context *ctx, nir_instr *instr,
1583 unsigned sysval_ubo = ctx->inputs->fixed_sysval_ubo >= 0 ?
1584 ctx->inputs->fixed_sysval_ubo :
1585 ctx->nir->info.num_ubos;
1589 pan_lookup_sysval(ctx->sysval_to_id, &ctx->info->sysvals, sysval);
1593 emit_ubo_read(ctx, instr, dest, (uniform * 16) + offset, NULL, 0,
1616 emit_fragment_store(compiler_context *ctx, unsigned src, unsigned src_z, unsigned src_s,
1619 assert(rt < ARRAY_SIZE(ctx->writeout_branch));
1620 assert(sample_iter < ARRAY_SIZE(ctx->writeout_branch[0]));
1622 midgard_instruction *br = ctx->writeout_branch[rt][sample_iter];
1626 emit_explicit_constant(ctx, src, src);
1648 emit_explicit_constant(ctx, src_z, src_z);
1654 emit_explicit_constant(ctx, src_s, src_s);
1661 br = emit_mir_instruction(ctx, ins);
1662 schedule_barrier(ctx);
1663 ctx->writeout_branch[rt][sample_iter] = br;
1668 br->branch.target_block = ctx->block_count - 1;
1672 emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr)
1679 emit_mir_instruction(ctx, ins);
1696 emit_vertex_builtin(compiler_context *ctx, nir_intrinsic_instr *instr)
1699 emit_attr_read(ctx, reg, vertex_builtin_arg(instr->intrinsic), 1, nir_type_int);
1703 emit_special(compiler_context *ctx, nir_intrinsic_instr *instr, unsigned idx)
1715 emit_mir_instruction(ctx, ld);
1719 emit_control_barrier(compiler_context *ctx)
1728 emit_mir_instruction(ctx, ins);
1745 output_load_rt_addr(compiler_context *ctx, nir_intrinsic_instr *instr)
1747 if (ctx->inputs->is_blend)
1748 return MIDGARD_COLOR_RT0 + ctx->inputs->blend.rt;
1751 var = nir_find_variable_with_driver_location(ctx->nir, nir_var_shader_out, nir_intrinsic_base(instr));
1768 emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr)
1785 emit_mir_instruction(ctx, discard);
1786 schedule_barrier(ctx);
1793 emit_image_op(ctx, instr, false);
1798 emit_sysval_read(ctx, &instr->instr, nr_comp, 0);
1848 emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0, 0, nr_comp);
1856 emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0, uindex, nr_comp);
1859 emit_global(ctx, &instr->instr, true, reg, src_offset, seg);
1860 } else if (ctx->stage == MESA_SHADER_FRAGMENT && !ctx->inputs->is_blend) {
1861 emit_varying_read(ctx, reg, offset, nr_comp, component, indirect_offset, t | nir_dest_bit_size(instr->dest), is_flat);
1862 } else if (ctx->inputs->is_blend) {
1863 /* ctx->blend_input will be precoloured to r0/r2, where
1866 unsigned *input = offset ? &ctx->blend_src1 : &ctx->blend_input;
1871 emit_mir_instruction(ctx, v_mov(*input, reg));
1872 } else if (ctx->stage == MESA_SHADER_VERTEX) {
1873 emit_attr_read(ctx, reg, offset, nr_comp, t);
1898 unsigned target = output_load_rt_addr(ctx, instr);
1909 ld.src[1] = nir_src_index(ctx, &instr->src[0]);
1913 if (ctx->quirks & MIDGARD_OLD_BLEND) {
1919 emit_mir_instruction(ctx, ld);
1934 unsigned index = output_load_rt_addr(ctx, instr);
1941 if (ctx->quirks & MIDGARD_OLD_BLEND) {
1950 emit_mir_instruction(ctx, ld);
1958 reg = nir_src_index(ctx, &instr->src[0]);
1960 if (ctx->stage == MESA_SHADER_FRAGMENT) {
1971 reg_z = nir_src_index(ctx, &instr->src[2]);
1973 reg_s = nir_src_index(ctx, &instr->src[3]);
1975 reg_2 = nir_src_index(ctx, &instr->src[4]);
1980 nir_find_variable_with_driver_location(ctx->nir, nir_var_shader_out,
1996 emit_explicit_constant(ctx, reg_2, reg_2);
1998 unsigned out = make_compiler_temp(ctx);
2001 emit_mir_instruction(ctx, ins);
2003 ctx->blend_src1 = out;
2005 ctx->blend_src1 = reg_2;
2009 emit_fragment_store(ctx, reg, reg_z, reg_s, rt, 0);
2010 } else if (ctx->stage == MESA_SHADER_VERTEX) {
2021 emit_explicit_constant(ctx, reg, reg);
2064 emit_mir_instruction(ctx, st);
2074 assert (ctx->stage == MESA_SHADER_FRAGMENT);
2075 reg = nir_src_index(ctx, &instr->src[0]);
2076 for (unsigned s = 0; s < ctx->blend_sample_iterations; s++)
2077 emit_fragment_store(ctx, reg, ~0, ~0,
2078 ctx->inputs->blend.rt + MIDGARD_COLOR_RT0,
2085 reg = nir_src_index(ctx, &instr->src[0]);
2086 emit_explicit_constant(ctx, reg, reg);
2096 emit_global(ctx, &instr->instr, false, reg, &instr->src[1], seg);
2101 emit_sysval_read(ctx, &instr->instr, 2, 0);
2107 emit_sysval_read(ctx, &instr->instr, 1, 0);
2111 emit_sysval_read(ctx, &instr->instr, 1, 4);
2116 emit_sysval_read(ctx, &instr->instr, 1, 8);
2120 emit_sysval_read(ctx, &instr->instr, 2, 0);
2128 emit_sysval_read(ctx, &instr->instr, 3, 0);
2132 emit_sysval_read(ctx, &instr->instr, 4, 0);
2139 emit_compute_builtin(ctx, instr);
2144 emit_vertex_builtin(ctx, instr);
2148 emit_special(ctx, instr, 96);
2152 emit_special(ctx, instr, 97);
2164 schedule_barrier(ctx);
2165 emit_control_barrier(ctx);
2166 schedule_barrier(ctx);
2169 ATOMIC_CASE(ctx, instr, add, add);
2170 ATOMIC_CASE(ctx, instr, and, and);
2171 ATOMIC_CASE(ctx, instr, comp_swap, cmpxchg);
2172 ATOMIC_CASE(ctx, instr, exchange, xchg);
2173 ATOMIC_CASE(ctx, instr, imax, imax);
2174 ATOMIC_CASE(ctx, instr, imin, imin);
2175 ATOMIC_CASE(ctx, instr, or, or);
2176 ATOMIC_CASE(ctx, instr, umax, umax);
2177 ATOMIC_CASE(ctx, instr, umin, umin);
2178 ATOMIC_CASE(ctx, instr, xor, xor);
2180 IMAGE_ATOMIC_CASE(ctx, instr, add, add);
2181 IMAGE_ATOMIC_CASE(ctx, instr, and, and);
2182 IMAGE_ATOMIC_CASE(ctx, instr, comp_swap, cmpxchg);
2183 IMAGE_ATOMIC_CASE(ctx, instr, exchange, xchg);
2184 IMAGE_ATOMIC_CASE(ctx, instr, imax, imax);
2185 IMAGE_ATOMIC_CASE(ctx, instr, imin, imin);
2186 IMAGE_ATOMIC_CASE(ctx, instr, or, or);
2187 IMAGE_ATOMIC_CASE(ctx, instr, umax, umax);
2188 IMAGE_ATOMIC_CASE(ctx, instr, umin, umin);
2189 IMAGE_ATOMIC_CASE(ctx, instr, xor, xor);
2231 compiler_context *ctx,
2273 set_tex_coord(compiler_context *ctx, nir_tex_instr *instr,
2285 unsigned coords = nir_src_index(ctx, &instr->src[coord_idx].src);
2287 emit_explicit_constant(ctx, coords, coords);
2325 ins->src[1] = make_compiler_temp_reg(ctx);
2336 emit_mir_instruction(ctx, ld);
2356 nir_src_index(ctx, &instr->src[ms_or_comparator_idx].src);
2358 emit_explicit_constant(ctx, sample_or_ref, sample_or_ref);
2361 ins->src[1] = make_compiler_temp_reg(ctx);
2371 emit_mir_instruction(ctx, mov);
2381 ins->src[1] = make_compiler_temp_reg(ctx);
2388 emit_mir_instruction(ctx, mov);
2412 emit_mir_instruction(ctx, mov);
2417 emit_texop_native(compiler_context *ctx, nir_tex_instr *instr,
2430 outmod = mir_determine_float_outmod(ctx, &dest, 0);
2455 int index = nir_src_index(ctx, &instr->src[i].src);
2461 set_tex_coord(ctx, instr, &ins);
2469 if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture))
2479 emit_explicit_constant(ctx, index, index);
2492 emit_explicit_constant(ctx, index, index);
2508 emit_mir_instruction(ctx, ins);
2512 emit_tex(compiler_context *ctx, nir_tex_instr *instr)
2517 emit_texop_native(ctx, instr, midgard_tex_op_normal);
2521 emit_texop_native(ctx, instr, midgard_tex_op_gradient);
2525 emit_texop_native(ctx, instr, midgard_tex_op_fetch);
2528 emit_sysval_read(ctx, &instr->instr, 4, 0);
2538 emit_jump(compiler_context *ctx, nir_jump_instr *instr)
2545 br.branch.target_break = ctx->current_loop_depth;
2546 emit_mir_instruction(ctx, br);
2556 emit_instr(compiler_context *ctx, struct nir_instr *instr)
2560 emit_load_const(ctx, nir_instr_as_load_const(instr));
2564 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
2568 emit_alu(ctx, nir_instr_as_alu(instr));
2572 emit_tex(ctx, nir_instr_as_tex(instr));
2576 emit_jump(ctx, nir_instr_as_jump(instr));
2594 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[idx] + 1); \
2597 attach_constants(ctx, alu, entry, alu->src[idx] + 1); \
2603 inline_alu_constants(compiler_context *ctx, midgard_block *block)
2623 void *entry = _mesa_hash_table_u64_search(ctx->ssa_constants, alu->src[1] + 1);
2624 unsigned scratch = make_compiler_temp(ctx);
2628 attach_constants(ctx, &ins, entry, alu->src[1] + 1);
2634 mir_insert_instruction_before(ctx, mir_prev_op(alu), ins);
2708 embedded_to_inline_constant(compiler_context *ctx, midgard_block *block)
2812 midgard_cull_dead_branch(compiler_context *ctx, midgard_block *block)
2834 midgard_legalize_invert(compiler_context *ctx, midgard_block *block)
2859 emit_fragment_epilogue(compiler_context *ctx, unsigned rt, unsigned sample_iter)
2862 midgard_instruction *br = ctx->writeout_branch[rt][sample_iter];
2865 ins.branch.target_block = ctx->block_count - 1;
2868 emit_mir_instruction(ctx, ins);
2870 ctx->current_block->epilogue = true;
2871 schedule_barrier(ctx);
2876 emit_block_init(compiler_context *ctx)
2878 midgard_block *this_block = ctx->after_block;
2879 ctx->after_block = NULL;
2882 this_block = create_empty_block(ctx);
2884 list_addtail(&this_block->base.link, &ctx->blocks);
2887 ++ctx->block_count;
2891 ctx->current_block = this_block;
2897 emit_block(compiler_context *ctx, nir_block *block)
2899 midgard_block *this_block = emit_block_init(ctx);
2902 emit_instr(ctx, instr);
2903 ++ctx->instruction_count;
2909 static midgard_block *emit_cf_list(struct compiler_context *ctx, struct exec_list *list);
2912 emit_if(struct compiler_context *ctx, nir_if *nif)
2914 midgard_block *before_block = ctx->current_block;
2919 midgard_instruction *then_branch = mir_last_in_block(ctx->current_block);
2925 midgard_block *then_block = emit_cf_list(ctx, &nif->then_list);
2926 midgard_block *end_then_block = ctx->current_block;
2930 midgard_instruction *then_exit = mir_last_in_block(ctx->current_block);
2934 int else_idx = ctx->block_count;
2935 int count_in = ctx->instruction_count;
2936 midgard_block *else_block = emit_cf_list(ctx, &nif->else_list);
2937 midgard_block *end_else_block = ctx->current_block;
2938 int after_else_idx = ctx->block_count;
2945 if (ctx->instruction_count == count_in) {
2956 ctx->after_block = create_empty_block(ctx);
2961 pan_block_add_successor(&end_then_block->base, &ctx->after_block->base);
2962 pan_block_add_successor(&end_else_block->base, &ctx->after_block->base);
2966 emit_loop(struct compiler_context *ctx, nir_loop *nloop)
2969 midgard_block *start_block = ctx->current_block;
2972 int loop_idx = ++ctx->current_loop_depth;
2975 int start_idx = ctx->block_count;
2978 midgard_block *loop_block = emit_cf_list(ctx, &nloop->body);
2983 emit_mir_instruction(ctx, br_back);
2987 pan_block_add_successor(&ctx->current_block->base, &loop_block->base);
2991 int break_block_idx = ctx->block_count;
2995 ctx->after_block = create_empty_block(ctx);
2997 mir_foreach_block_from(ctx, start_block, _block) {
3014 pan_block_add_successor(_block, &ctx->after_block->base);
3020 --ctx->current_loop_depth;
3023 ++ctx->loop_count;
3027 emit_cf_list(struct compiler_context *ctx, struct exec_list *list)
3034 midgard_block *block = emit_block(ctx, nir_cf_node_as_block(node));
3043 emit_if(ctx, nir_cf_node_as_if(node));
3047 emit_loop(ctx, nir_cf_node_as_loop(node));
3064 midgard_get_first_tag_from_block(compiler_context *ctx, unsigned block_idx)
3066 midgard_block *initial_block = mir_get_block(ctx, block_idx);
3068 mir_foreach_block_from(ctx, initial_block, _v) {
3088 mir_add_writeout_loops(compiler_context *ctx)
3090 for (unsigned rt = 0; rt < ARRAY_SIZE(ctx->writeout_branch); ++rt) {
3092 midgard_instruction *br = ctx->writeout_branch[rt][s];
3096 pan_block_add_successor(&(mir_get_block(ctx, popped - 1)->base),
3097 &ctx->current_block->base);
3098 br->branch.target_block = emit_fragment_epilogue(ctx, rt, s);
3106 next_br = ctx->writeout_branch[rt][s + 1];
3108 if (!next_br && (rt + 1) < ARRAY_SIZE(ctx->writeout_branch))
3109 next_br = ctx->writeout_branch[rt + 1][0];
3115 emit_mir_instruction(ctx, uncond);
3116 pan_block_add_successor(&ctx->current_block->base,
3117 &(mir_get_block(ctx, popped)->base));
3118 schedule_barrier(ctx);
3136 compiler_context *ctx = rzalloc(NULL, compiler_context);
3137 ctx->sysval_to_id = panfrost_init_sysvals(&info->sysvals,
3139 ctx);
3141 ctx->inputs = inputs;
3142 ctx->nir = nir;
3143 ctx->info = info;
3144 ctx->stage = nir->info.stage;
3152 ctx->blend_sample_iterations =
3155 ctx->blend_input = ~0;
3156 ctx->blend_src1 = ~0;
3157 ctx->quirks = midgard_get_quirks(inputs->gpu_id);
3161 ctx->ssa_constants = _mesa_hash_table_u64_create(ctx);
3169 if (ctx->stage == MESA_SHADER_VERTEX) {
3184 inputs->is_blend, ctx->quirks & MIDGARD_BROKEN_BLEND_LOADS);
3197 optimise_nir(nir, ctx->quirks, inputs->is_blend, inputs->is_blit);
3211 list_inithead(&ctx->blocks);
3212 ctx->block_count = 0;
3213 ctx->func = func;
3214 ctx->already_emitted = calloc(BITSET_WORDS(func->impl->ssa_alloc), sizeof(BITSET_WORD));
3217 emit_block_init(ctx);
3222 emit_mir_instruction(ctx, wait);
3224 ++ctx->instruction_count;
3227 emit_cf_list(ctx, &func->impl->body);
3228 free(ctx->already_emitted);
3234 mir_foreach_block(ctx, _block) {
3236 inline_alu_constants(ctx, block);
3237 embedded_to_inline_constant(ctx, block);
3245 progress |= midgard_opt_dead_code_eliminate(ctx);
3247 mir_foreach_block(ctx, _block) {
3249 progress |= midgard_opt_copy_prop(ctx, block);
3250 progress |= midgard_opt_combine_projection(ctx, block);
3251 progress |= midgard_opt_varying_projection(ctx, block);
3255 mir_foreach_block(ctx, _block) {
3257 midgard_lower_derivatives(ctx, block);
3258 midgard_legalize_invert(ctx, block);
3259 midgard_cull_dead_branch(ctx, block);
3262 if (ctx->stage == MESA_SHADER_FRAGMENT)
3263 mir_add_writeout_loops(ctx);
3267 mir_analyze_helper_requirements(ctx);
3270 mir_print_shader(ctx);
3273 midgard_schedule_program(ctx);
3274 mir_ra(ctx);
3277 mir_print_shader(ctx);
3280 mir_analyze_helper_terminate(ctx);
3289 mir_foreach_block(ctx, _block) {
3295 mir_foreach_block(ctx, _block) {
3308 mir_foreach_block(ctx, _block) {
3316 emit_binary_bundle(ctx, block, bundle, binary, lookahead);
3327 info->midgard.first_tag = midgard_get_first_tag_from_block(ctx, 0);
3329 info->ubo_mask = ctx->ubo_mask & ((1 << ctx->nir->info.num_ubos) - 1);
3350 mir_foreach_block(ctx, _block) {
3375 ctx->nir->info.label ?: "",
3376 ctx->inputs->is_blend ? "PAN_SHADER_BLEND" :
3377 gl_shader_stage_name(ctx->stage),
3378 nr_ins, nr_bundles, ctx->quadword_count,
3380 ctx->loop_count,
3381 ctx->spills, ctx->fills);
3384 _mesa_hash_table_u64_destroy(ctx->ssa_constants);
3385 _mesa_hash_table_u64_destroy(ctx->sysval_to_id);
3387 ralloc_free(ctx);