Lines Matching refs:instr
218 midgard_nir_lower_global_load_instr(nir_builder *b, nir_instr *instr, void *data)
220 if (instr->type != nir_instr_type_intrinsic)
223 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
234 b->cursor = nir_before_instr(instr);
257 nir_ssa_dest_init(&shared_load->instr, &shared_load->dest,
259 nir_builder_instr_insert(b, &shared_load->instr);
286 mdg_should_scalarize(const nir_instr *instr, const void *_unused)
288 const nir_alu_instr *alu = nir_instr_as_alu(instr);
310 midgard_vectorize_filter(const nir_instr *instr, const void *data)
312 if (instr->type != nir_instr_type_alu)
315 const nir_alu_instr *alu = nir_instr_as_alu(instr);
473 emit_load_const(compiler_context *ctx, nir_load_const_instr *instr)
475 nir_ssa_def def = instr->def;
479 assert(instr->def.num_components * instr->def.bit_size <= sizeof(*consts) * 8);
482 nir_const_value_to_array(consts->u##bits, instr->value, \
483 instr->def.num_components, u##bits)
485 switch (instr->def.bit_size) {
534 #define ATOMIC_CASE_IMPL(ctx, instr, nir, op, is_shared) \
536 emit_atomic(ctx, instr, is_shared, midgard_op_##op, ~0); \
539 #define ATOMIC_CASE(ctx, instr, nir, op) \
540 ATOMIC_CASE_IMPL(ctx, instr, shared_atomic_##nir, atomic_##op, true); \
541 ATOMIC_CASE_IMPL(ctx, instr, global_atomic_##nir, atomic_##op, false);
543 #define IMAGE_ATOMIC_CASE(ctx, instr, nir, op) \
545 midgard_instruction ins = emit_image_op(ctx, instr, true); \
546 emit_atomic(ctx, instr, false, midgard_op_atomic_##op, ins.dest); \
637 mir_copy_src(midgard_instruction *ins, nir_alu_instr *instr, unsigned i, unsigned to, bool *abs, bool *neg, bool *not, enum midgard_roundmode *roundmode, bool is_int, unsigned bcast_count)
639 nir_alu_src src = instr->src[i];
649 if (nir_accepts_inot(instr->op, i) && pan_has_source_mod(&src, nir_op_inot))
669 ins->src_types[to] = nir_op_infos[instr->op].input_types[i] | bits;
684 mir_is_bcsel_float(nir_alu_instr *instr)
705 nir_alu_src s = instr->src[i];
713 nir_alu_src s = instr->src[i];
721 nir_dest *dest = &instr->dest.dest;
730 emit_alu(compiler_context *ctx, nir_alu_instr *instr)
732 nir_dest *dest = &instr->dest.dest;
740 if (instr->op == nir_op_fddx || instr->op == nir_op_fddy) {
741 midgard_emit_derivatives(ctx, instr);
748 unsigned nr_inputs = nir_op_infos[instr->op].num_inputs;
761 ASSERTED unsigned src_bitsize = nir_src_bit_size(instr->src[0].src);
766 switch (instr->op) {
907 if (instr->op == nir_op_f2f16 || instr->op == nir_op_f2f32 ||
908 instr->op == nir_op_f2f64)
924 instr->op == nir_op_fge ? midgard_alu_op_fle :
925 instr->op == nir_op_fge32 ? midgard_alu_op_fle :
926 instr->op == nir_op_ige32 ? midgard_alu_op_ile :
927 instr->op == nir_op_uge32 ? midgard_alu_op_ule :
936 bool mixed = nir_is_non_scalar_swizzle(&instr->src[0], nr_components);
937 bool is_float = mir_is_bcsel_float(instr);
954 mesa_loge("Unhandled ALU op %s\n", nir_op_infos[instr->op].name);
960 if (op == midgard_alu_op_imov && nir_src_is_const(instr->src[0].src)
961 && nir_src_bit_size(instr->src[0].src) == 32
962 && nir_is_same_comp_swizzle(instr->src[0].swizzle,
963 nir_src_num_components(instr->src[0].src))) {
972 if (instr->op == nir_op_umul_high || instr->op == nir_op_imul_high) {
976 } else if (instr->op == nir_op_fsat) {
978 } else if (instr->op == nir_op_fsat_signed_mali) {
980 } else if (instr->op == nir_op_fclamp_pos_mali) {
995 .dest_type = nir_op_infos[instr->op].output_type
1008 mir_copy_src(&ins, instr, 0, 1, &ins.src_abs[1], &ins.src_neg[1], &ins.src_invert[1], roundptr, is_int, broadcast_swizzle);
1013 if (instr->op == nir_op_b32csel) {
1029 mir_copy_src(&ins, instr, i, to, &ins.src_abs[to], &ins.src_neg[to], &ins.src_invert[to], roundptr, is_int, broadcast_swizzle);
1032 if (instr->op == nir_op_b32csel && ins.src_invert[2]) {
1039 if (instr->op == nir_op_fneg || instr->op == nir_op_fabs) {
1041 if (instr->op == nir_op_fneg)
1044 if (instr->op == nir_op_fabs)
1055 ins.mask &= instr->dest.write_mask;
1062 if (instr->op == nir_op_b2f32 || instr->op == nir_op_b2i32) {
1073 if (instr->op == nir_op_b2f32)
1080 } else if (instr->op == nir_op_b2f16) {
1098 } else if (instr->op == nir_op_pack_32_2x16) {
1102 } else if (instr->op == nir_op_pack_32_4x8) {
1106 } else if (instr->op == nir_op_unpack_32_2x16) {
1110 } else if (instr->op == nir_op_unpack_32_4x8) {
1166 mir_set_intr_mask(nir_instr *instr, midgard_instruction *ins, bool is_read)
1168 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1195 nir_instr *instr,
1205 unsigned dest_size = (instr->type == nir_instr_type_intrinsic) ?
1206 nir_dest_bit_size(nir_instr_as_intrinsic(instr)->dest) : 32;
1226 if (instr->type == nir_instr_type_intrinsic)
1227 mir_set_intr_mask(instr, &ins, true);
1256 nir_instr *instr,
1264 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
1278 mir_set_intr_mask(instr, &ins, is_read);
1326 mir_set_intr_mask(instr, &ins, is_read);
1345 * `image_direct_address` should be ~0 when instr is not an image_atomic
1350 nir_intrinsic_instr *instr,
1361 unsigned dest = nir_dest_index(&instr->dest);
1363 unsigned val = nir_src_index(ctx, &instr->src[val_src]);
1364 unsigned bitsize = nir_src_bit_size(instr->src[val_src]);
1376 nir_src *src_offset = nir_get_io_offset_src(instr);
1380 unsigned xchg_val = nir_src_index(ctx, &instr->src[xchg_val_src]);
1412 mir_set_intr_mask(&instr->instr, &ins, true);
1483 emit_image_op(compiler_context *ctx, nir_intrinsic_instr *instr, bool is_atomic)
1485 enum glsl_sampler_dim dim = nir_intrinsic_image_dim(instr);
1489 bool is_array = nir_intrinsic_image_array(instr);
1490 bool is_store = instr->intrinsic == nir_intrinsic_image_store;
1495 unsigned coord_reg = nir_src_index(ctx, &instr->src[1]);
1498 nir_src *index = &instr->src[0];
1508 unsigned val = nir_src_index(ctx, &instr->src[3]);
1511 nir_alu_type type = nir_intrinsic_src_type(instr);
1514 ins.src_types[0] = base_type | nir_src_bit_size(instr->src[3]);
1520 nir_alu_type type = nir_intrinsic_dest_type(instr);
1521 ins = ld_image(type, nir_dest_index(&instr->dest), PACK_LDST_ATTRIB_OFS(address));
1522 ins.mask = mask_of(nir_intrinsic_dest_components(instr));
1577 emit_sysval_read(compiler_context *ctx, nir_instr *instr,
1586 int sysval = panfrost_sysval_for_instr(instr, &nir_dest);
1593 emit_ubo_read(ctx, instr, dest, (uniform * 16) + offset, NULL, 0,
1672 emit_compute_builtin(compiler_context *ctx, nir_intrinsic_instr *instr)
1674 unsigned reg = nir_dest_index(&instr->dest);
1678 ins.load_store.arg_reg = compute_builtin_arg(instr->intrinsic);
1696 emit_vertex_builtin(compiler_context *ctx, nir_intrinsic_instr *instr)
1698 unsigned reg = nir_dest_index(&instr->dest);
1699 emit_attr_read(ctx, reg, vertex_builtin_arg(instr->intrinsic), 1, nir_type_int);
1703 emit_special(compiler_context *ctx, nir_intrinsic_instr *instr, unsigned idx)
1705 unsigned reg = nir_dest_index(&instr->dest);
1745 output_load_rt_addr(compiler_context *ctx, nir_intrinsic_instr *instr)
1751 var = nir_find_variable_with_driver_location(ctx->nir, nir_var_shader_out, nir_intrinsic_base(instr));
1768 emit_intrinsic(compiler_context *ctx, nir_intrinsic_instr *instr)
1772 switch (instr->intrinsic) {
1775 bool conditional = instr->intrinsic == nir_intrinsic_discard_if;
1780 discard.src[0] = mir_get_branch_cond(&instr->src[0],
1793 emit_image_op(ctx, instr, false);
1797 unsigned nr_comp = nir_intrinsic_dest_components(instr);
1798 emit_sysval_read(ctx, &instr->instr, nr_comp, 0);
1810 bool is_ubo = instr->intrinsic == nir_intrinsic_load_ubo;
1811 bool is_global = instr->intrinsic == nir_intrinsic_load_global ||
1812 instr->intrinsic == nir_intrinsic_load_global_constant;
1813 bool is_shared = instr->intrinsic == nir_intrinsic_load_shared;
1814 bool is_scratch = instr->intrinsic == nir_intrinsic_load_scratch;
1815 bool is_flat = instr->intrinsic == nir_intrinsic_load_input;
1816 bool is_kernel = instr->intrinsic == nir_intrinsic_load_kernel_input;
1817 bool is_interp = instr->intrinsic == nir_intrinsic_load_interpolated_input;
1823 (is_flat) ? nir_intrinsic_dest_type(instr) :
1829 offset = nir_intrinsic_base(instr);
1832 unsigned nr_comp = nir_intrinsic_dest_components(instr);
1834 nir_src *src_offset = nir_get_io_offset_src(instr);
1844 nir_intrinsic_component(instr) : 0;
1845 reg = nir_dest_index(&instr->dest);
1848 emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0, 0, nr_comp);
1850 nir_src index = instr->src[0];
1856 emit_ubo_read(ctx, &instr->instr, reg, offset, indirect_offset, 0, uindex, nr_comp);
1859 emit_global(ctx, &instr->instr, true, reg, src_offset, seg);
1861 emit_varying_read(ctx, reg, offset, nr_comp, component, indirect_offset, t | nir_dest_bit_size(instr->dest), is_flat);
1891 reg = nir_dest_index(&instr->dest);
1898 unsigned target = output_load_rt_addr(ctx, instr);
1902 if (nir_src_is_const(instr->src[0])) {
1903 unsigned sample = nir_src_as_uint(instr->src[0]);
1909 ld.src[1] = nir_src_index(ctx, &instr->src[0]);
1924 reg = nir_dest_index(&instr->dest);
1926 unsigned bits = nir_dest_bit_size(instr->dest);
1934 unsigned index = output_load_rt_addr(ctx, instr);
1956 assert(nir_src_is_const(instr->src[1]) && "no indirect outputs");
1958 reg = nir_src_index(ctx, &instr->src[0]);
1961 bool combined = instr->intrinsic ==
1969 writeout = nir_intrinsic_component(instr);
1971 reg_z = nir_src_index(ctx, &instr->src[2]);
1973 reg_s = nir_src_index(ctx, &instr->src[3]);
1975 reg_2 = nir_src_index(ctx, &instr->src[4]);
1981 nir_intrinsic_base(instr));
1995 if (instr->src[4].is_ssa) {
2011 assert(instr->intrinsic == nir_intrinsic_store_output);
2023 offset = nir_intrinsic_base(instr) + nir_src_as_uint(instr->src[1]);
2025 unsigned dst_component = nir_intrinsic_component(instr);
2026 unsigned nr_comp = nir_src_num_components(instr->src[0]);
2033 switch (nir_alu_type_get_base_type(nir_intrinsic_src_type(instr))) {
2075 reg = nir_src_index(ctx, &instr->src[0]);
2085 reg = nir_src_index(ctx, &instr->src[0]);
2089 if (instr->intrinsic == nir_intrinsic_store_global)
2091 else if (instr->intrinsic == nir_intrinsic_store_shared)
2096 emit_global(ctx, &instr->instr, false, reg, &instr->src[1], seg);
2101 emit_sysval_read(ctx, &instr->instr, 2, 0);
2107 emit_sysval_read(ctx, &instr->instr, 1, 0);
2111 emit_sysval_read(ctx, &instr->instr, 1, 4);
2116 emit_sysval_read(ctx, &instr->instr, 1, 8);
2120 emit_sysval_read(ctx, &instr->instr, 2, 0);
2128 emit_sysval_read(ctx, &instr->instr, 3, 0);
2132 emit_sysval_read(ctx, &instr->instr, 4, 0);
2139 emit_compute_builtin(ctx, instr);
2144 emit_vertex_builtin(ctx, instr);
2148 emit_special(ctx, instr, 96);
2152 emit_special(ctx, instr, 97);
2169 ATOMIC_CASE(ctx, instr, add, add);
2170 ATOMIC_CASE(ctx, instr, and, and);
2171 ATOMIC_CASE(ctx, instr, comp_swap, cmpxchg);
2172 ATOMIC_CASE(ctx, instr, exchange, xchg);
2173 ATOMIC_CASE(ctx, instr, imax, imax);
2174 ATOMIC_CASE(ctx, instr, imin, imin);
2175 ATOMIC_CASE(ctx, instr, or, or);
2176 ATOMIC_CASE(ctx, instr, umax, umax);
2177 ATOMIC_CASE(ctx, instr, umin, umin);
2178 ATOMIC_CASE(ctx, instr, xor, xor);
2180 IMAGE_ATOMIC_CASE(ctx, instr, add, add);
2181 IMAGE_ATOMIC_CASE(ctx, instr, and, and);
2182 IMAGE_ATOMIC_CASE(ctx, instr, comp_swap, cmpxchg);
2183 IMAGE_ATOMIC_CASE(ctx, instr, exchange, xchg);
2184 IMAGE_ATOMIC_CASE(ctx, instr, imax, imax);
2185 IMAGE_ATOMIC_CASE(ctx, instr, imin, imin);
2186 IMAGE_ATOMIC_CASE(ctx, instr, or, or);
2187 IMAGE_ATOMIC_CASE(ctx, instr, umax, umax);
2188 IMAGE_ATOMIC_CASE(ctx, instr, umin, umin);
2189 IMAGE_ATOMIC_CASE(ctx, instr, xor, xor);
2192 fprintf(stderr, "Unhandled intrinsic %s\n", nir_intrinsic_infos[instr->intrinsic].name);
2260 mdg_texture_mode(nir_tex_instr *instr)
2262 if (instr->op == nir_texop_tg4 && instr->is_shadow)
2264 else if (instr->op == nir_texop_tg4)
2265 return TEXTURE_GATHER_X + instr->component;
2266 else if (instr->is_shadow)
2273 set_tex_coord(compiler_context *ctx, nir_tex_instr *instr,
2276 int coord_idx = nir_tex_instr_src_index(instr, nir_tex_src_coord);
2280 int comparator_idx = nir_tex_instr_src_index(instr, nir_tex_src_comparator);
2281 int ms_idx = nir_tex_instr_src_index(instr, nir_tex_src_ms_index);
2285 unsigned coords = nir_src_index(ctx, &instr->src[coord_idx].src);
2289 ins->src_types[1] = nir_tex_instr_src_type(instr, coord_idx) |
2290 nir_src_bit_size(instr->src[coord_idx].src);
2292 unsigned nr_comps = instr->coord_components;
2306 if (instr->is_shadow && comparator_idx < 0) {
2316 if (instr->is_array) {
2321 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
2356 nir_src_index(ctx, &instr->src[ms_or_comparator_idx].src);
2417 emit_texop_native(compiler_context *ctx, nir_tex_instr *instr,
2420 nir_dest *dest = &instr->dest;
2422 int texture_index = instr->texture_index;
2423 int sampler_index = instr->sampler_index;
2425 nir_alu_type dest_base = nir_alu_type_get_base_type(instr->dest_type);
2438 .dest_type = instr->dest_type,
2443 .format = midgard_tex_format(instr->sampler_dim),
2446 .mode = mdg_texture_mode(instr)
2450 if (instr->is_shadow && !instr->is_new_style_shadow && instr->op != nir_texop_tg4)
2454 for (unsigned i = 0; i < instr->num_srcs; ++i) {
2455 int index = nir_src_index(ctx, &instr->src[i].src);
2456 unsigned sz = nir_src_bit_size(instr->src[i].src);
2457 nir_alu_type T = nir_tex_instr_src_type(instr, i) | sz;
2459 switch (instr->src[i].src_type) {
2461 set_tex_coord(ctx, instr, &ins);
2469 if (!is_txf && pan_attach_constant_bias(ctx, instr->src[i].src, &ins.texture))
2502 fprintf(stderr, "Unknown texture source type: %d\n", instr->src[i].src_type);
2512 emit_tex(compiler_context *ctx, nir_tex_instr *instr)
2514 switch (instr->op) {
2517 emit_texop_native(ctx, instr, midgard_tex_op_normal);
2521 emit_texop_native(ctx, instr, midgard_tex_op_gradient);
2525 emit_texop_native(ctx, instr, midgard_tex_op_fetch);
2528 emit_sysval_read(ctx, &instr->instr, 4, 0);
2531 fprintf(stderr, "Unhandled texture op: %d\n", instr->op);
2538 emit_jump(compiler_context *ctx, nir_jump_instr *instr)
2540 switch (instr->type) {
2556 emit_instr(compiler_context *ctx, struct nir_instr *instr)
2558 switch (instr->type) {
2560 emit_load_const(ctx, nir_instr_as_load_const(instr));
2564 emit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
2568 emit_alu(ctx, nir_instr_as_alu(instr));
2572 emit_tex(ctx, nir_instr_as_tex(instr));
2576 emit_jump(ctx, nir_instr_as_jump(instr));
2901 nir_foreach_instr(instr, block) {
2902 emit_instr(ctx, instr);