Lines Matching defs:instr
119 bi_emit_jump(bi_builder *b, nir_jump_instr *instr)
123 switch (instr->type) {
375 bi_is_intr_immediate(nir_intrinsic_instr *instr, unsigned *immediate, unsigned max)
377 nir_src *offset = nir_get_io_offset_src(instr);
382 *immediate = nir_intrinsic_base(instr) + nir_src_as_uint(*offset);
399 bi_copy_component(bi_builder *b, nir_intrinsic_instr *instr, bi_index tmp)
401 unsigned component = nir_intrinsic_component(instr);
402 unsigned nr = instr->num_components;
404 unsigned bitsize = nir_dest_bit_size(instr->dest);
415 bi_make_vec_to(b, bi_dest_index(&instr->dest),
416 srcs, channels, nr, nir_dest_bit_size(instr->dest));
420 bi_emit_load_attr(bi_builder *b, nir_intrinsic_instr *instr)
422 nir_alu_type T = nir_intrinsic_dest_type(instr);
424 nir_src *offset = nir_get_io_offset_src(instr);
425 unsigned component = nir_intrinsic_component(instr);
426 enum bi_vecsize vecsize = (instr->num_components + component - 1);
428 unsigned base = nir_intrinsic_base(instr);
430 bool immediate = bi_is_intr_immediate(instr, &imm_index, 16);
431 bi_index dest = (component == 0) ? bi_dest_index(&instr->dest) : bi_temp(b->shader);
439 bi_index idx = bi_src_index(&instr->src[0]);
453 bi_copy_component(b, instr, dest);
492 bi_emit_load_vary(bi_builder *b, nir_intrinsic_instr *instr)
497 bool smooth = instr->intrinsic == nir_intrinsic_load_interpolated_input;
500 unsigned component = nir_intrinsic_component(instr);
501 enum bi_vecsize vecsize = (instr->num_components + component - 1);
502 bi_index dest = (component == 0) ? bi_dest_index(&instr->dest) : bi_temp(b->shader);
504 unsigned sz = nir_dest_bit_size(instr->dest);
507 nir_intrinsic_instr *parent = nir_src_as_intrinsic(instr->src[0]);
533 nir_src *offset = nir_get_io_offset_src(instr);
535 bool immediate = bi_is_intr_immediate(instr, &imm_index, 20);
542 bi_varying_offset(b->shader, instr));
551 unsigned base = nir_intrinsic_base(instr);
558 unsigned vbase = bi_varying_base_bytes(b->shader, instr);
588 bi_copy_component(b, instr, dest);
684 panfrost_sysval_for_instr(&intr->instr, NULL),
742 bi_emit_load_blend_input(bi_builder *b, nir_intrinsic_instr *instr)
744 nir_io_semantics sem = nir_intrinsic_io_semantics(instr);
746 unsigned size = nir_alu_type_get_type_size(nir_intrinsic_dest_type(instr));
754 bi_emit_collect_to(b, bi_dest_index(&instr->dest), srcs, size == 32 ? 4 : 2);
834 bi_emit_fragment_out(bi_builder *b, nir_intrinsic_instr *instr)
836 bool combined = instr->intrinsic ==
839 unsigned writeout = combined ? nir_intrinsic_component(instr) :
847 nir_var_shader_out, nir_intrinsic_base(instr));
851 bi_index src0 = bi_src_index(&instr->src[0]);
871 nir_alu_type T = nir_intrinsic_src_type(instr);
873 bi_index rgba = bi_src_index(&instr->src[0]);
880 if (nir_src_num_components(instr->src[0]) < 4)
890 z = bi_src_index(&instr->src[2]);
893 s = bi_src_index(&instr->src[3]);
903 bi_index color = bi_src_index(&instr->src[0]);
904 bi_index color2 = dual ? bi_src_index(&instr->src[4]) : bi_null();
905 nir_alu_type T2 = dual ? nir_intrinsic_dest_type(instr) : 0;
920 nir_src_num_components(instr->src[0]),
921 nir_alu_type_get_type_size(nir_intrinsic_src_type(instr)));
924 bi_emit_blend_op(b, color, nir_intrinsic_src_type(instr),
962 bifrost_nir_specialize_idvs(nir_builder *b, nir_instr *instr, void *data)
966 if (instr->type != nir_instr_type_intrinsic)
969 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
975 nir_instr_remove(instr);
983 bi_emit_store_vary(bi_builder *b, nir_intrinsic_instr *instr)
990 ASSERTED nir_alu_type T = nir_intrinsic_src_type(instr);
996 bool immediate = bi_is_intr_immediate(instr, &imm_index, 16);
1004 unsigned nr = util_last_bit(nir_intrinsic_write_mask(instr));
1005 assert(nr > 0 && nr <= nir_intrinsic_src_components(instr, 0));
1007 bi_index data = bi_src_index(&instr->src[0]);
1015 if (nr < nir_intrinsic_src_components(instr, 0)) {
1019 split->nr_dests = nir_intrinsic_src_components(instr, 0);
1033 bool psiz = (nir_intrinsic_io_semantics(instr).location == VARYING_SLOT_PSIZ);
1060 bi_store(b, nr * nir_src_bit_size(instr->src[0]),
1063 varying ? bi_varying_offset(b->shader, instr) : 0);
1074 bi_src_index(nir_get_io_offset_src(instr)),
1075 bi_imm_u32(nir_intrinsic_base(instr)),
1087 bi_emit_load_ubo(bi_builder *b, nir_intrinsic_instr *instr)
1089 nir_src *offset = nir_get_io_offset_src(instr);
1094 bool kernel_input = (instr->intrinsic == nir_intrinsic_load_kernel_input);
1096 bi_load_ubo_to(b, instr->num_components * nir_dest_bit_size(instr->dest),
1097 bi_dest_index(&instr->dest), offset_is_const ?
1099 kernel_input ? bi_zero() : bi_src_index(&instr->src[0]));
1103 bi_emit_load_push_constant(bi_builder *b, nir_intrinsic_instr *instr)
1107 nir_src *offset = &instr->src[0];
1109 uint32_t base = nir_intrinsic_base(instr) + nir_src_as_uint(*offset);
1112 unsigned bits = nir_dest_bit_size(instr->dest) *
1113 nir_dest_num_components(instr->dest);
1125 bi_emit_collect_to(b, bi_dest_index(&instr->dest), channels, n);
1165 bi_emit_load(bi_builder *b, nir_intrinsic_instr *instr, enum bi_seg seg)
1168 unsigned bits = instr->num_components * nir_dest_bit_size(instr->dest);
1169 bi_index dest = bi_dest_index(&instr->dest);
1170 bi_index addr_lo = bi_extract(b, bi_src_index(&instr->src[0]), 0);
1171 bi_index addr_hi = bi_addr_high(b, &instr->src[0]);
1180 bi_emit_store(bi_builder *b, nir_intrinsic_instr *instr, enum bi_seg seg)
1183 assert(nir_intrinsic_write_mask(instr) ==
1184 BITFIELD_MASK(instr->num_components));
1187 bi_index addr_lo = bi_extract(b, bi_src_index(&instr->src[1]), 0);
1188 bi_index addr_hi = bi_addr_high(b, &instr->src[1]);
1192 bi_store(b, instr->num_components * nir_src_bit_size(instr->src[0]),
1193 bi_src_index(&instr->src[0]),
1381 bi_emit_image_index(bi_builder *b, nir_intrinsic_instr *instr)
1383 nir_src src = instr->src[0];
1400 bi_emit_image_load(bi_builder *b, nir_intrinsic_instr *instr)
1402 enum glsl_sampler_dim dim = nir_intrinsic_image_dim(instr);
1403 unsigned coord_comps = nir_image_intrinsic_coord_components(instr);
1404 bool array = nir_intrinsic_image_array(instr);
1407 bi_index coords = bi_src_index(&instr->src[1]);
1410 bi_index dest = bi_dest_index(&instr->dest);
1411 enum bi_register_format regfmt = bi_reg_fmt_for_nir(nir_intrinsic_dest_type(instr));
1412 enum bi_vecsize vecsize = instr->num_components - 1;
1417 if (b->shader->arch >= 9 && nir_src_is_const(instr->src[0])) {
1419 nir_src_as_uint(instr->src[0]));
1426 bi_emit_image_index(b, instr), regfmt,
1430 bi_split_dest(b, instr->dest);
1434 bi_emit_lea_image(bi_builder *b, nir_intrinsic_instr *instr)
1436 enum glsl_sampler_dim dim = nir_intrinsic_image_dim(instr);
1437 bool array = nir_intrinsic_image_array(instr);
1439 unsigned coord_comps = nir_image_intrinsic_coord_components(instr);
1444 enum bi_register_format type = (instr->intrinsic == nir_intrinsic_image_store) ?
1445 bi_reg_fmt_for_nir(nir_intrinsic_src_type(instr)) :
1448 bi_index coords = bi_src_index(&instr->src[1]);
1453 if (b->shader->arch >= 9 && nir_src_is_const(instr->src[0])) {
1455 nir_src_as_uint(instr->src[0]));
1462 bi_emit_image_index(b, instr), type);
1475 bi_emit_image_store(bi_builder *b, nir_intrinsic_instr *instr)
1478 bi_emit_split_i32(b, a, bi_emit_lea_image(b, instr), 3);
1480 bi_st_cvt(b, bi_src_index(&instr->src[3]), a[0], a[1], a[2],
1481 bi_reg_fmt_for_nir(nir_intrinsic_src_type(instr)),
1482 instr->num_components - 1);
1520 bi_emit_load_frag_coord(bi_builder *b, nir_intrinsic_instr *instr)
1539 bi_make_vec_to(b, bi_dest_index(&instr->dest), src, NULL, 4, 32);
1543 bi_emit_ld_tile(bi_builder *b, nir_intrinsic_instr *instr)
1545 bi_index dest = bi_dest_index(&instr->dest);
1546 nir_alu_type T = nir_intrinsic_dest_type(instr);
1549 unsigned size = nir_dest_bit_size(instr->dest);
1550 unsigned nr = instr->num_components;
1556 nir_var_shader_out, nir_intrinsic_base(instr));
1574 bi_emit_intrinsic(bi_builder *b, nir_intrinsic_instr *instr)
1576 bi_index dst = nir_intrinsic_infos[instr->intrinsic].has_dest ?
1577 bi_dest_index(&instr->dest) : bi_null();
1580 switch (instr->intrinsic) {
1591 bi_emit_load_blend_input(b, instr);
1593 bi_emit_load_vary(b, instr);
1595 bi_emit_load_attr(b, instr);
1602 bi_emit_fragment_out(b, instr);
1604 bi_emit_store_vary(b, instr);
1611 bi_emit_fragment_out(b, instr);
1616 bi_emit_load_ubo(b, instr);
1620 bi_emit_load_push_constant(b, instr);
1625 bi_emit_load(b, instr, BI_SEG_NONE);
1629 bi_emit_store(b, instr, BI_SEG_NONE);
1633 bi_emit_load(b, instr, BI_SEG_TL);
1637 bi_emit_store(b, instr, BI_SEG_TL);
1641 bi_emit_load(b, instr, BI_SEG_WLS);
1645 bi_emit_store(b, instr, BI_SEG_WLS);
1670 assert(nir_src_bit_size(instr->src[1]) == 32);
1672 bi_index addr = bi_src_index(&instr->src[0]);
1683 bi_emit_atomic_i32_to(b, dst, addr, bi_src_index(&instr->src[1]),
1684 instr->intrinsic);
1685 bi_split_dest(b, instr->dest);
1697 assert(nir_src_bit_size(instr->src[3]) == 32);
1700 bi_emit_lea_image(b, instr),
1701 bi_src_index(&instr->src[3]),
1702 instr->intrinsic);
1703 bi_split_dest(b, instr->dest);
1714 assert(nir_src_bit_size(instr->src[1]) == 32);
1717 bi_src_index(&instr->src[0]),
1718 bi_src_index(&instr->src[1]),
1719 instr->intrinsic);
1721 bi_split_dest(b, instr->dest);
1725 bi_emit_image_load(b, instr);
1729 bi_emit_image_store(b, instr);
1733 bi_emit_axchg_to(b, dst, bi_src_index(&instr->src[0]),
1734 &instr->src[1], BI_SEG_NONE);
1735 bi_split_dest(b, instr->dest);
1739 bi_emit_axchg_to(b, dst, bi_emit_lea_image(b, instr),
1740 &instr->src[3], BI_SEG_NONE);
1741 bi_split_dest(b, instr->dest);
1745 bi_emit_axchg_to(b, dst, bi_src_index(&instr->src[0]),
1746 &instr->src[1], BI_SEG_WLS);
1747 bi_split_dest(b, instr->dest);
1751 bi_emit_acmpxchg_to(b, dst, bi_src_index(&instr->src[0]),
1752 &instr->src[1], &instr->src[2], BI_SEG_NONE);
1753 bi_split_dest(b, instr->dest);
1757 bi_emit_acmpxchg_to(b, dst, bi_emit_lea_image(b, instr),
1758 &instr->src[3], &instr->src[4], BI_SEG_NONE);
1759 bi_split_dest(b, instr->dest);
1763 bi_emit_acmpxchg_to(b, dst, bi_src_index(&instr->src[0]),
1764 &instr->src[1], &instr->src[2], BI_SEG_WLS);
1765 bi_split_dest(b, instr->dest);
1769 bi_emit_load_frag_coord(b, instr);
1773 bi_emit_ld_tile(b, instr);
1777 bi_discard_b32(b, bi_src_index(&instr->src[0]));
1786 bi_load_sysval_nir(b, instr, 2, 0);
1791 bi_load_sysval_nir(b, instr, 1, 0);
1795 bi_load_sysval_nir(b, instr, 1, 0);
1799 bi_load_sysval_nir(b, instr, 1, 4);
1803 bi_load_sysval_nir(b, instr, 1, 8);
1807 bi_load_sysval_nir(b, instr, 1, 0);
1811 bi_load_sysval_nir(b, instr, 1, 8);
1818 bi_load_sysval_nir(b, instr, 3, 0);
1822 bi_load_sysval_nir(b, instr,
1823 nir_dest_num_components(instr->dest), 0);
1827 bi_load_sysval_nir(b, instr,
1828 nir_dest_num_components(instr->dest), 0);
1912 bi_split_dest(b, instr->dest);
1916 fprintf(stderr, "Unhandled intrinsic %s\n", nir_intrinsic_infos[instr->intrinsic].name);
1922 bi_emit_load_const(bi_builder *b, nir_load_const_instr *instr)
1925 assert(instr->def.num_components <= (32 / instr->def.bit_size));
1931 for (unsigned i = 0; i < instr->def.num_components; ++i) {
1932 unsigned v = nir_const_value_as_uint(instr->value[i], instr->def.bit_size);
1933 acc |= (v << (i * instr->def.bit_size));
1936 bi_mov_i32_to(b, bi_get_index(instr->def.index, false, 0), bi_imm_u32(acc));
2272 bi_emit_alu(bi_builder *b, nir_alu_instr *instr)
2274 bi_index dst = bi_dest_index(&instr->dest.dest);
2275 unsigned srcs = nir_op_infos[instr->op].num_inputs;
2276 unsigned sz = nir_dest_bit_size(instr->dest.dest);
2277 unsigned comps = nir_dest_num_components(instr->dest.dest);
2278 unsigned src_sz = srcs > 0 ? nir_src_bit_size(instr->src[0].src) : 0;
2284 if (!instr->dest.dest.is_ssa) {
2286 assert(instr->dest.write_mask);
2295 switch (instr->op) {
2300 srcs > 0 ? bi_src_index(&instr->src[0].src) : bi_null(),
2301 srcs > 1 ? bi_src_index(&instr->src[1].src) : bi_null(),
2302 srcs > 2 ? bi_src_index(&instr->src[2].src) : bi_null(),
2303 srcs > 3 ? bi_src_index(&instr->src[3].src) : bi_null(),
2307 instr->src[0].swizzle[0],
2308 instr->src[1].swizzle[0],
2309 srcs > 2 ? instr->src[2].swizzle[0] : 0,
2310 srcs > 3 ? instr->src[3].swizzle[0] : 0,
2325 bi_index vec = bi_src_index(&instr->src[0].src);
2326 unsigned chan = instr->src[0].swizzle[0];
2333 bi_mov_i32_to(b, dst, bi_extract(b, bi_src_index(&instr->src[0].src), 0));
2337 bi_mov_i32_to(b, dst, bi_extract(b, bi_src_index(&instr->src[0].src), 1));
2342 bi_extract(b, bi_src_index(&instr->src[0].src), instr->src[0].swizzle[0]),
2343 bi_extract(b, bi_src_index(&instr->src[1].src), instr->src[1].swizzle[0]));
2348 bi_extract(b, bi_src_index(&instr->src[0].src), 0),
2349 bi_extract(b, bi_src_index(&instr->src[0].src), 1));
2353 bi_index src = bi_src_index(&instr->src[0].src);
2362 bi_index src = bi_src_index(&instr->src[0].src);
2373 bi_index idx = bi_src_index(&instr->src[0].src);
2377 comps > 0 ? instr->src[0].swizzle[0] : 0,
2378 comps > 1 ? instr->src[0].swizzle[1] : 0,
2379 comps > 2 ? instr->src[0].swizzle[2] : 0,
2380 comps > 3 ? instr->src[0].swizzle[3] : 0,
2388 assert(nir_src_num_components(instr->src[0].src) == 2);
2391 bi_index idx = bi_src_index(&instr->src[0].src);
2395 instr->src[0].swizzle[0],
2396 instr->src[0].swizzle[1]
2407 bi_index idx = bi_src_index(&instr->src[0].src);
2408 bi_index s0 = bi_extract(b, idx, instr->src[0].swizzle[0]);
2410 bi_extract(b, idx, instr->src[0].swizzle[1]) : s0;
2419 if (instr->op == nir_op_f2f16_rtz)
2421 else if (instr->op == nir_op_f2f16_rtne)
2433 bi_index idx = bi_src_index(&instr->src[0].src);
2434 bi_index s0 = bi_extract(b, idx, instr->src[0].swizzle[0]);
2435 bi_index s1 = bi_extract(b, idx, instr->src[0].swizzle[1]);
2452 nir_alu_src *src = &instr->src[0];
2462 if (instr->op == nir_op_u2f16)
2474 bi_index idx = bi_src_index(&instr->src[0].src);
2479 chan[i] = instr->src[0].swizzle[i] * factor;
2502 bi_index idx = bi_src_index(&instr->src[0].src);
2503 bi_index s0 = bi_extract(b, idx, instr->src[0].swizzle[0]);
2504 bi_index s1 = bi_alu_src_index(b, instr->src[1], comps);
2505 bi_index s2 = bi_alu_src_index(b, instr->src[2], comps);
2507 if (!bi_nir_is_replicated(&instr->src[0])) {
2509 bi_half(bi_extract(b, idx, instr->src[0].swizzle[1]), false));
2520 bi_index s0 = srcs > 0 ? bi_alu_src_index(b, instr->src[0], comps) : bi_null();
2521 bi_index s1 = srcs > 1 ? bi_alu_src_index(b, instr->src[1], comps) : bi_null();
2522 bi_index s2 = srcs > 2 ? bi_alu_src_index(b, instr->src[2], comps) : bi_null();
2524 switch (instr->op) {
2644 bi_csel_to(b, nir_op_infos[instr->op].input_types[0], sz, dst,
2650 bi_csel_to(b, nir_op_infos[instr->op].input_types[0], sz, dst,
2656 bi_index bit = bi_imm_u32(instr->op == nir_op_fddx_must_abs_mali ? 1 : 2);
2669 switch (instr->op) {
2685 switch (instr->op) {
2883 bi_icmp_to(b, nir_type_int, sz, dst, s0, s1, bi_translate_cmpf(instr->op), BI_RESULT_TYPE_M1);
2892 bi_icmp_to(b, nir_type_uint, sz, dst, s0, s1, bi_translate_cmpf(instr->op), BI_RESULT_TYPE_M1);
2903 bi_fcmp_to(b, sz, dst, s0, s1, bi_translate_cmpf(instr->op), BI_RESULT_TYPE_M1);
2910 bi_fround_to(b, sz, dst, s0, bi_nir_round(instr->op));
3020 fprintf(stderr, "Unhandled ALU op %s\n", nir_op_infos[instr->op].name);
3168 bi_emit_texc_offset_ms_index(bi_builder *b, nir_tex_instr *instr)
3172 int offs_idx = nir_tex_instr_src_index(instr, nir_tex_src_offset);
3174 (!nir_src_is_const(instr->src[offs_idx].src) ||
3175 nir_src_as_uint(instr->src[offs_idx].src) != 0)) {
3176 unsigned nr = nir_src_num_components(instr->src[offs_idx].src);
3177 bi_index idx = bi_src_index(&instr->src[offs_idx].src);
3185 int ms_idx = nir_tex_instr_src_index(instr, nir_tex_src_ms_index);
3187 (!nir_src_is_const(instr->src[ms_idx].src) ||
3188 nir_src_as_uint(instr->src[ms_idx].src) != 0)) {
3190 bi_src_index(&instr->src[ms_idx].src), dest,
3204 bi_emit_valhall_offsets(bi_builder *b, nir_tex_instr *instr)
3208 int offs_idx = nir_tex_instr_src_index(instr, nir_tex_src_offset);
3209 int ms_idx = nir_tex_instr_src_index(instr, nir_tex_src_ms_index);
3210 int lod_idx = nir_tex_instr_src_index(instr, nir_tex_src_lod);
3214 (!nir_src_is_const(instr->src[offs_idx].src) ||
3215 nir_src_as_uint(instr->src[offs_idx].src) != 0)) {
3216 unsigned nr = nir_src_num_components(instr->src[offs_idx].src);
3217 bi_index idx = bi_src_index(&instr->src[offs_idx].src);
3236 (!nir_src_is_const(instr->src[ms_idx].src) ||
3237 nir_src_as_uint(instr->src[ms_idx].src) != 0)) {
3239 bi_src_index(&instr->src[ms_idx].src));
3244 (!nir_src_is_const(instr->src[lod_idx].src) ||
3245 nir_src_as_uint(instr->src[lod_idx].src) != 0) &&
3246 nir_tex_instr_src_type(instr, lod_idx) != nir_type_float) {
3248 bi_src_index(&instr->src[lod_idx].src), dest,
3384 bi_emit_texc(bi_builder *b, nir_tex_instr *instr)
3387 .op = bi_tex_op(instr->op),
3389 .shadow_or_clamp_disable = instr->is_shadow,
3390 .array = instr->is_array,
3391 .dimension = bifrost_tex_format(instr->sampler_dim),
3392 .format = bi_texture_format(instr->dest_type | nir_dest_bit_size(instr->dest), BI_CLAMP_NONE), /* TODO */
3402 (instr->op == nir_texop_tg4 ?
3403 BIFROST_TEXTURE_FETCH_GATHER4_R + instr->component :
3414 for (unsigned i = 0; i < instr->num_srcs; ++i) {
3415 bi_index index = bi_src_index(&instr->src[i].src);
3416 unsigned sz = nir_src_bit_size(instr->src[i].src);
3417 unsigned components = nir_src_num_components(instr->src[i].src);
3418 ASSERTED nir_alu_type base = nir_tex_instr_src_type(instr, i);
3421 switch (instr->src[i].src_type) {
3423 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
3449 nir_src_is_const(instr->src[i].src) &&
3450 nir_src_as_uint(instr->src[i].src) == 0) {
3486 bi_emit_texc_offset_ms_index(b, instr);
3496 if (instr->texture_index)
3497 index = bi_iadd_u32(b, index, bi_imm_u32(instr->texture_index), false);
3504 if (instr->sampler_index)
3505 index = bi_iadd_u32(b, index, bi_imm_u32(instr->sampler_index), false);
3526 desc.immediate_indices = direct && (instr->sampler_index < 16);
3529 desc.sampler_index_or_mode = instr->sampler_index;
3530 desc.index = instr->texture_index;
3534 if (direct && instr->sampler_index == instr->texture_index) {
3536 desc.index = instr->texture_index;
3539 desc.index = instr->sampler_index;
3541 bi_imm_u32(instr->texture_index));
3545 desc.index = instr->texture_index;
3549 desc.index = instr->sampler_index;
3566 unsigned res_size = nir_dest_bit_size(instr->dest) == 16 ? 2 : 4;
3579 !nir_tex_instr_has_implicit_derivative(instr), sr_count, 0);
3580 I->register_format = bi_reg_fmt_for_nir(instr->dest_type);
3584 bi_emit_collect_to(b, bi_dest_index(&instr->dest), w,
3585 DIV_ROUND_UP(nir_dest_num_components(instr->dest) * res_size, 4));
3604 bi_emit_tex_valhall(bi_builder *b, nir_tex_instr *instr)
3610 (instr->op == nir_texop_tex) ||
3611 (instr->op == nir_texop_txl) ||
3612 (instr->op == nir_texop_txb);
3617 bi_index sampler = bi_imm_u32(instr->sampler_index);
3618 bi_index texture = bi_imm_u32(instr->texture_index);
3621 for (unsigned i = 0; i < instr->num_srcs; ++i) {
3622 bi_index index = bi_src_index(&instr->src[i].src);
3623 unsigned sz = nir_src_bit_size(instr->src[i].src);
3624 unsigned components = nir_src_num_components(instr->src[i].src);
3626 switch (instr->src[i].src_type) {
3628 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
3641 if (components == 3 && !instr->is_array) {
3647 if (instr->is_array) {
3655 if (nir_src_is_const(instr->src[i].src) &&
3656 nir_src_as_uint(instr->src[i].src) == 0) {
3685 assert(instr->texture_index == 0);
3690 assert(instr->sampler_index == 0);
3702 bi_index offsets = bi_emit_valhall_offsets(b, instr);
3727 unsigned res_size = nir_dest_bit_size(instr->dest) == 16 ? 2 : 4;
3728 enum bi_register_format regfmt = bi_reg_fmt_for_nir(instr->dest_type);
3729 enum bi_dimension dim = valhall_tex_dimension(instr->sampler_dim);
3732 switch (instr->op) {
3737 instr->is_array, dim, regfmt, instr->is_shadow,
3743 instr->is_array, dim, regfmt, explicit_offset,
3748 instr->is_array, dim, instr->component, false,
3749 regfmt, instr->is_shadow, explicit_offset,
3758 bi_emit_collect_to(b, bi_dest_index(&instr->dest), w,
3759 DIV_ROUND_UP(nir_dest_num_components(instr->dest) * res_size, 4));
3767 bi_emit_texs(bi_builder *b, nir_tex_instr *instr)
3769 int coord_idx = nir_tex_instr_src_index(instr, nir_tex_src_coord);
3771 bi_index coords = bi_src_index(&instr->src[coord_idx].src);
3773 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE) {
3777 bi_texs_cube_to(b, nir_dest_bit_size(instr->dest),
3778 bi_dest_index(&instr->dest),
3780 instr->sampler_index, instr->texture_index);
3782 bi_texs_2d_to(b, nir_dest_bit_size(instr->dest),
3783 bi_dest_index(&instr->dest),
3786 instr->op != nir_texop_tex, /* zero LOD */
3787 instr->sampler_index, instr->texture_index);
3790 bi_split_dest(b, instr->dest);
3794 bi_is_simple_tex(nir_tex_instr *instr)
3796 if (instr->op != nir_texop_tex && instr->op != nir_texop_txl)
3799 if (instr->dest_type != nir_type_float32 &&
3800 instr->dest_type != nir_type_float16)
3803 if (instr->is_shadow || instr->is_array)
3806 switch (instr->sampler_dim) {
3814 if (instr->op == nir_texop_txl)
3822 for (unsigned i = 0; i < instr->num_srcs; ++i) {
3823 if (instr->src[i].src_type != nir_tex_src_lod &&
3824 instr->src[i].src_type != nir_tex_src_coord)
3829 unsigned idx_bits = instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE ? 2 : 3;
3830 if (MAX2(instr->sampler_index, instr->texture_index) >= (1 << idx_bits))
3833 int lod_idx = nir_tex_instr_src_index(instr, nir_tex_src_lod);
3837 nir_src lod = instr->src[lod_idx].src;
3842 bi_emit_tex(bi_builder *b, nir_tex_instr *instr)
3844 switch (instr->op) {
3846 bi_load_sysval_to(b, bi_dest_index(&instr->dest),
3847 panfrost_sysval_for_instr(&instr->instr, NULL),
3848 nir_dest_num_components(instr->dest), 0);
3862 bi_emit_tex_valhall(b, instr);
3863 else if (bi_is_simple_tex(instr))
3864 bi_emit_texs(b, instr);
3866 bi_emit_texc(b, instr);
3870 bi_emit_instr(bi_builder *b, struct nir_instr *instr)
3872 switch (instr->type) {
3874 bi_emit_load_const(b, nir_instr_as_load_const(instr));
3878 bi_emit_intrinsic(b, nir_instr_as_intrinsic(instr));
3882 bi_emit_alu(b, nir_instr_as_alu(instr));
3886 bi_emit_tex(b, nir_instr_as_tex(instr));
3890 bi_emit_jump(b, nir_instr_as_jump(instr));
3923 nir_foreach_instr(instr, block) {
3924 bi_emit_instr(&_b, instr);
4263 should_split_wrmask(const nir_instr *instr, UNUSED const void *data)
4265 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
4281 bi_lower_bit_size(const nir_instr *instr, UNUSED void *data)
4283 if (instr->type != nir_instr_type_alu)
4286 nir_alu_instr *alu = nir_instr_as_alu(instr);
4306 bi_vectorize_filter(const nir_instr *instr, const void *data)
4309 if (instr->type != nir_instr_type_alu)
4312 const nir_alu_instr *alu = nir_instr_as_alu(instr);
4336 bi_scalarize_filter(const nir_instr *instr, const void *data)
4338 if (instr->type != nir_instr_type_alu)
4341 const nir_alu_instr *alu = nir_instr_as_alu(instr);
4365 nir_invalidate_divergence(struct nir_builder *b, nir_instr *instr,
4368 return nir_foreach_ssa_def(instr, nir_invalidate_divergence_ssa, NULL);
4389 nir_instr *instr, void *data)
4391 if (instr->type != nir_instr_type_intrinsic)
4394 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
4409 b->cursor = nir_before_instr(&intr->instr);
4419 nir_instr_rewrite_src_ssa(instr, &intr->src[0], replicated);
4590 b->cursor = nir_before_instr(&intr->instr);
4594 nir_instr_rewrite_src_ssa(&intr->instr, &intr->src[0], cast);
4610 b->cursor = nir_before_instr(&intr->instr);
4625 nir_instr *instr, UNUSED void *data)
4627 if (instr->type != nir_instr_type_intrinsic)
4630 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
4654 nir_instr *instr, void *data)
4656 if (instr->type != nir_instr_type_intrinsic)
4659 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
4673 b->cursor = nir_before_instr(&intr->instr);
4695 nir_instr_rewrite_src_ssa(instr, &intr->src[0],
4703 nir_instr_remove(&prev->instr);
4787 bi_gather_texcoords(nir_builder *b, nir_instr *instr, void *data)
4791 if (instr->type != nir_instr_type_tex)
4794 nir_tex_instr *tex = nir_instr_as_tex(instr);