Lines Matching defs:instr

140    nir_foreach_instr(instr, block) {
141 if (instr->type != nir_instr_type_intrinsic)
144 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
414 nir_foreach_instr(instr, block) {
415 nir_emit_instr(instr);
420 fs_visitor::nir_emit_instr(nir_instr *instr)
422 const fs_builder abld = bld.annotate(NULL, instr);
424 switch (instr->type) {
426 nir_emit_alu(abld, nir_instr_as_alu(instr), true);
436 nir_emit_vs_intrinsic(abld, nir_instr_as_intrinsic(instr));
439 nir_emit_tcs_intrinsic(abld, nir_instr_as_intrinsic(instr));
442 nir_emit_tes_intrinsic(abld, nir_instr_as_intrinsic(instr));
445 nir_emit_gs_intrinsic(abld, nir_instr_as_intrinsic(instr));
448 nir_emit_fs_intrinsic(abld, nir_instr_as_intrinsic(instr));
452 nir_emit_cs_intrinsic(abld, nir_instr_as_intrinsic(instr));
460 nir_emit_bs_intrinsic(abld, nir_instr_as_intrinsic(instr));
463 nir_emit_task_intrinsic(abld, nir_instr_as_intrinsic(instr));
466 nir_emit_mesh_intrinsic(abld, nir_instr_as_intrinsic(instr));
474 nir_emit_texture(abld, nir_instr_as_tex(instr));
478 nir_emit_load_const(abld, nir_instr_as_load_const(instr));
489 nir_emit_jump(abld, nir_instr_as_jump(instr));
499 * match instr.
502 fs_visitor::optimize_extract_to_float(nir_alu_instr *instr,
505 if (!instr->src[0].src.is_ssa ||
506 !instr->src[0].src.ssa->parent_instr)
509 if (instr->src[0].src.ssa->parent_instr->type != nir_instr_type_alu)
513 nir_instr_as_alu(instr->src[0].src.ssa->parent_instr);
537 fs_visitor::optimize_frontfacing_ternary(nir_alu_instr *instr,
540 nir_intrinsic_instr *src0 = nir_src_as_intrinsic(instr->src[0].src);
544 if (!nir_src_is_const(instr->src[1].src) ||
545 !nir_src_is_const(instr->src[2].src))
548 const float value1 = nir_src_as_float(instr->src[1].src);
549 const float value2 = nir_src_as_float(instr->src[2].src);
692 nir_alu_instr *instr,
697 need_dest ? get_nir_dest(instr->dest.dest) : bld.null_reg_ud();
700 (nir_alu_type)(nir_op_infos[instr->op].output_type |
701 nir_dest_bit_size(instr->dest.dest)));
703 assert(!instr->dest.saturate);
705 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
707 assert(!instr->src[i].abs);
708 assert(!instr->src[i].negate);
710 op[i] = get_nir_src(instr->src[i].src);
712 (nir_alu_type)(nir_op_infos[instr->op].input_types[i] |
713 nir_src_bit_size(instr->src[i].src)));
721 switch (instr->op) {
738 if (nir_op_infos[instr->op].output_size == 0) {
742 assert(util_bitcount(instr->dest.write_mask) == 1);
743 channel = ffs(instr->dest.write_mask) - 1;
748 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
749 assert(nir_op_infos[instr->op].input_sizes[i] < 2);
750 op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]);
757 fs_visitor::resolve_inot_sources(const fs_builder &bld, nir_alu_instr *instr,
761 nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[i].src);
764 /* The source of the inot is now the source of instr. */
778 nir_alu_instr *instr)
783 nir_alu_instr *inot_instr = nir_src_as_alu_instr(instr->src[0].src);
794 if (nir_dest_bit_size(instr->dest.dest) != 32 ||
816 * If \c instr is not the \c nir_op_fsign, then \c fsign_src is the index of
817 * the source of \c instr that is a \c nir_op_fsign.
820 fs_visitor::emit_fsign(const fs_builder &bld, const nir_alu_instr *instr,
825 assert(instr->op == nir_op_fsign || instr->op == nir_op_fmul);
826 assert(fsign_src < nir_op_infos[instr->op].num_inputs);
828 if (instr->op != nir_op_fsign) {
830 nir_src_as_alu_instr(instr->src[fsign_src].src);
843 (nir_alu_type)(nir_op_infos[instr->op].input_types[0] |
849 if (nir_op_infos[instr->op].output_size == 0) {
853 assert(util_bitcount(instr->dest.write_mask) == 1);
854 channel = ffs(instr->dest.write_mask) - 1;
872 if (instr->op == nir_op_fsign)
892 if (instr->op == nir_op_fsign)
918 if (instr->op == nir_op_fsign) {
941 * \param instr The multiplication instruction
943 * \param fsign_src The source of \c instr that may or may not be a
947 can_fuse_fmul_fsign(nir_alu_instr *instr, unsigned fsign_src)
949 assert(instr->op == nir_op_fmul);
952 nir_src_as_alu_instr(instr->src[fsign_src].src);
956 * 1. instr->src[fsign_src] must be a nir_op_fsign.
970 fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr,
978 fs_reg result = prepare_alu_destination_and_sources(bld, instr, op, need_dest);
986 switch (instr->op) {
1007 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
1013 switch (instr->op) {
1022 for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
1023 if (!instr->src[i].src.is_ssa &&
1024 instr->dest.dest.reg.reg == instr->src[i].src.reg.reg) {
1032 if (!(instr->dest.write_mask & (1 << i)))
1035 if (instr->op == nir_op_mov) {
1037 offset(op[0], bld, instr->src[0].swizzle[i]));
1040 offset(op[i], bld, instr->src[i].swizzle[0]));
1050 if (!(instr->dest.write_mask & (1 << i)))
1061 if (optimize_extract_to_float(instr, result))
1071 if (nir_op_f2f16 == instr->op)
1074 rnd = brw_rnd_mode_from_nir_op(instr->op);
1100 if (try_emit_b2fi_of_inot(bld, result, instr))
1146 nir_alu_instr *extract_instr = nir_src_as_alu_instr(instr->src[0].src);
1206 emit_fsign(bld, instr, result, op, 0);
1276 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1282 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1301 if (can_fuse_fmul_fsign(instr, i)) {
1302 emit_fsign(bld, instr, result, op, i);
1328 const bool ud = instr->op == nir_op_umul_32x16;
1330 assert(nir_dest_bit_size(instr->dest.dest) == 32);
1355 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1361 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1362 if (nir_dest_bit_size(instr->dest.dest) == 32) {
1373 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1389 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1430 const uint32_t bit_size = nir_src_bit_size(instr->src[0].src);
1434 bld.CMP(dest, op[0], op[1], brw_cmod_for_nir_comparison(instr->op));
1463 brw_cmod_for_nir_comparison(instr->op));
1481 nir_alu_instr *inot_src_instr = nir_src_as_alu_instr(instr->src[0].src);
1501 nir_dest_bit_size(instr->dest.dest)));
1541 resolve_inot_sources(bld, instr, op);
1547 resolve_inot_sources(bld, instr, op);
1553 resolve_inot_sources(bld, instr, op);
1588 uint32_t bit_size = nir_src_bit_size(instr->src[0].src);
1594 if (instr->op == nir_op_f2b32) {
1613 zero = instr->op == nir_op_f2b32 ? brw_imm_f(0.0f) : brw_imm_d(0);
1616 zero = instr->op == nir_op_f2b32 ?
1737 if (instr->op == nir_op_unpack_64_2x32_split_x)
1746 if (instr->op == nir_op_unpack_32_2x16_split_x)
1758 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1763 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1768 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1774 assert(nir_dest_bit_size(instr->dest.dest) == 32);
1779 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1801 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1830 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1834 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1838 assert(nir_dest_bit_size(instr->dest.dest) < 64);
1902 if (instr->op == nir_op_sdot_4x8_iadd_sat)
1913 if (instr->op == nir_op_udot_4x8_uadd_sat)
1924 if (instr->op == nir_op_sudot_4x8_iadd_sat)
1951 if (optimize_frontfacing_ternary(instr, result))
1961 unsigned byte = nir_src_as_uint(instr->src[1].src);
1969 if (nir_dest_bit_size(instr->dest.dest) == 64) {
1970 const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8);
1972 if (instr->op == nir_op_extract_i8) {
1991 const brw_reg_type type = brw_int_type(1, instr->op == nir_op_extract_i8);
1999 const brw_reg_type type = brw_int_type(2, instr->op == nir_op_extract_i16);
2000 unsigned word = nir_src_as_uint(instr->src[1].src);
2014 (instr->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) == BRW_NIR_BOOLEAN_NEEDS_RESOLVE) {
2024 nir_load_const_instr *instr)
2027 brw_reg_type_from_bit_size(instr->def.bit_size, BRW_REGISTER_TYPE_D);
2028 fs_reg reg = bld.vgrf(reg_type, instr->def.num_components);
2030 switch (instr->def.bit_size) {
2032 for (unsigned i = 0; i < instr->def.num_components; i++)
2033 bld.MOV(offset(reg, bld, i), setup_imm_b(bld, instr->value[i].i8));
2037 for (unsigned i = 0; i < instr->def.num_components; i++)
2038 bld.MOV(offset(reg, bld, i), brw_imm_w(instr->value[i].i16));
2042 for (unsigned i = 0; i < instr->def.num_components; i++)
2043 bld.MOV(offset(reg, bld, i), brw_imm_d(instr->value[i].i32));
2050 for (unsigned i = 0; i < instr->def.num_components; i++) {
2052 setup_imm_df(bld, instr->value[i].f64));
2055 for (unsigned i = 0; i < instr->def.num_components; i++)
2056 bld.MOV(offset(reg, bld, i), brw_imm_q(instr->value[i].i64));
2064 nir_ssa_values[instr->def.index] = reg;
2670 fs_visitor::get_indirect_offset(nir_intrinsic_instr *instr)
2672 nir_src *offset_src = nir_get_io_offset_src(instr);
2677 * into instr->const_index[0].
2688 nir_intrinsic_instr *instr)
2693 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
2694 dest = get_nir_dest(instr->dest);
2696 switch (instr->intrinsic) {
2702 assert(nir_dest_bit_size(instr->dest) == 32);
2703 fs_reg src = fs_reg(ATTR, nir_intrinsic_base(instr) * 4, dest.type);
2704 src = offset(src, bld, nir_intrinsic_component(instr));
2705 src = offset(src, bld, nir_src_as_uint(instr->src[0]));
2707 for (unsigned i = 0; i < instr->num_components; i++)
2721 nir_emit_intrinsic(bld, instr);
2728 nir_intrinsic_instr *instr)
2731 const nir_src &vertex_src = instr->src[0];
2771 nir_intrinsic_instr *instr)
2775 const nir_src &vertex_src = instr->src[0];
2838 nir_intrinsic_instr *instr)
2849 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
2850 dst = get_nir_dest(instr->dest);
2852 switch (instr->intrinsic) {
2913 assert(nir_dest_bit_size(instr->dest) == 32);
2914 fs_reg indirect_offset = get_indirect_offset(instr);
2915 unsigned imm_offset = instr->const_index[0];
2919 eight_patch ? get_tcs_eight_patch_icp_handle(bld, instr)
2920 : get_tcs_single_patch_icp_handle(bld, instr);
2926 unsigned num_components = instr->num_components;
2927 unsigned first_component = nir_intrinsic_component(instr);
2987 assert(nir_dest_bit_size(instr->dest) == 32);
2988 fs_reg indirect_offset = get_indirect_offset(instr);
2989 unsigned imm_offset = instr->const_index[0];
2990 unsigned first_component = nir_intrinsic_component(instr);
3008 instr->num_components + first_component;
3013 for (unsigned i = 0; i < instr->num_components; i++) {
3020 inst->size_written = instr->num_components * REG_SIZE;
3033 instr->num_components + first_component;
3038 for (unsigned i = 0; i < instr->num_components; i++) {
3045 inst->size_written = instr->num_components * REG_SIZE;
3055 assert(nir_src_bit_size(instr->src[0]) == 32);
3056 fs_reg value = get_nir_src(instr->src[0]);
3057 fs_reg indirect_offset = get_indirect_offset(instr);
3058 unsigned imm_offset = instr->const_index[0];
3059 unsigned mask = instr->const_index[1];
3069 unsigned first_component = nir_intrinsic_component(instr);
3105 nir_emit_intrinsic(bld, instr);
3112 nir_intrinsic_instr *instr)
3118 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3119 dest = get_nir_dest(instr->dest);
3121 switch (instr->intrinsic) {
3134 assert(nir_dest_bit_size(instr->dest) == 32);
3135 fs_reg indirect_offset = get_indirect_offset(instr);
3136 unsigned imm_offset = instr->const_index[0];
3137 unsigned first_component = nir_intrinsic_component(instr);
3147 for (int i = 0; i < instr->num_components; i++) {
3163 instr->num_components + first_component;
3168 for (unsigned i = 0; i < instr->num_components; i++) {
3175 inst->size_written = instr->num_components * REG_SIZE;
3187 unsigned num_components = instr->num_components;
3216 nir_emit_intrinsic(bld, instr);
3223 nir_intrinsic_instr *instr)
3229 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3230 dest = get_nir_dest(instr->dest);
3232 switch (instr->intrinsic) {
3244 emit_gs_input_load(dest, instr->src[0], instr->const_index[0],
3245 instr->src[1], instr->num_components,
3246 nir_intrinsic_component(instr));
3250 emit_gs_vertex(instr->src[0], instr->const_index[0]);
3254 emit_gs_end_primitive(instr->src[0]);
3258 bld.MOV(this->final_gs_vertex_count, get_nir_src(instr->src[0]));
3270 nir_emit_intrinsic(bld, instr);
3446 nir_intrinsic_instr *instr)
3451 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3452 dest = get_nir_dest(instr->dest);
3454 switch (instr->intrinsic) {
3483 gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
3492 const fs_reg src = get_nir_src(instr->src[0]);
3493 const unsigned store_offset = nir_src_as_uint(instr->src[1]);
3494 const unsigned location = nir_intrinsic_base(instr) +
3499 for (unsigned j = 0; j < instr->num_components; j++)
3500 bld.MOV(offset(new_dest, bld, nir_intrinsic_component(instr) + j),
3507 const unsigned l = GET_FIELD(nir_intrinsic_base(instr),
3510 const unsigned load_offset = nir_src_as_uint(instr->src[0]);
3519 for (unsigned j = 0; j < instr->num_components; j++) {
3521 offset(tmp, bld, nir_intrinsic_component(instr) + j));
3539 if (instr->intrinsic == nir_intrinsic_demote_if ||
3540 instr->intrinsic == nir_intrinsic_discard_if ||
3541 instr->intrinsic == nir_intrinsic_terminate_if) {
3542 nir_alu_instr *alu = nir_src_as_alu_instr(instr->src[0]);
3547 (alu->instr.pass_flags & BRW_NIR_BOOLEAN_MASK) != BRW_NIR_BOOLEAN_NEEDS_RESOLVE ||
3585 cmp = bld.CMP(bld.null_reg_f(), get_nir_src(instr->src[0]),
3601 if (instr->intrinsic == nir_intrinsic_terminate ||
3602 instr->intrinsic == nir_intrinsic_terminate_if) {
3621 assert(nir_dest_bit_size(instr->dest) == 32);
3622 unsigned base = nir_intrinsic_base(instr);
3623 unsigned comp = nir_intrinsic_component(instr);
3624 unsigned num_components = instr->num_components;
3651 assert(nir_src_as_uint(instr->src[0]) == 0);
3652 fs_reg interp = interp_reg(nir_intrinsic_base(instr),
3653 nir_intrinsic_component(instr));
3665 enum brw_barycentric_mode bary = brw_barycentric_mode(instr);
3674 (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr);
3676 if (nir_src_is_const(instr->src[0])) {
3677 unsigned msg_data = nir_src_as_uint(instr->src[0]) << 4;
3686 const fs_reg sample_src = retype(get_nir_src(instr->src[0]),
3689 if (nir_src_is_always_uniform(instr->src[0])) {
3741 (enum glsl_interp_mode) nir_intrinsic_interp_mode(instr);
3743 nir_const_value *const_offset = nir_src_as_const_value(instr->src[0]);
3746 assert(nir_src_bit_size(instr->src[0]) == 32);
3757 fs_reg src = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_D);
3774 assert(instr->src[0].ssa &&
3775 instr->src[0].ssa->parent_instr->type == nir_instr_type_intrinsic);
3777 nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3786 dst_xy = retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_F);
3793 for (unsigned int i = 0; i < instr->num_components; i++) {
3795 component(interp_reg(nir_intrinsic_base(instr),
3796 nir_intrinsic_component(instr) + i), 0);
3812 nir_emit_intrinsic(bld, instr);
3819 nir_intrinsic_instr *instr)
3825 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
3826 dest = get_nir_dest(instr->dest);
3828 switch (instr->intrinsic) {
3855 gl_system_value sv = nir_system_value_from_intrinsic(instr->intrinsic);
3865 assert(nir_dest_bit_size(instr->dest) == 32);
3892 nir_emit_shared_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
3897 nir_emit_shared_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
3902 assert(nir_intrinsic_base(instr) == 0);
3904 const unsigned bit_size = nir_dest_bit_size(instr->dest);
3907 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[0]);
3915 assert(nir_dest_bit_size(instr->dest) <= 32);
3916 assert(nir_intrinsic_align(instr) > 0);
3917 if (nir_dest_bit_size(instr->dest) == 32 &&
3918 nir_intrinsic_align(instr) >= 4) {
3919 assert(nir_dest_num_components(instr->dest) <= 4);
3920 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
3924 inst->size_written = instr->num_components * dispatch_width * 4;
3926 assert(nir_dest_num_components(instr->dest) == 1);
3939 assert(nir_intrinsic_base(instr) == 0);
3941 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
3944 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
3951 fs_reg data = get_nir_src(instr->src[0]);
3954 assert(nir_src_bit_size(instr->src[0]) <= 32);
3955 assert(nir_intrinsic_write_mask(instr) ==
3956 (1u << instr->num_components) - 1);
3957 assert(nir_intrinsic_align(instr) > 0);
3958 if (nir_src_bit_size(instr->src[0]) == 32 &&
3959 nir_intrinsic_align(instr) >= 4) {
3960 assert(nir_src_num_components(instr->src[0]) <= 4);
3962 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
3966 assert(nir_src_num_components(instr->src[0]) == 1);
3993 nir_emit_intrinsic(bld, instr);
4024 nir_intrinsic_instr *instr)
4029 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
4030 dest = get_nir_dest(instr->dest);
4032 switch (instr->intrinsic) {
4049 nir_emit_intrinsic(bld, instr);
4127 nir_intrinsic_instr *instr)
4129 fs_reg image = retype(get_nir_src_imm(instr->src[0]), BRW_REGISTER_TYPE_UD);
4137 nir_intrinsic_instr *instr)
4141 instr->intrinsic == nir_intrinsic_store_ssbo ||
4142 instr->intrinsic == nir_intrinsic_store_ssbo_block_intel;
4145 if (nir_src_is_const(instr->src[src])) {
4146 return brw_imm_ud(nir_src_as_uint(instr->src[src]));
4148 return bld.emit_uniformize(get_nir_src(instr->src[src]));
4257 nir_intrinsic_instr *instr)
4264 if (nir_intrinsic_has_memory_scope(instr)) {
4265 switch (nir_intrinsic_memory_scope(instr)) {
4290 fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr)
4293 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
4294 dest = get_nir_dest(instr->dest);
4296 switch (instr->intrinsic) {
4322 const nir_intrinsic_info *info = &nir_intrinsic_infos[instr->intrinsic];
4326 switch (instr->intrinsic) {
4340 get_nir_image_intrinsic_image(bld, instr);
4346 bld.emit_uniformize(get_nir_src(instr->src[0]));
4350 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
4352 brw_imm_ud(nir_image_intrinsic_coord_components(instr));
4355 if (instr->intrinsic == nir_intrinsic_image_load ||
4356 instr->intrinsic == nir_intrinsic_bindless_image_load) {
4357 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4362 inst->size_written = instr->num_components * dispatch_width * 4;
4363 } else if (instr->intrinsic == nir_intrinsic_image_store ||
4364 instr->intrinsic == nir_intrinsic_bindless_image_store) {
4365 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4366 srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(instr->src[3]);
4372 int op = brw_aop_for_nir_intrinsic(instr);
4382 data = get_nir_src(instr->src[3]);
4385 fs_reg sources[2] = { data, get_nir_src(instr->src[4]) };
4401 assert(nir_intrinsic_image_dim(instr) != GLSL_SAMPLER_DIM_CUBE);
4408 fs_reg image = retype(get_nir_src_imm(instr->src[0]),
4412 assert(nir_src_as_uint(instr->src[1]) == 0);
4415 if (instr->intrinsic == nir_intrinsic_image_size)
4433 for (unsigned c = 0; c < instr->dest.ssa.num_components; ++c) {
4443 get_nir_image_intrinsic_image(bld, instr);
4444 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
4446 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4452 inst->size_written = instr->num_components * dispatch_width * 4;
4459 get_nir_image_intrinsic_image(bld, instr);
4460 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
4461 srcs[SURFACE_LOGICAL_SRC_DATA] = get_nir_src(instr->src[2]);
4463 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
4472 assert(nir_intrinsic_execution_scope(instr) == NIR_SCOPE_NONE);
4483 instr->intrinsic == nir_intrinsic_begin_invocation_interlock ?
4486 switch (instr->intrinsic) {
4488 nir_variable_mode modes = nir_intrinsic_memory_modes(instr);
4520 ugm_fence = instr->intrinsic != nir_intrinsic_memory_barrier_shared &&
4521 instr->intrinsic != nir_intrinsic_memory_barrier_image;
4522 slm_fence = instr->intrinsic == nir_intrinsic_group_memory_barrier ||
4523 instr->intrinsic == nir_intrinsic_memory_barrier ||
4524 instr->intrinsic == nir_intrinsic_memory_barrier_shared;
4525 tgm_fence = instr->intrinsic == nir_intrinsic_group_memory_barrier ||
4526 instr->intrinsic == nir_intrinsic_memory_barrier ||
4527 instr->intrinsic == nir_intrinsic_memory_barrier_image;
4528 urb_fence = instr->intrinsic == nir_intrinsic_memory_barrier;
4566 lsc_fence_descriptor_for_intrinsic(devinfo, instr);
4622 instr->intrinsic == nir_intrinsic_end_invocation_interlock ||
4657 if (instr->intrinsic == nir_intrinsic_end_invocation_interlock ||
4685 uint32_t id = nir_intrinsic_param_idx(instr);
4695 assert(instr->const_index[0] % 4 == 0 ||
4696 instr->const_index[0] % type_sz(dest.type) == 0);
4698 fs_reg src(UNIFORM, instr->const_index[0] / 4, dest.type);
4700 if (nir_src_is_const(instr->src[0])) {
4701 unsigned load_offset = nir_src_as_uint(instr->src[0]);
4706 src.offset = load_offset + instr->const_index[0] % 4;
4708 for (unsigned j = 0; j < instr->num_components; j++) {
4712 fs_reg indirect = retype(get_nir_src(instr->src[0]),
4720 assert(instr->const_index[1] >=
4721 instr->num_components * (int) type_sz(dest.type));
4722 unsigned read_size = instr->const_index[1] -
4723 (instr->num_components - 1) * type_sz(dest.type);
4729 for (unsigned j = 0; j < instr->num_components; j++) {
4742 for (unsigned j = 0; j < instr->num_components; j++) {
4757 if (nir_src_is_const(instr->src[0])) {
4758 const unsigned index = nir_src_as_uint(instr->src[0]);
4766 bld.MOV(surf_index, get_nir_src(instr->src[0]));
4770 if (!nir_src_is_const(instr->src[1])) {
4771 fs_reg base_offset = retype(get_nir_src(instr->src[1]),
4774 for (int i = 0; i < instr->num_components; i++)
4777 nir_dest_bit_size(instr->dest) / 8);
4790 const unsigned load_offset = nir_src_as_uint(instr->src[1]);
4793 if (nir_src_is_const(instr->src[0])) {
4794 const unsigned ubo_block = nir_src_as_uint(instr->src[0]);
4811 for (unsigned i = 0; i < instr->num_components; i++) {
4825 for (unsigned c = 0; c < instr->num_components;) {
4828 const unsigned count = MIN2(instr->num_components - c,
4852 assert(nir_dest_bit_size(instr->dest) <= 32);
4853 assert(nir_intrinsic_align(instr) > 0);
4855 srcs[A64_LOGICAL_ADDRESS] = get_nir_src(instr->src[0]);
4858 brw_imm_ud(nir_intrinsic_access(instr) & ACCESS_INCLUDE_HELPERS);
4860 if (nir_dest_bit_size(instr->dest) == 32 &&
4861 nir_intrinsic_align(instr) >= 4) {
4862 assert(nir_dest_num_components(instr->dest) <= 4);
4864 srcs[A64_LOGICAL_ARG] = brw_imm_ud(instr->num_components);
4869 inst->size_written = instr->num_components *
4872 const unsigned bit_size = nir_dest_bit_size(instr->dest);
4873 assert(nir_dest_num_components(instr->dest) == 1);
4888 assert(nir_src_bit_size(instr->src[0]) <= 32);
4889 assert(nir_intrinsic_write_mask(instr) ==
4890 (1u << instr->num_components) - 1);
4891 assert(nir_intrinsic_align(instr) > 0);
4894 srcs[A64_LOGICAL_ADDRESS] = get_nir_src(instr->src[1]);
4896 brw_imm_ud(nir_intrinsic_access(instr) & ACCESS_INCLUDE_HELPERS);
4898 if (nir_src_bit_size(instr->src[0]) == 32 &&
4899 nir_intrinsic_align(instr) >= 4) {
4900 assert(nir_src_num_components(instr->src[0]) <= 4);
4902 srcs[A64_LOGICAL_SRC] = get_nir_src(instr->src[0]); /* Data */
4903 srcs[A64_LOGICAL_ARG] = brw_imm_ud(instr->num_components);
4908 assert(nir_src_num_components(instr->src[0]) == 1);
4909 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
4913 bld.MOV(tmp, retype(get_nir_src(instr->src[0]), data_type));
4916 srcs[A64_LOGICAL_ARG] = brw_imm_ud(nir_src_bit_size(instr->src[0]));
4934 nir_emit_global_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
4940 nir_emit_global_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
4944 assert(nir_dest_bit_size(instr->dest) == 32);
4945 assert(instr->num_components == 8 || instr->num_components == 16);
4947 const fs_builder ubld = bld.exec_all().group(instr->num_components, 0);
4950 bool is_pred_const = nir_src_is_const(instr->src[1]);
4951 if (is_pred_const && nir_src_as_uint(instr->src[1]) == 0) {
4958 fs_reg addr = bld.emit_uniformize(get_nir_src(instr->src[0]));
4967 fs_reg pred = bld.emit_uniformize(get_nir_src(instr->src[1]));
4980 srcs[A64_LOGICAL_ARG] = brw_imm_ud(instr->num_components);
4999 for (unsigned i = 0; i < instr->num_components; i++) {
5009 const unsigned bit_size = nir_dest_bit_size(instr->dest);
5012 get_nir_ssbo_intrinsic_index(bld, instr);
5013 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
5021 assert(nir_dest_bit_size(instr->dest) <= 32);
5022 assert(nir_intrinsic_align(instr) > 0);
5023 if (nir_dest_bit_size(instr->dest) == 32 &&
5024 nir_intrinsic_align(instr) >= 4) {
5025 assert(nir_dest_num_components(instr->dest) <= 4);
5026 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
5030 inst->size_written = instr->num_components * dispatch_width * 4;
5032 assert(nir_dest_num_components(instr->dest) == 1);
5046 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
5049 get_nir_ssbo_intrinsic_index(bld, instr);
5050 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[2]);
5054 fs_reg data = get_nir_src(instr->src[0]);
5057 assert(nir_src_bit_size(instr->src[0]) <= 32);
5058 assert(nir_intrinsic_write_mask(instr) ==
5059 (1u << instr->num_components) - 1);
5060 assert(nir_intrinsic_align(instr) > 0);
5061 if (nir_src_bit_size(instr->src[0]) == 32 &&
5062 nir_intrinsic_align(instr) >= 4) {
5063 assert(nir_src_num_components(instr->src[0]) <= 4);
5065 srcs[SURFACE_LOGICAL_SRC_IMM_ARG] = brw_imm_ud(instr->num_components);
5069 assert(nir_src_num_components(instr->src[0]) == 1);
5082 assert(nir_src_bit_size(instr->src[0]) == 32);
5083 fs_reg src = get_nir_src(instr->src[0]);
5085 unsigned store_offset = nir_src_as_uint(instr->src[1]);
5086 unsigned num_components = instr->num_components;
5087 unsigned first_component = nir_intrinsic_component(instr);
5089 fs_reg new_dest = retype(offset(outputs[instr->const_index[0]], bld,
5108 nir_emit_ssbo_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
5114 nir_emit_ssbo_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
5118 assert(nir_src_num_components(instr->src[0]) == 1);
5119 unsigned ssbo_index = nir_src_is_const(instr->src[0]) ?
5120 nir_src_as_uint(instr->src[0]) : 0;
5179 assert(nir_dest_num_components(instr->dest) == 1);
5180 const unsigned bit_size = nir_dest_bit_size(instr->dest);
5199 const fs_reg nir_addr = get_nir_src(instr->src[0]);
5205 assert(nir_dest_num_components(instr->dest) == 1);
5206 assert(nir_dest_bit_size(instr->dest) <= 32);
5207 assert(nir_intrinsic_align(instr) > 0);
5208 if (nir_dest_bit_size(instr->dest) == 32 &&
5209 nir_intrinsic_align(instr) >= 4) {
5211 assert(nir_dest_bit_size(instr->dest) == 32 &&
5212 nir_intrinsic_align(instr) >= 4);
5245 assert(nir_src_num_components(instr->src[0]) == 1);
5246 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
5272 const fs_reg nir_addr = get_nir_src(instr->src[1]);
5274 fs_reg data = get_nir_src(instr->src[0]);
5277 assert(nir_src_num_components(instr->src[0]) == 1);
5278 assert(nir_src_bit_size(instr->src[0]) <= 32);
5279 assert(nir_intrinsic_write_mask(instr) == 1);
5280 assert(nir_intrinsic_align(instr) > 0);
5281 if (nir_src_bit_size(instr->src[0]) == 32 &&
5282 nir_intrinsic_align(instr) >= 4) {
5350 bld.CMP(bld.null_reg_d(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ);
5382 bld.CMP(bld.null_reg_d(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ);
5402 fs_reg value = get_nir_src(instr->src[0]);
5403 if (instr->intrinsic == nir_intrinsic_vote_feq) {
5404 const unsigned bit_size = nir_src_bit_size(instr->src[0]);
5443 const fs_reg value = retype(get_nir_src(instr->src[0]),
5458 if (instr->dest.ssa.bit_size > 32) {
5468 const fs_reg value = get_nir_src(instr->src[0]);
5469 const fs_reg invocation = get_nir_src(instr->src[1]);
5480 const fs_reg value = get_nir_src(instr->src[0]);
5486 const fs_reg value = get_nir_src(instr->src[0]);
5487 const fs_reg index = get_nir_src(instr->src[1]);
5510 const fs_reg value = get_nir_src(instr->src[0]);
5511 const unsigned index = nir_src_as_uint(instr->src[1]);
5519 const fs_reg value = get_nir_src(instr->src[0]);
5527 assert(nir_src_bit_size(instr->src[0]) == 32);
5549 const fs_reg value = get_nir_src(instr->src[0]);
5550 if (nir_src_bit_size(instr->src[0]) == 32) {
5570 const fs_reg value = get_nir_src(instr->src[0]);
5571 if (nir_src_bit_size(instr->src[0]) == 32) {
5591 fs_reg src = get_nir_src(instr->src[0]);
5592 nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr);
5593 unsigned cluster_size = nir_intrinsic_cluster_size(instr);
5600 nir_src_bit_size(instr->src[0])));
5640 fs_reg src = get_nir_src(instr->src[0]);
5641 nir_op redop = (nir_op)nir_intrinsic_reduction_op(instr);
5646 nir_src_bit_size(instr->src[0])));
5659 if (instr->intrinsic == nir_intrinsic_exclusive_scan) {
5680 assert(nir_dest_bit_size(instr->dest) == 32);
5682 fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[0]));
5688 const unsigned total = instr->num_components * dispatch_width;
5716 assert(nir_src_bit_size(instr->src[0]) == 32);
5718 fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[1]));
5719 fs_reg src = get_nir_src(instr->src[0]);
5725 const unsigned total = instr->num_components * dispatch_width;
5754 assert(nir_dest_bit_size(instr->dest) == 32);
5757 instr->intrinsic == nir_intrinsic_load_ssbo_block_intel;
5758 fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[is_ssbo ? 1 : 0]));
5762 get_nir_ssbo_intrinsic_index(bld, instr) : fs_reg(brw_imm_ud(GFX7_BTI_SLM));
5769 const unsigned total = instr->num_components * dispatch_width;
5794 assert(nir_src_bit_size(instr->src[0]) == 32);
5797 instr->intrinsic == nir_intrinsic_store_ssbo_block_intel;
5799 fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[is_ssbo ? 2 : 1]));
5800 fs_reg src = get_nir_src(instr->src[0]);
5804 get_nir_ssbo_intrinsic_index(bld, instr) : fs_reg(brw_imm_ud(GFX7_BTI_SLM));
5811 const unsigned total = instr->num_components * dispatch_width;
5853 switch (nir_intrinsic_base(instr)) {
5936 bld.emit_uniformize(get_nir_src(instr->src[0])),
5937 get_nir_src(instr->src[1]));
5954 const bool synchronous = nir_intrinsic_synchronous(instr);
5965 fs_reg globals = get_nir_src(instr->src[0]);
5967 srcs[RT_LOGICAL_SRC_BVH_LEVEL] = get_nir_src(instr->src[1]);
5968 srcs[RT_LOGICAL_SRC_TRACE_RAY_CONTROL] = get_nir_src(instr->src[2]);
5988 assert(instr->intrinsic < nir_num_intrinsics);
5989 fprintf(stderr, "intrinsic: %s\n", nir_intrinsic_infos[instr->intrinsic].name);
5997 int op, nir_intrinsic_instr *instr)
6004 assert(nir_dest_bit_size(instr->dest) == 32 ||
6005 (nir_dest_bit_size(instr->dest) == 64 && devinfo->has_lsc));
6008 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
6009 dest = get_nir_dest(instr->dest);
6012 srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr);
6013 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
6020 data = get_nir_src(instr->src[2]);
6024 fs_reg sources[2] = { data, get_nir_src(instr->src[3]) };
6038 int op, nir_intrinsic_instr *instr)
6041 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
6042 dest = get_nir_dest(instr->dest);
6045 srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr);
6046 srcs[SURFACE_LOGICAL_SRC_ADDRESS] = get_nir_src(instr->src[1]);
6051 fs_reg data = get_nir_src(instr->src[2]);
6054 fs_reg sources[2] = { data, get_nir_src(instr->src[3]) };
6068 int op, nir_intrinsic_instr *instr)
6071 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
6072 dest = get_nir_dest(instr->dest);
6082 data = get_nir_src(instr->src[1]);
6085 fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
6092 if (nir_src_is_const(instr->src[0])) {
6094 brw_imm_ud(instr->const_index[0] + nir_src_as_uint(instr->src[0]));
6098 retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD),
6099 brw_imm_ud(instr->const_index[0]));
6110 int op, nir_intrinsic_instr *instr)
6113 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
6114 dest = get_nir_dest(instr->dest);
6122 fs_reg data = get_nir_src(instr->src[1]);
6125 fs_reg sources[2] = { data, get_nir_src(instr->src[2]) };
6132 if (nir_src_is_const(instr->src[0])) {
6134 brw_imm_ud(instr->const_index[0] + nir_src_as_uint(instr->src[0]));
6138 retype(get_nir_src(instr->src[0]), BRW_REGISTER_TYPE_UD),
6139 brw_imm_ud(instr->const_index[0]));
6162 int op, nir_intrinsic_instr *instr)
6165 if (nir_intrinsic_infos[instr->intrinsic].has_dest)
6166 dest = get_nir_dest(instr->dest);
6168 fs_reg addr = get_nir_src(instr->src[0]);
6172 data = expand_to_32bit(bld, get_nir_src(instr->src[1]));
6178 expand_to_32bit(bld, get_nir_src(instr->src[2]))
6190 switch (nir_dest_bit_size(instr->dest)) {
6213 int op, nir_intrinsic_instr *instr)
6215 assert(nir_intrinsic_infos[instr->intrinsic].has_dest);
6216 fs_reg dest = get_nir_dest(instr->dest);
6218 fs_reg addr = get_nir_src(instr->src[0]);
6221 fs_reg data = expand_to_32bit(bld, get_nir_src(instr->src[1]));
6227 expand_to_32bit(bld, get_nir_src(instr->src[2]))
6239 switch (nir_dest_bit_size(instr->dest)) {
6261 fs_visitor::nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr)
6263 unsigned texture = instr->texture_index;
6264 unsigned sampler = instr->sampler_index;
6274 if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6278 for (unsigned i = 0; i < instr->num_srcs; i++) {
6279 fs_reg src = get_nir_src(instr->src[i].src);
6280 switch (instr->src[i].src_type) {
6283 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F);
6289 switch (instr->op) {
6305 if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && devinfo->verx10 == 125)
6306 assert(instr->coord_components >= 3u);
6310 lod_components = nir_tex_instr_src_size(instr, i);
6316 switch (instr->op) {
6319 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_UD);
6323 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_D);
6327 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F);
6333 retype(get_nir_src_imm(instr->src[i].src), BRW_REGISTER_TYPE_F);
6341 if (brw_texture_offset(instr, i, &offset_bits)) {
6375 assert(nir_tex_instr_src_index(instr, nir_tex_src_texture_offset) == -1);
6381 assert(nir_tex_instr_src_index(instr, nir_tex_src_sampler_offset) == -1);
6387 assert(instr->op == nir_texop_txf_ms);
6397 (instr->op == nir_texop_txf_ms ||
6398 instr->op == nir_texop_samples_identical)) {
6403 instr->coord_components,
6411 srcs[TEX_LOGICAL_SRC_COORD_COMPONENTS] = brw_imm_d(instr->coord_components);
6415 switch (instr->op) {
6464 fs_reg dst = retype(get_nir_dest(instr->dest), BRW_REGISTER_TYPE_D);
6486 if (instr->op == nir_texop_tg4) {
6487 if (instr->component == 1 &&
6494 header_bits |= instr->component << 16;
6498 fs_reg dst = bld.vgrf(brw_type_for_nir_type(devinfo, instr->dest_type), 4);
6502 const unsigned dest_size = nir_tex_instr_dest_size(instr);
6504 instr->op != nir_texop_tg4 && instr->op != nir_texop_query_levels) {
6505 unsigned write_mask = instr->dest.is_ssa ?
6506 nir_ssa_def_components_read(&instr->dest.ssa):
6522 if (instr->op == nir_texop_query_levels) {
6539 } else if (instr->op == nir_texop_txs &&
6547 bld.LOAD_PAYLOAD(get_nir_dest(instr->dest), nir_dest, dest_size, 0);
6551 fs_visitor::nir_emit_jump(const fs_builder &bld, nir_jump_instr *instr)
6553 switch (instr->type) {