Lines Matching defs:bld
50 bld.emit(SHADER_OPCODE_HALT_TARGET);
92 fs_reg reg = bld.vgrf(BRW_REGISTER_TYPE_F, 4 * reg_size);
95 outputs[loc + i] = offset(reg, bld, 4 * i);
172 const fs_builder abld = v->bld.annotate("gl_InvocationID", NULL);
215 v->bld.annotate("gl_HelperInvocation", NULL);
291 const fs_builder abld = bld.annotate("gl_SubgroupInvocation", NULL);
324 nir_locals[reg->index] = bld.vgrf(reg_type, size);
370 cond_reg = offset(cond_reg, bld, cond->src[0].swizzle[0]);
377 fs_inst *inst = bld.MOV(bld.null_reg_d(),
381 bld.IF(BRW_PREDICATE_NORMAL)->predicate_inverse = invert;
386 bld.emit(BRW_OPCODE_ELSE);
390 bld.emit(BRW_OPCODE_ENDIF);
400 bld.emit(BRW_OPCODE_DO);
404 bld.emit(BRW_OPCODE_WHILE);
422 const fs_builder abld = bld.annotate(NULL, instr);
530 op0 = offset(op0, bld, src0->src[0].swizzle[0]);
532 bld.MOV(result, subscript(op0, type, element));
572 bld.OR(subscript(tmp, BRW_REGISTER_TYPE_W, 1),
593 bld.OR(subscript(tmp, BRW_REGISTER_TYPE_W, 1),
614 bld.OR(tmp, g1_6, brw_imm_d(0x3f800000));
616 bld.AND(retype(result, BRW_REGISTER_TYPE_D), tmp, brw_imm_d(0xbf800000));
622 emit_find_msb_using_lzd(const fs_builder &bld,
650 temp = bld.vgrf(BRW_REGISTER_TYPE_D);
652 bld.ASR(temp, src, brw_imm_d(31));
653 bld.XOR(temp, temp, src);
656 bld.LZD(retype(result, BRW_REGISTER_TYPE_UD),
664 inst = bld.ADD(result, retype(result, BRW_REGISTER_TYPE_D), brw_imm_d(31));
691 fs_visitor::prepare_alu_destination_and_sources(const fs_builder &bld,
697 need_dest ? get_nir_dest(instr->dest.dest) : bld.null_reg_ud();
745 result = offset(result, bld, channel);
750 op[i] = offset(op[i], bld, instr->src[i].swizzle[channel]);
757 fs_visitor::resolve_inot_sources(const fs_builder &bld, nir_alu_instr *instr,
765 prepare_alu_destination_and_sources(bld, inot_instr, &op[i], false);
776 fs_visitor::try_emit_b2fi_of_inot(const fs_builder &bld,
803 prepare_alu_destination_and_sources(bld, inot_instr, &op, false);
808 bld.ADD(result, op, brw_imm_d(1));
820 fs_visitor::emit_fsign(const fs_builder &bld, const nir_alu_instr *instr,
857 op[0] = offset(op[0], bld, fsign_instr->src[0].swizzle[channel]);
866 bld.CMP(bld.null_reg_f(), op[0], zero, BRW_CONDITIONAL_NZ);
870 bld.AND(result, op[0], brw_imm_uw(0x8000u));
873 inst = bld.OR(result, result, brw_imm_uw(0x3c00u));
876 inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UW));
886 bld.CMP(bld.null_reg_f(), op[0], brw_imm_f(0.0f), BRW_CONDITIONAL_NZ);
890 bld.AND(result, op[0], brw_imm_ud(0x80000000u));
893 inst = bld.OR(result, result, brw_imm_ud(0x3f800000u));
896 inst = bld.XOR(result, result, retype(op[1], BRW_REGISTER_TYPE_UD));
909 bld.MOV(zero, setup_imm_df(bld, 0.0));
910 bld.CMP(bld.null_reg_df(), op[0], zero, BRW_CONDITIONAL_NZ);
912 bld.MOV(result, zero);
915 bld.AND(r, subscript(op[0], BRW_REGISTER_TYPE_UD, 1),
920 bld.OR(r, r, brw_imm_ud(0x3ff00000u)));
929 inst = bld.XOR(result_int64, result_int64,
970 fs_visitor::nir_emit_alu(const fs_builder &bld, nir_alu_instr *instr,
975 bld.shader->nir->info.float_controls_execution_mode;
978 fs_reg result = prepare_alu_destination_and_sources(bld, instr, op, need_dest);
1026 temp = bld.vgrf(result.type, 4);
1036 bld.MOV(offset(temp, bld, i),
1037 offset(op[0], bld, instr->src[0].swizzle[i]));
1039 bld.MOV(offset(temp, bld, i),
1040 offset(op[i], bld, instr->src[i].swizzle[0]));
1053 bld.MOV(offset(result, bld, i), offset(temp, bld, i));
1063 inst = bld.MOV(result, op[0]);
1077 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(), brw_imm_d(rnd));
1089 inst = bld.MOV(result, op[0]);
1100 if (try_emit_b2fi_of_inot(bld, result, instr))
1132 inst = bld.MOV(result, op[0]);
1150 prepare_alu_destination_and_sources(bld, extract_instr, op, false);
1159 prepare_alu_destination_and_sources(bld, extract_instr, op, false);
1169 inst = bld.MOV(result, op[0]);
1174 inst = bld.MOV(result, op[0]);
1181 inst = bld.MOV(result, op[0]);
1188 inst = bld.MOV(result, op[0]);
1195 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1202 inst = bld.MOV(result, op[0]);
1206 emit_fsign(bld, instr, result, op, 0);
1210 inst = bld.emit(SHADER_OPCODE_RCP, result, op[0]);
1214 inst = bld.emit(SHADER_OPCODE_EXP2, result, op[0]);
1218 inst = bld.emit(SHADER_OPCODE_LOG2, result, op[0]);
1222 inst = bld.emit(SHADER_OPCODE_SIN, result, op[0]);
1226 inst = bld.emit(SHADER_OPCODE_COS, result, op[0]);
1230 inst = bld.emit(FS_OPCODE_DDX_FINE, result, op[0]);
1234 inst = bld.emit(FS_OPCODE_DDX_COARSE, result, op[0]);
1237 inst = bld.emit(FS_OPCODE_DDY_FINE, result, op[0]);
1241 inst = bld.emit(FS_OPCODE_DDY_COARSE, result, op[0]);
1248 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1253 inst = bld.ADD(result, op[0], op[1]);
1257 inst = bld.ADD3(result, op[0], op[1], op[2]);
1262 inst = bld.ADD(result, op[0], op[1]);
1267 bld.emit(SHADER_OPCODE_ISUB_SAT, result, op[0], op[1]);
1271 bld.emit(SHADER_OPCODE_USUB_SAT, result, op[0], op[1]);
1277 inst = bld.AVG(result, op[0], op[1]);
1283 fs_reg tmp = bld.vgrf(result.type);
1291 bld.XOR(tmp, op[0], op[1]);
1292 bld.AND(tmp, tmp, retype(brw_imm_ud(1), result.type));
1293 bld.AVG(result, op[0], op[1]);
1294 inst = bld.ADD(result, result, tmp);
1302 emit_fsign(bld, instr, result, op, i);
1314 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1318 inst = bld.MUL(result, op[0], op[1]);
1323 bld.MUL(result, op[0], op[1]);
1350 bld.MUL(result, retype(op[0], dword_type), op[1]);
1356 bld.MUL(result, op[0], op[1]);
1363 bld.emit(SHADER_OPCODE_MULH, result, op[0], op[1]);
1365 fs_reg tmp = bld.vgrf(brw_reg_type_from_bit_size(32, op[0].type));
1366 bld.MUL(tmp, op[0], op[1]);
1367 bld.MOV(result, subscript(tmp, result.type, 1));
1374 bld.emit(SHADER_OPCODE_INT_QUOTIENT, result, op[0], op[1]);
1390 bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]);
1395 bld.emit(SHADER_OPCODE_INT_REMAINDER, result, op[0], op[1]);
1398 inst = bld.MOV(bld.null_reg_d(), result);
1410 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_D);
1411 inst = bld.XOR(tmp, op[0], op[1]);
1419 inst = bld.ADD(result, result, op[1]);
1432 dest = bld.vgrf(op[0].type, 1);
1434 bld.CMP(dest, op[0], op[1], brw_cmod_for_nir_comparison(instr->op));
1437 bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0));
1445 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type));
1460 dest = bld.vgrf(op[0].type, 1);
1462 bld.CMP(dest, op[0], op[1],
1466 bld.MOV(result, subscript(dest, BRW_REGISTER_TYPE_UD, 0));
1474 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), retype(dest, src_type));
1490 prepare_alu_destination_and_sources(bld, inot_src_instr, op, false);
1491 resolve_inot_sources(bld, inot_src_instr, op);
1520 bld.AND(result, op[0], op[1]);
1524 bld.OR(result, op[0], op[1]);
1528 bld.XOR(result, op[0], op[1]);
1537 bld.NOT(result, op[0]);
1541 resolve_inot_sources(bld, instr, op);
1543 bld.XOR(result, op[0], op[1]);
1547 resolve_inot_sources(bld, instr, op);
1549 bld.OR(result, op[0], op[1]);
1553 resolve_inot_sources(bld, instr, op);
1555 bld.AND(result, op[0], op[1]);
1579 inst = bld.emit(SHADER_OPCODE_SQRT, result, op[0]);
1583 inst = bld.emit(SHADER_OPCODE_RSQ, result, op[0]);
1597 bld.MOV(zero, setup_imm_df(bld, 0.0));
1601 bld.MOV(zero, brw_imm_q(0));
1608 bld.CMP(tmp, op[0], zero, BRW_CONDITIONAL_NZ);
1609 bld.MOV(result, subscript(tmp, BRW_REGISTER_TYPE_UD, 0));
1619 bld.CMP(result, op[0], zero, BRW_CONDITIONAL_NZ);
1625 inst = bld.RNDZ(result, op[0]);
1629 bld.ADD(result, result, brw_imm_f(1.0f)));
1630 inst = bld.MOV(result, result); /* for potential saturation */
1637 bld.RNDD(temp, op[0]);
1639 inst = bld.MOV(result, temp);
1643 inst = bld.RNDD(result, op[0]);
1646 inst = bld.FRC(result, op[0]);
1649 inst = bld.RNDE(result, op[0]);
1653 bld.ADD(result, result, brw_imm_f(1.0f)));
1654 inst = bld.MOV(result, result); /* for potential saturation */
1659 fs_reg tmp16 = bld.vgrf(BRW_REGISTER_TYPE_D);
1660 fs_reg tmp32 = bld.vgrf(BRW_REGISTER_TYPE_F);
1661 fs_reg zero = bld.vgrf(BRW_REGISTER_TYPE_F);
1671 bld.CMP(bld.null_reg_f(), abs_src0, brw_imm_f(ldexpf(1.0, -14)),
1674 bld.AND(retype(zero, BRW_REGISTER_TYPE_UD),
1678 bld.emit(BRW_OPCODE_F32TO16, tmp16, op[0]);
1679 bld.emit(BRW_OPCODE_F16TO32, tmp32, tmp16);
1681 inst = bld.SEL(result, zero, tmp32);
1689 inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_L);
1695 inst = bld.emit_minmax(result, op[0], op[1], BRW_CONDITIONAL_GE);
1714 inst = bld.emit(BRW_OPCODE_F16TO32, result,
1722 inst = bld.emit(BRW_OPCODE_F16TO32, result,
1728 bld.emit(FS_OPCODE_PACK, result, op[0], op[1]);
1732 bld.emit(FS_OPCODE_PACK, result, op, 4);
1738 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 0));
1740 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UD, 1));
1747 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 0));
1749 bld.MOV(result, subscript(op[0], BRW_REGISTER_TYPE_UW, 1));
1754 inst = bld.emit(SHADER_OPCODE_POW, result, op[0], op[1]);
1759 bld.BFREV(result, op[0]);
1764 bld.CBIT(result, op[0]);
1769 emit_find_msb_using_lzd(bld, result, op[0], false);
1775 bld.LZD(retype(result, BRW_REGISTER_TYPE_UD), op[0]);
1782 emit_find_msb_using_lzd(bld, result, op[0], true);
1784 bld.FBH(retype(result, BRW_REGISTER_TYPE_UD), op[0]);
1791 bld.CMP(bld.null_reg_d(), result, brw_imm_d(-1), BRW_CONDITIONAL_NZ);
1793 inst = bld.ADD(result, result, brw_imm_d(31));
1818 bld.AND(temp, src, negated_src);
1819 emit_find_msb_using_lzd(bld, result, temp, false);
1821 bld.FBL(result, op[0]);
1831 bld.BFE(result, op[2], op[1], op[0]);
1835 bld.BFI1(result, op[0], op[1]);
1839 bld.BFI2(result, op[0], op[1], op[2]);
1875 bld.SHL(result, op[0], op[1]);
1878 bld.ASR(result, op[0], op[1]);
1881 bld.SHR(result, op[0], op[1]);
1885 bld.ROL(result, op[0], op[1]);
1888 bld.ROR(result, op[0], op[1]);
1892 bld.emit(FS_OPCODE_PACK_HALF_2x16_SPLIT, result, op[0], op[1]);
1897 inst = bld.DP4A(retype(result, BRW_REGISTER_TYPE_D),
1908 inst = bld.DP4A(retype(result, BRW_REGISTER_TYPE_UD),
1919 inst = bld.DP4A(retype(result, BRW_REGISTER_TYPE_D),
1932 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1936 inst = bld.MAD(result, op[2], op[1], op[0]);
1943 bld.emit(SHADER_OPCODE_RND_MODE, bld.null_reg_ud(),
1947 inst = bld.LRP(result, op[0], op[1], op[2]);
1954 bld.CMP(bld.null_reg_d(), op[0], brw_imm_d(0), BRW_CONDITIONAL_NZ);
1955 inst = bld.SEL(result, op[1], op[2]);
1974 fs_reg w_temp = bld.vgrf(BRW_REGISTER_TYPE_W);
1975 bld.MOV(w_temp, subscript(op[0], type, byte));
1976 bld.MOV(result, w_temp);
1981 bld.SHR(result,
1986 bld.AND(result,
1992 bld.MOV(result, subscript(op[0], type, byte));
2001 bld.MOV(result, subscript(op[0], type, word));
2016 bld.AND(masked, result, brw_imm_d(1));
2018 bld.MOV(retype(result, BRW_REGISTER_TYPE_D), masked);
2023 fs_visitor::nir_emit_load_const(const fs_builder &bld,
2028 fs_reg reg = bld.vgrf(reg_type, instr->def.num_components);
2033 bld.MOV(offset(reg, bld, i), setup_imm_b(bld, instr->value[i].i8));
2038 bld.MOV(offset(reg, bld, i), brw_imm_w(instr->value[i].i16));
2043 bld.MOV(offset(reg, bld, i), brw_imm_d(instr->value[i].i32));
2051 bld.MOV(retype(offset(reg, bld, i), BRW_REGISTER_TYPE_DF),
2052 setup_imm_df(bld, instr->value[i].f64));
2056 bld.MOV(offset(reg, bld, i), brw_imm_q(instr->value[i].i64));
2075 reg = bld.vgrf(reg_type, src.ssa->num_components);
2082 reg = offset(nir_locals[src.reg.reg->index], bld,
2128 bld.vgrf(reg_type, dest.ssa.num_components);
2129 bld.UNDEF(nir_ssa_values[dest.ssa.index]);
2134 return offset(nir_locals[dest.reg.reg->index], bld,
2140 fs_visitor::emit_percomp(const fs_builder &bld, const fs_inst &inst,
2148 new_inst->dst = offset(new_inst->dst, bld, i);
2151 new_inst->src[j] = offset(new_inst->src[j], bld, i);
2153 bld.emit(new_inst);
2158 emit_pixel_interpolater_send(const fs_builder &bld,
2166 brw_wm_prog_data(bld.shader->stage_prog_data);
2168 fs_inst *inst = bld.emit(opcode, dst, src, desc);
2189 intexp2(const fs_builder &bld, const fs_reg &x)
2193 fs_reg result = bld.vgrf(x.type, 1);
2194 fs_reg one = bld.vgrf(x.type, 1);
2196 bld.MOV(one, retype(brw_imm_d(1), one.type));
2197 bld.SHL(result, one, x);
2247 const fs_builder abld = bld.annotate("end primitive");
2250 fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2269 const fs_builder abld = bld.annotate("emit control data bits");
2270 const fs_builder fwa_bld = bld.exec_all();
2313 fs_reg dword_index = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2314 fs_reg prev_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2330 fs_reg channel = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2391 const fs_builder abld = bld.annotate("set stream control data bits", NULL);
2394 fs_reg sid = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2398 fs_reg shift_count = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2406 fs_reg mask = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2444 bld.annotate("emit vertex: emit control data bits");
2468 abld.AND(bld.null_reg_d(), vertex_count,
2476 abld.CMP(bld.null_reg_d(), vertex_count, brw_imm_ud(0u),
2526 bld.MOV(offset(dst, bld, i),
2536 fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2556 fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_UW, 1);
2557 fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2558 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2559 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2562 bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210)));
2564 bld.SHL(channel_offsets, sequence, brw_imm_ud(2u));
2566 bld.SHL(vertex_offset_bytes,
2569 bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets);
2575 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2586 bld.MOV(icp_handle,
2594 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2597 bld.SHL(icp_offset_bytes,
2605 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2623 fs_reg tmp = bld.vgrf(dst.type, read_components);
2624 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, tmp, srcs,
2629 bld.MOV(offset(dst, bld, i),
2630 offset(tmp, bld, i + first_component));
2633 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, dst, srcs,
2643 fs_reg tmp = bld.vgrf(dst.type, read_components);
2650 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, tmp,
2655 bld.MOV(offset(dst, bld, i),
2656 offset(tmp, bld, i + first_component));
2659 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, dst,
2687 fs_visitor::nir_emit_vs_intrinsic(const fs_builder &bld,
2704 src = offset(src, bld, nir_intrinsic_component(instr));
2705 src = offset(src, bld, nir_src_as_uint(instr->src[0]));
2708 bld.MOV(offset(dest, bld, i), offset(src, bld, i));
2721 nir_emit_intrinsic(bld, instr);
2727 fs_visitor::get_tcs_single_patch_icp_handle(const fs_builder &bld,
2737 icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2739 bld.MOV(icp_handle,
2752 icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2755 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2756 bld.SHL(vertex_offset_bytes,
2761 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2770 fs_visitor::get_tcs_eight_patch_icp_handle(const fs_builder &bld,
2796 fs_reg icp_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2797 fs_reg sequence = bld.vgrf(BRW_REGISTER_TYPE_UW, 1);
2798 fs_reg channel_offsets = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2799 fs_reg vertex_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2800 fs_reg icp_offset_bytes = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2803 bld.MOV(sequence, fs_reg(brw_imm_v(0x76543210)));
2805 bld.SHL(channel_offsets, sequence, brw_imm_ud(2u));
2807 bld.SHL(vertex_offset_bytes,
2810 bld.ADD(icp_offset_bytes, vertex_offset_bytes, channel_offsets);
2816 bld.emit(SHADER_OPCODE_MOV_INDIRECT, icp_handle,
2837 fs_visitor::nir_emit_tcs_intrinsic(const fs_builder &bld,
2854 bld.MOV(dst, fs_reg(eight_patch ? brw_vec8_grf(2, 0)
2858 bld.MOV(retype(dst, invocation_id.type), invocation_id);
2861 bld.MOV(retype(dst, BRW_REGISTER_TYPE_D),
2869 fs_reg m0 = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
2872 const fs_builder chanbld = bld.exec_all().group(1, 0);
2875 bld.exec_all().MOV(m0, brw_imm_ud(0u));
2883 bld.exec_all().group(2, 0).MOV(m0_10ub, r0_11ub);
2904 bld.emit(SHADER_OPCODE_BARRIER, bld.null_reg_ud(), m0);
2919 eight_patch ? get_tcs_eight_patch_icp_handle(bld, instr)
2920 : get_tcs_single_patch_icp_handle(bld, instr);
2936 fs_reg tmp = bld.vgrf(dst.type, read_components);
2937 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, tmp, srcs,
2940 bld.MOV(offset(dst, bld, i),
2941 offset(tmp, bld, i + first_component));
2944 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, dst, srcs,
2955 fs_reg tmp = bld.vgrf(dst.type, read_components);
2956 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, tmp,
2959 bld.MOV(offset(dst, bld, i),
2960 offset(tmp, bld, i + first_component));
2963 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, dst,
2978 inst->dst = bld.vgrf(dst.type, 4);
2980 bld.MOV(dst, offset(inst->dst, bld, 3));
2999 fs_reg patch_handle = bld.vgrf(BRW_REGISTER_TYPE_UD, 1);
3000 bld.MOV(patch_handle, output_handles);
3009 fs_reg tmp = bld.vgrf(dst.type, read_components);
3010 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, tmp,
3014 bld.MOV(offset(dst, bld, i),
3015 offset(tmp, bld, i + first_component));
3018 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, dst,
3034 fs_reg tmp = bld.vgrf(dst.type, read_components);
3035 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, tmp,
3039 bld.MOV(offset(dst, bld, i),
3040 offset(tmp, bld, i + first_component));
3043 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, dst,
3082 sources[i + first_component] = offset(value, bld, i);
3095 bld.LOAD_PAYLOAD(srcs[URB_LOGICAL_SRC_DATA], sources, length, 0);
3097 fs_inst *inst = bld.emit(SHADER_OPCODE_URB_WRITE_LOGICAL, reg_undef,
3105 nir_emit_intrinsic(bld, instr);
3111 fs_visitor::nir_emit_tes_intrinsic(const fs_builder &bld,
3123 bld.MOV(dest, fs_reg(brw_vec1_grf(0, 1)));
3128 bld.MOV(offset(dest, bld, i), fs_reg(brw_vec8_grf(1 + i, 0)));
3149 bld.MOV(offset(dest, bld, i), component(src, comp));
3164 fs_reg tmp = bld.vgrf(dest.type, read_components);
3165 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, tmp,
3169 bld.MOV(offset(dest, bld, i),
3170 offset(tmp, bld, i + first_component));
3173 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, dest,
3197 fs_reg tmp = bld.vgrf(dest.type, read_components);
3198 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, tmp,
3201 bld.MOV(offset(dest, bld, i),
3202 offset(tmp, bld, i + first_component));
3205 inst = bld.emit(SHADER_OPCODE_URB_READ_LOGICAL, dest,
3216 nir_emit_intrinsic(bld, instr);
3222 fs_visitor::nir_emit_gs_intrinsic(const fs_builder &bld,
3236 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD),
3258 bld.MOV(this->final_gs_vertex_count, get_nir_src(instr->src[0]));
3265 bld.MOV(dest, val);
3270 nir_emit_intrinsic(bld, instr);
3279 fetch_render_target_array_index(const fs_builder &bld)
3281 if (bld.shader->devinfo->ver >= 12) {
3285 const fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_UD);
3286 bld.AND(idx, brw_uw1_reg(BRW_GENERAL_REGISTER_FILE, 1, 3),
3289 } else if (bld.shader->devinfo->ver >= 6) {
3293 const fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_UD);
3294 bld.AND(idx, brw_uw1_reg(BRW_GENERAL_REGISTER_FILE, 0, 1),
3310 fs_visitor::emit_non_coherent_fb_read(const fs_builder &bld, const fs_reg &dst,
3313 const struct intel_device_info *devinfo = bld.shader->devinfo;
3315 assert(bld.shader->stage == MESA_SHADER_FRAGMENT);
3321 const fs_reg coords = bld.vgrf(BRW_REGISTER_TYPE_UD, 3);
3322 bld.MOV(offset(coords, bld, 0), pixel_x);
3323 bld.MOV(offset(coords, bld, 1), pixel_y);
3324 bld.MOV(offset(coords, bld, 2), fetch_render_target_array_index(bld));
3373 fs_inst *inst = bld.emit(op, dst, srcs, ARRAY_SIZE(srcs));
3384 emit_coherent_fb_read(const fs_builder &bld, const fs_reg &dst, unsigned target)
3386 assert(bld.shader->devinfo->ver >= 9);
3387 fs_inst *inst = bld.emit(FS_OPCODE_FB_READ_LOGICAL, dst);
3395 alloc_temporary(const fs_builder &bld, unsigned size, fs_reg *regs, unsigned n)
3401 const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_F, size);
3420 return alloc_temporary(v->bld, 4, &v->dual_src_output, 1);
3423 return alloc_temporary(v->bld, 4, v->outputs,
3427 return alloc_temporary(v->bld, 1, &v->frag_depth, 1);
3430 return alloc_temporary(v->bld, 1, &v->frag_stencil, 1);
3433 return alloc_temporary(v->bld, 1, &v->sample_mask, 1);
3437 return alloc_temporary(v->bld, 4,
3445 fs_visitor::nir_emit_fs_intrinsic(const fs_builder &bld,
3456 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D),
3465 bld.MOV(dest, sample_pos);
3466 bld.MOV(offset(dest, bld, 1), offset(sample_pos, bld, 1));
3472 bld.MOV(dest, fetch_render_target_array_index(bld));
3487 bld.MOV(dest, val);
3500 bld.MOV(offset(new_dest, bld, nir_intrinsic_component(instr) + j),
3501 offset(src, bld, j));
3512 const fs_reg tmp = bld.vgrf(dest.type, 4);
3515 emit_coherent_fb_read(bld, tmp, target);
3517 emit_non_coherent_fb_read(bld, tmp, target);
3520 bld.MOV(offset(dest, bld, j),
3521 offset(tmp, bld, nir_intrinsic_component(instr) + j));
3567 nir_emit_alu(bld, alu, false);
3585 cmp = bld.CMP(bld.null_reg_f(), get_nir_src(instr->src[0]),
3591 cmp = bld.CMP(bld.null_reg_f(), some_reg, some_reg, BRW_CONDITIONAL_NZ);
3597 fs_inst *jump = bld.emit(BRW_OPCODE_HALT);
3637 bld.MOV(offset(dest, bld, i),
3642 bld.MOV(offset(dest, bld, i),
3655 bld.MOV(offset(dest, bld, 0), component(interp, 3));
3656 bld.MOV(offset(dest, bld, 1), component(interp, 1));
3657 bld.MOV(offset(dest, bld, 2), component(interp, 0));
3666 const fs_reg srcs[] = { offset(this->delta_xy[bary], bld, 0),
3667 offset(this->delta_xy[bary], bld, 1) };
3668 bld.LOAD_PAYLOAD(dest, srcs, ARRAY_SIZE(srcs), 0);
3679 emit_pixel_interpolater_send(bld,
3690 const fs_reg sample_id = bld.emit_uniformize(sample_src);
3692 bld.exec_all().group(1, 0)
3694 emit_pixel_interpolater_send(bld,
3707 bld.emit(BRW_OPCODE_DO);
3710 const fs_reg sample_id = bld.emit_uniformize(sample_src);
3715 bld.CMP(bld.null_reg_ud(),
3719 bld.exec_all().group(1, 0)
3722 emit_pixel_interpolater_send(bld,
3733 bld.emit(BRW_OPCODE_WHILE));
3750 emit_pixel_interpolater_send(bld,
3759 emit_pixel_interpolater_send(bld,
3802 bld.emit(FS_OPCODE_LINTERP, tmp, dst_xy, interp);
3803 bld.MUL(offset(dest, bld, i), tmp, this->pixel_w);
3805 bld.emit(FS_OPCODE_LINTERP, offset(dest, bld, i), dst_xy, interp);
3812 nir_emit_intrinsic(bld, instr);
3818 fs_visitor::nir_emit_cs_intrinsic(const fs_builder &bld,
3836 bld.exec_all().group(1, 0).emit(FS_OPCODE_SCHEDULING_FENCE);
3846 bld.AND(retype(dest, BRW_REGISTER_TYPE_UD),
3850 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD), subgroup_id);
3860 bld.MOV(offset(dest, bld, i), offset(val, bld, i));
3876 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
3892 nir_emit_shared_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
3897 nir_emit_shared_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
3922 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
3929 fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD);
3930 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
3932 bld.MOV(dest, subscript(read_result, dest.type, 0));
3963 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
3969 srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD);
3970 bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data);
3972 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
3986 bld.MOV(retype(offset(dest, bld, i), BRW_REGISTER_TYPE_UD),
3993 nir_emit_intrinsic(bld, instr);
3999 emit_rt_lsc_fence(const fs_builder &bld,
4003 const intel_device_info *devinfo = bld.shader->devinfo;
4005 const fs_builder ubld = bld.exec_all().group(8, 0);
4023 fs_visitor::nir_emit_bs_intrinsic(const fs_builder &bld,
4034 bld.MOV(dest, retype(brw_vec1_grf(2, 0), dest.type));
4038 bld.MOV(dest, retype(brw_vec1_grf(2, 2), dest.type));
4043 bld.MOV(ud_dest, retype(brw_vec1_grf(0, 3), ud_dest.type));
4044 bld.AND(ud_dest, ud_dest, brw_imm_ud(0xf));
4049 nir_emit_intrinsic(bld, instr);
4055 brw_nir_reduction_op_identity(const fs_builder &bld,
4073 return setup_imm_df(bld, value.f64);
4126 fs_visitor::get_nir_image_intrinsic_image(const brw::fs_builder &bld,
4132 return bld.emit_uniformize(surf_index);
4136 fs_visitor::get_nir_ssbo_intrinsic_index(const brw::fs_builder &bld,
4148 return bld.emit_uniformize(get_nir_src(instr->src[src]));
4176 fs_visitor::swizzle_nir_scratch_addr(const brw::fs_builder &bld,
4184 fs_reg addr = bld.vgrf(BRW_REGISTER_TYPE_UD);
4189 bld.SHL(addr, nir_addr, brw_imm_ud(chan_index_bits - 2));
4190 bld.OR(addr, addr, chan_index);
4195 fs_reg addr_hi = bld.vgrf(BRW_REGISTER_TYPE_UD);
4196 bld.AND(addr_hi, nir_addr, brw_imm_ud(~0x3u));
4197 bld.SHL(addr_hi, addr_hi, brw_imm_ud(chan_index_bits));
4198 fs_reg chan_addr = bld.vgrf(BRW_REGISTER_TYPE_UD);
4199 bld.SHL(chan_addr, chan_index, brw_imm_ud(2));
4200 bld.AND(addr, nir_addr, brw_imm_ud(0x3u));
4201 bld.OR(addr, addr, addr_hi);
4202 bld.OR(addr, addr, chan_addr);
4223 increment_a64_address(const fs_builder &bld, fs_reg address, uint32_t v)
4225 if (bld.shader->devinfo->has_64bit_int) {
4226 bld.ADD(address, address, brw_imm_ud(v));
4229 fs_reg high = offset(low, bld, 1);
4232 bld.ADD(low, low, brw_imm_ud(v))->conditional_mod = BRW_CONDITIONAL_O;
4233 bld.ADD(high, high, brw_imm_ud(0x1))->predicate = BRW_PREDICATE_NORMAL;
4238 emit_fence(const fs_builder &bld, enum opcode opcode,
4245 fs_reg dst = bld.vgrf(BRW_REGISTER_TYPE_UD);
4246 fs_inst *fence = bld.emit(opcode, dst, brw_vec8_grf(0, 0),
4290 fs_visitor::nir_emit_intrinsic(const fs_builder &bld, nir_intrinsic_instr *instr)
4340 get_nir_image_intrinsic_image(bld, instr);
4346 bld.emit_uniformize(get_nir_src(instr->src[0]));
4360 bld.emit(SHADER_OPCODE_TYPED_SURFACE_READ_LOGICAL,
4368 bld.emit(SHADER_OPCODE_TYPED_SURFACE_WRITE_LOGICAL,
4384 fs_reg tmp = bld.vgrf(data.type, 2);
4386 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
4392 bld.emit(SHADER_OPCODE_TYPED_ATOMIC_LOGICAL,
4410 image = bld.emit_uniformize(image);
4426 const fs_builder ubld = bld.exec_all().group(8, 0);
4434 bld.MOV(offset(retype(dest, tmp.type), bld, c),
4443 get_nir_image_intrinsic_image(bld, instr);
4450 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
4459 get_nir_image_intrinsic_image(bld, instr);
4466 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
4561 const fs_builder ubld = bld.group(8, 0);
4672 const fs_reg shader_clock = get_timestamp(bld);
4675 bld.LOAD_PAYLOAD(dest, srcs, ARRAY_SIZE(srcs), 0);
4681 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), brw_imm_d(1));
4686 bld.emit(SHADER_OPCODE_MOV_RELOC_IMM,
4709 bld.MOV(offset(dest, bld, j), offset(src, bld, j));
4730 bld.emit(SHADER_OPCODE_MOV_INDIRECT,
4731 offset(dest, bld, j), offset(src, bld, j),
4744 bld.emit(SHADER_OPCODE_MOV_INDIRECT,
4745 subscript(offset(dest, bld, j), BRW_REGISTER_TYPE_UD, i),
4746 subscript(offset(src, bld, j), BRW_REGISTER_TYPE_UD, i),
4766 bld.MOV(surf_index, get_nir_src(instr->src[0]));
4767 surf_index = bld.emit_uniformize(surf_index);
4775 VARYING_PULL_CONSTANT_LOAD(bld, offset(dest, bld, i), surf_index,
4812 bld.MOV(offset(dest, bld, i),
4822 const fs_builder ubld = bld.exec_all().group(block_sz / 4, 0);
4840 bld.MOV(offset(dest, bld, c + d), component(consts, d));
4867 bld.emit(SHADER_OPCODE_A64_UNTYPED_READ_LOGICAL, dest,
4874 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
4878 bld.emit(SHADER_OPCODE_A64_BYTE_SCATTERED_READ_LOGICAL, tmp,
4880 bld.MOV(dest, subscript(tmp, dest.type, 0));
4905 bld.emit(SHADER_OPCODE_A64_UNTYPED_WRITE_LOGICAL, fs_reg(),
4912 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
4913 bld.MOV(tmp, retype(get_nir_src(instr->src[0]), data_type));
4918 bld.emit(SHADER_OPCODE_A64_BYTE_SCATTERED_WRITE_LOGICAL, fs_reg(),
4934 nir_emit_global_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
4940 nir_emit_global_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
4947 const fs_builder ubld = bld.exec_all().group(instr->num_components, 0);
4958 fs_reg addr = bld.emit_uniformize(get_nir_src(instr->src[0]));
4967 fs_reg pred = bld.emit_uniformize(get_nir_src(instr->src[1]));
4968 fs_inst *mov = ubld.MOV(bld.null_reg_d(), pred);
5000 bld.MOV(retype(offset(dest, bld, i), BRW_REGISTER_TYPE_UD),
5012 get_nir_ssbo_intrinsic_index(bld, instr);
5028 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
5035 fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD);
5036 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
5038 bld.MOV(dest, subscript(read_result, dest.type, 0));
5049 get_nir_ssbo_intrinsic_index(bld, instr);
5066 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
5072 srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD);
5073 bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data);
5075 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
5089 fs_reg new_dest = retype(offset(outputs[instr->const_index[0]], bld,
5092 bld.MOV(offset(new_dest, bld, j + first_component),
5093 offset(src, bld, j));
5108 nir_emit_ssbo_atomic(bld, brw_aop_for_nir_intrinsic(instr), instr);
5114 nir_emit_ssbo_atomic_float(bld, brw_aop_for_nir_intrinsic(instr), instr);
5130 const fs_builder ubld = bld.exec_all().group(8, 0);
5172 bld.MOV(retype(dest, ret_payload.type), component(buffer_size, 0));
5184 const fs_builder ubld = bld.exec_all().group(1, 0);
5215 swizzle_nir_scratch_addr(bld, nir_addr, false);
5218 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_READ_LOGICAL,
5223 swizzle_nir_scratch_addr(bld, nir_addr, true);
5225 bld.emit(SHADER_OPCODE_DWORD_SCATTERED_READ_LOGICAL,
5230 swizzle_nir_scratch_addr(bld, nir_addr, false);
5232 fs_reg read_result = bld.vgrf(BRW_REGISTER_TYPE_UD);
5233 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_READ_LOGICAL,
5235 bld.MOV(dest, read_result);
5250 const fs_builder ubld = bld.exec_all().group(1, 0);
5287 swizzle_nir_scratch_addr(bld, nir_addr, false);
5290 bld.emit(SHADER_OPCODE_UNTYPED_SURFACE_WRITE_LOGICAL,
5297 swizzle_nir_scratch_addr(bld, nir_addr, true);
5299 bld.emit(SHADER_OPCODE_DWORD_SCATTERED_WRITE_LOGICAL,
5303 srcs[SURFACE_LOGICAL_SRC_DATA] = bld.vgrf(BRW_REGISTER_TYPE_UD);
5304 bld.MOV(srcs[SURFACE_LOGICAL_SRC_DATA], data);
5307 swizzle_nir_scratch_addr(bld, nir_addr, false);
5309 bld.emit(SHADER_OPCODE_BYTE_SCATTERED_WRITE_LOGICAL,
5321 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), brw_imm_d(dispatch_width));
5325 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D),
5337 const fs_builder ubld = bld.exec_all().group(1, 0);
5350 bld.CMP(bld.null_reg_d(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ);
5365 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0));
5369 const fs_builder ubld = bld.exec_all().group(1, 0);
5382 bld.CMP(bld.null_reg_d(), get_nir_src(instr->src[0]), brw_imm_d(0), BRW_CONDITIONAL_NZ);
5397 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0));
5409 fs_reg uniformized = bld.emit_uniformize(value);
5410 const fs_builder ubld = bld.exec_all().group(1, 0);
5423 bld.CMP(bld.null_reg_d(), value, uniformized, BRW_CONDITIONAL_Z);
5438 bld.MOV(retype(dest, BRW_REGISTER_TYPE_D), component(res1, 0));
5455 bld.exec_all().group(1, 0).MOV(flag, brw_imm_ud(0u));
5456 bld.CMP(bld.null_reg_ud(), value, brw_imm_ud(0u), BRW_CONDITIONAL_NZ);
5463 bld.MOV(dest, flag);
5470 fs_reg tmp = bld.vgrf(value.type);
5472 bld.exec_all().emit(SHADER_OPCODE_BROADCAST, tmp, value,
5473 bld.emit_uniformize(invocation));
5475 bld.MOV(retype(dest, value.type), fs_reg(component(tmp, 0)));
5481 bld.MOV(retype(dest, value.type), bld.emit_uniformize(value));
5489 bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, index);
5494 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
5495 bld.exec_all().emit(SHADER_OPCODE_FIND_LIVE_CHANNEL, tmp);
5496 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD),
5502 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
5503 bld.exec_all().emit(SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL, tmp);
5504 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD),
5513 bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, retype(dest, value.type),
5520 const fs_reg tmp = bld.vgrf(value.type);
5528 const fs_builder ubld = bld.exec_all();
5531 bld.MOV(retype(dest, value.type), tmp);
5533 const fs_builder ubld = bld.exec_all().group(dispatch_width / 2, 0);
5544 bld.MOV(retype(dest, value.type), tmp);
5552 const fs_reg tmp = bld.vgrf(value.type);
5553 const fs_builder ubld = bld.exec_all();
5556 bld.MOV(retype(dest, value.type), tmp);
5561 fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W);
5562 bld.XOR(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION],
5564 bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, idx);
5573 const fs_reg tmp = bld.vgrf(value.type);
5574 const fs_builder ubld = bld.exec_all();
5577 bld.MOV(retype(dest, value.type), tmp);
5582 fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W);
5583 bld.XOR(idx, nir_system_values[SYSTEM_VALUE_SUBGROUP_INVOCATION],
5585 bld.emit(SHADER_OPCODE_SHUFFLE, retype(dest, value.type), value, idx);
5602 fs_reg identity = brw_nir_reduction_op_identity(bld, redop, src.type);
5609 fs_reg scan = bld.vgrf(src.type);
5610 bld.exec_all().emit(SHADER_OPCODE_SEL_EXEC, scan, src, identity);
5612 bld.emit_scan(brw_op, scan, cluster_size, cond_mod);
5628 bld.group(group_size, i).MOV(horiz_offset(dest, i * group_size),
5632 bld.emit(SHADER_OPCODE_CLUSTER_BROADCAST, dest, scan,
5648 fs_reg identity = brw_nir_reduction_op_identity(bld, redop, src.type);
5655 fs_reg scan = bld.vgrf(src.type);
5656 const fs_builder allbld = bld.exec_all();
5664 fs_reg shifted = bld.vgrf(src.type);
5665 fs_reg idx = bld.vgrf(BRW_REGISTER_TYPE_W);
5673 bld.emit_scan(brw_op, scan, dispatch_width, cond_mod);
5675 bld.MOV(retype(dest, src.type), scan);
5682 fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[0]));
5684 const fs_builder ubld1 = bld.exec_all().group(1, 0);
5685 const fs_builder ubld8 = bld.exec_all().group(8, 0);
5686 const fs_builder ubld16 = bld.exec_all().group(16, 0);
5718 fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[1]));
5721 const fs_builder ubld1 = bld.exec_all().group(1, 0);
5722 const fs_builder ubld8 = bld.exec_all().group(8, 0);
5723 const fs_builder ubld16 = bld.exec_all().group(16, 0);
5758 fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[is_ssbo ? 1 : 0]));
5762 get_nir_ssbo_intrinsic_index(bld, instr) : fs_reg(brw_imm_ud(GFX7_BTI_SLM));
5765 const fs_builder ubld1 = bld.exec_all().group(1, 0);
5766 const fs_builder ubld8 = bld.exec_all().group(8, 0);
5767 const fs_builder ubld16 = bld.exec_all().group(16, 0);
5799 fs_reg address = bld.emit_uniformize(get_nir_src(instr->src[is_ssbo ? 2 : 1]));
5804 get_nir_ssbo_intrinsic_index(bld, instr) : fs_reg(brw_imm_ud(GFX7_BTI_SLM));
5807 const fs_builder ubld1 = bld.exec_all().group(1, 0);
5808 const fs_builder ubld8 = bld.exec_all().group(8, 0);
5809 const fs_builder ubld16 = bld.exec_all().group(16, 0);
5851 fs_reg raw_id = bld.vgrf(BRW_REGISTER_TYPE_UD);
5852 bld.emit(SHADER_OPCODE_READ_SR_REG, raw_id, brw_imm_ud(0));
5855 bld.AND(raw_id, raw_id, brw_imm_ud(0x3fff));
5857 bld.SHR(retype(dest, BRW_REGISTER_TYPE_UD), raw_id, brw_imm_ud(9));
5874 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UD);
5875 bld.AND(tmp, raw_id, brw_imm_ud(INTEL_MASK(7, 7)));
5876 bld.SHL(dst, tmp, brw_imm_ud(3));
5877 bld.AND(tmp, raw_id, brw_imm_ud(INTEL_MASK(8, 8)));
5878 bld.SHL(tmp, tmp, brw_imm_ud(1));
5879 bld.OR(dst, dst, tmp);
5880 bld.AND(tmp, raw_id, brw_imm_ud(INTEL_MASK(5, 4)));
5881 bld.SHL(tmp, tmp, brw_imm_ud(3));
5882 bld.OR(dst, dst, tmp);
5887 bld.AND(raw_id, raw_id, brw_imm_ud(INTEL_MASK(2, 0)));
5888 bld.SHL(raw_id, raw_id, brw_imm_ud(4));
5889 bld.OR(dst, dst, raw_id);
5895 fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UW);
5896 const fs_builder ubld8 = bld.exec_all().group(8, 0);
5898 if (bld.dispatch_width() == 16) {
5902 bld.ADD(dst, dst, tmp);
5920 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UD),
5933 emit_rt_lsc_fence(bld, LSC_FENCE_LOCAL, LSC_FLUSH_TYPE_NONE);
5935 bld.emit(SHADER_OPCODE_BTD_SPAWN_LOGICAL, bld.null_reg_ud(),
5936 bld.emit_uniformize(get_nir_src(instr->src[0])),
5949 emit_rt_lsc_fence(bld, LSC_FENCE_LOCAL, LSC_FLUSH_TYPE_NONE);
5950 bld.emit(SHADER_OPCODE_BTD_RETIRE_LOGICAL);
5961 emit_rt_lsc_fence(bld, LSC_FENCE_LOCAL, LSC_FLUSH_TYPE_NONE);
5966 srcs[RT_LOGICAL_SRC_GLOBALS] = bld.emit_uniformize(globals);
5970 bld.emit(RT_OPCODE_TRACE_RAY_LOGICAL, bld.null_reg_ud(),
5980 bld.emit(BRW_OPCODE_SYNC, bld.null_reg_ud(), brw_imm_ud(TGL_SYNC_ALLWR));
5981 emit_rt_lsc_fence(bld, LSC_FENCE_LOCAL, LSC_FLUSH_TYPE_INVALIDATE);
5996 fs_visitor::nir_emit_ssbo_atomic(const fs_builder &bld,
6012 srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr);
6023 fs_reg tmp = bld.vgrf(data.type, 2);
6025 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
6032 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
6037 fs_visitor::nir_emit_ssbo_atomic_float(const fs_builder &bld,
6045 srcs[SURFACE_LOGICAL_SRC_SURFACE] = get_nir_ssbo_intrinsic_index(bld, instr);
6053 fs_reg tmp = bld.vgrf(data.type, 2);
6055 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
6062 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL,
6067 fs_visitor::nir_emit_shared_atomic(const fs_builder &bld,
6084 fs_reg tmp = bld.vgrf(data.type, 2);
6086 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
6097 bld.ADD(srcs[SURFACE_LOGICAL_SRC_ADDRESS],
6104 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_LOGICAL,
6109 fs_visitor::nir_emit_shared_atomic_float(const fs_builder &bld,
6124 fs_reg tmp = bld.vgrf(data.type, 2);
6126 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
6137 bld.ADD(srcs[SURFACE_LOGICAL_SRC_ADDRESS],
6144 bld.emit(SHADER_OPCODE_UNTYPED_ATOMIC_FLOAT_LOGICAL,
6149 expand_to_32bit(const fs_builder &bld, const fs_reg &src)
6152 fs_reg src32 = bld.vgrf(BRW_REGISTER_TYPE_UD);
6153 bld.MOV(src32, retype(src, BRW_REGISTER_TYPE_UW));
6161 fs_visitor::nir_emit_global_atomic(const fs_builder &bld,
6172 data = expand_to_32bit(bld, get_nir_src(instr->src[1]));
6175 fs_reg tmp = bld.vgrf(data.type, 2);
6178 expand_to_32bit(bld, get_nir_src(instr->src[2]))
6180 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
6192 fs_reg dest32 = bld.vgrf(BRW_REGISTER_TYPE_UD);
6193 bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_INT16_LOGICAL, dest32,
6195 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UW), dest32);
6199 bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_LOGICAL, dest,
6203 bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_INT64_LOGICAL, dest,
6212 fs_visitor::nir_emit_global_atomic_float(const fs_builder &bld,
6221 fs_reg data = expand_to_32bit(bld, get_nir_src(instr->src[1]));
6224 fs_reg tmp = bld.vgrf(data.type, 2);
6227 expand_to_32bit(bld, get_nir_src(instr->src[2]))
6229 bld.LOAD_PAYLOAD(tmp, sources, 2, 0);
6241 fs_reg dest32 = bld.vgrf(BRW_REGISTER_TYPE_UD);
6242 bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_FLOAT16_LOGICAL, dest32,
6244 bld.MOV(retype(dest, BRW_REGISTER_TYPE_UW), dest32);
6248 bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_FLOAT32_LOGICAL, dest,
6252 bld.emit(SHADER_OPCODE_A64_UNTYPED_ATOMIC_FLOAT64_LOGICAL, dest,
6261 fs_visitor::nir_emit_texture(const fs_builder &bld, nir_tex_instr *instr)
6361 bld.ADD(tmp, src, brw_imm_ud(texture));
6362 srcs[TEX_LOGICAL_SRC_SURFACE] = bld.emit_uniformize(tmp);
6369 bld.ADD(tmp, src, brw_imm_ud(sampler));
6370 srcs[TEX_LOGICAL_SRC_SAMPLER] = bld.emit_uniformize(tmp);
6377 srcs[TEX_LOGICAL_SRC_SURFACE_HANDLE] = bld.emit_uniformize(src);
6383 srcs[TEX_LOGICAL_SRC_SAMPLER_HANDLE] = bld.emit_uniformize(src);
6470 bld.MOV(dst, brw_imm_ud(0u));
6473 bld.OR(tmp, srcs[TEX_LOGICAL_SRC_MCS],
6474 offset(srcs[TEX_LOGICAL_SRC_MCS], bld, 1));
6475 bld.CMP(dst, tmp, brw_imm_ud(0u), BRW_CONDITIONAL_EQ);
6477 bld.CMP(dst, srcs[TEX_LOGICAL_SRC_MCS], brw_imm_ud(0u),
6498 fs_reg dst = bld.vgrf(brw_type_for_nir_type(devinfo, instr->dest_type), 4);
6499 fs_inst *inst = bld.emit(opcode, dst, srcs, ARRAY_SIZE(srcs));
6520 nir_dest[i] = offset(dst, bld, i);
6531 fs_inst *mov = bld.MOV(bld.null_reg_d(), dst);
6533 nir_dest[0] = bld.vgrf(BRW_REGISTER_TYPE_D);
6534 fs_inst *sel = bld.SEL(nir_dest[0], offset(dst, bld, 3), brw_imm_d(0));
6537 nir_dest[0] = offset(dst, bld, 3);
6542 fs_reg depth = offset(dst, bld, 2);
6544 bld.emit_minmax(nir_dest[2], depth, brw_imm_d(1), BRW_CONDITIONAL_GE);
6547 bld.LOAD_PAYLOAD(get_nir_dest(instr->dest), nir_dest, dest_size, 0);
6551 fs_visitor::nir_emit_jump(const fs_builder &bld, nir_jump_instr *instr)
6555 bld.emit(BRW_OPCODE_BREAK);
6558 bld.emit(BRW_OPCODE_CONTINUE);
6561 bld.emit(BRW_OPCODE_HALT);
6613 shuffle_src_to_dst(const fs_builder &bld,
6621 type_sz(dst.type) * bld.dispatch_width() * components,
6622 offset(src, bld, first_component),
6623 type_sz(src.type) * bld.dispatch_width() * components));
6625 bld.MOV(retype(offset(dst, bld, i), src.type),
6626 offset(src, bld, i + first_component));
6632 type_sz(dst.type) * bld.dispatch_width() *
6634 offset(src, bld, first_component),
6635 type_sz(src.type) * bld.dispatch_width() * components));
6642 subscript(offset(dst, bld, i / size_ratio),
6644 bld.MOV(shuffle_component_i,
6645 retype(offset(src, bld, i + first_component), shuffle_type));
6651 type_sz(dst.type) * bld.dispatch_width() * components,
6652 offset(src, bld, first_component / size_ratio),
6653 type_sz(src.type) * bld.dispatch_width() *
6662 subscript(offset(src, bld, (first_component + i) / size_ratio),
6664 bld.MOV(retype(offset(dst, bld, i), shuffle_type),
6671 shuffle_from_32bit_read(const fs_builder &bld,
6688 shuffle_src_to_dst(bld, dst, src, first_component, components);
6692 setup_imm_df(const fs_builder &bld, double v)
6694 const struct intel_device_info *devinfo = bld.shader->devinfo;
6704 const fs_builder ubld = bld.exec_all().group(1, 0);
6731 const fs_builder ubld = bld.exec_all().group(1, 0);
6740 setup_imm_b(const fs_builder &bld, int8_t v)
6742 const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_B);
6743 bld.MOV(tmp, brw_imm_w(v));
6748 setup_imm_ub(const fs_builder &bld, uint8_t v)
6750 const fs_reg tmp = bld.vgrf(BRW_REGISTER_TYPE_UB);
6751 bld.MOV(tmp, brw_imm_uw(v));