Lines Matching refs:intrin
33 lower_subgroups_64bit_split_intrinsic(nir_builder *b, nir_intrinsic_instr *intrin,
38 comp = nir_unpack_64_2x32_split_x(b, intrin->src[0].ssa);
40 comp = nir_unpack_64_2x32_split_y(b, intrin->src[0].ssa);
42 nir_intrinsic_instr *intr = nir_intrinsic_instr_create(b->shader, intrin->intrinsic);
44 intr->const_index[0] = intrin->const_index[0];
45 intr->const_index[1] = intrin->const_index[1];
47 if (nir_intrinsic_infos[intrin->intrinsic].num_srcs == 2)
48 nir_src_copy(&intr->src[1], &intrin->src[1]);
56 lower_subgroup_op_to_32bit(nir_builder *b, nir_intrinsic_instr *intrin)
58 assert(intrin->src[0].ssa->bit_size == 64);
59 nir_intrinsic_instr *intr_x = lower_subgroups_64bit_split_intrinsic(b, intrin, 0);
60 nir_intrinsic_instr *intr_y = lower_subgroups_64bit_split_intrinsic(b, intrin, 1);
107 lower_subgroup_op_to_scalar(nir_builder *b, nir_intrinsic_instr *intrin,
111 assert(intrin->dest.ssa.num_components > 1);
113 nir_ssa_def *value = nir_ssa_for_src(b, intrin->src[0],
114 intrin->num_components);
117 for (unsigned i = 0; i < intrin->num_components; i++) {
119 nir_intrinsic_instr_create(b->shader, intrin->intrinsic);
121 1, intrin->dest.ssa.bit_size, NULL);
127 if (nir_intrinsic_infos[intrin->intrinsic].num_srcs > 1) {
128 assert(nir_intrinsic_infos[intrin->intrinsic].num_srcs == 2);
129 nir_src_copy(&chan_intrin->src[1], &intrin->src[1]);
132 chan_intrin->const_index[0] = intrin->const_index[0];
133 chan_intrin->const_index[1] = intrin->const_index[1];
143 return nir_vec(b, reads, intrin->num_components);
147 lower_vote_eq_to_scalar(nir_builder *b, nir_intrinsic_instr *intrin)
149 assert(intrin->src[0].is_ssa);
150 nir_ssa_def *value = intrin->src[0].ssa;
153 for (unsigned i = 0; i < intrin->num_components; i++) {
155 nir_intrinsic_instr_create(b->shader, intrin->intrinsic);
157 1, intrin->dest.ssa.bit_size, NULL);
173 lower_vote_eq(nir_builder *b, nir_intrinsic_instr *intrin)
175 assert(intrin->src[0].is_ssa);
176 nir_ssa_def *value = intrin->src[0].ssa;
180 for (unsigned i = 0; i < intrin->num_components; i++) {
184 if (intrin->intrinsic == nir_intrinsic_vote_feq) {
201 lower_shuffle_to_swizzle(nir_builder *b, nir_intrinsic_instr *intrin,
204 unsigned mask = nir_src_as_uint(intrin->src[1]);
211 swizzle->num_components = intrin->num_components;
212 nir_src_copy(&swizzle->src[0], &intrin->src[0]);
215 intrin->dest.ssa.num_components,
216 intrin->dest.ssa.bit_size, NULL);
231 lower_to_shuffle(nir_builder *b, nir_intrinsic_instr *intrin,
234 if (intrin->intrinsic == nir_intrinsic_shuffle_xor &&
236 nir_src_is_const(intrin->src[1])) {
238 lower_shuffle_to_swizzle(b, intrin, options);
245 switch (intrin->intrinsic) {
247 assert(intrin->src[1].is_ssa);
248 index = nir_ixor(b, index, intrin->src[1].ssa);
252 assert(intrin->src[1].is_ssa);
253 index = nir_isub(b, index, intrin->src[1].ssa);
257 assert(intrin->src[1].is_ssa);
258 index = nir_iadd(b, index, intrin->src[1].ssa);
262 assert(intrin->src[1].is_ssa);
264 intrin->src[1].ssa);
290 shuffle->num_components = intrin->num_components;
291 nir_src_copy(&shuffle->src[0], &intrin->src[0]);
294 intrin->dest.ssa.num_components,
295 intrin->dest.ssa.bit_size, NULL);
319 lower_shuffle(nir_builder *b, nir_intrinsic_instr *intrin)
321 assert(intrin->src[0].is_ssa);
322 assert(intrin->src[1].is_ssa);
323 nir_ssa_def *val = intrin->src[0].ssa;
324 nir_ssa_def *id = intrin->src[1].ssa;
557 lower_dynamic_quad_broadcast(nir_builder *b, nir_intrinsic_instr *intrin,
561 return lower_to_shuffle(b, intrin, options);
569 qbcst->num_components = intrin->num_components;
571 nir_src_copy(&qbcst->src[0], &intrin->src[0]);
573 intrin->dest.ssa.num_components,
574 intrin->dest.ssa.bit_size, NULL);
586 dst = nir_bcsel(b, nir_ieq(b, intrin->src[1].ssa,
597 lower_read_invocation_to_cond(nir_builder *b, nir_intrinsic_instr *intrin)
599 return nir_read_invocation_cond_ir3(b, intrin->dest.ssa.bit_size,
600 intrin->src[0].ssa,
601 nir_ieq(b, intrin->src[1].ssa,
610 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
611 switch (intrin->intrinsic) {
615 return nir_ssa_for_src(b, intrin->src[0], 1);
624 return lower_vote_eq(b, intrin);
626 if (options->lower_to_scalar && intrin->num_components > 1)
627 return lower_vote_eq_to_scalar(b, intrin);
636 if (options->lower_to_scalar && intrin->num_components > 1)
637 return lower_subgroup_op_to_scalar(b, intrin, false);
640 return lower_read_invocation_to_cond(b, intrin);
645 if (options->lower_to_scalar && intrin->num_components > 1)
646 return lower_subgroup_op_to_scalar(b, intrin, false);
658 switch (intrin->intrinsic) {
681 intrin->dest.ssa.num_components,
682 intrin->dest.ssa.bit_size);
686 if (intrin->dest.ssa.num_components == options->ballot_components &&
687 intrin->dest.ssa.bit_size == options->ballot_bit_size)
692 intrin->src[0].ssa);
695 intrin->dest.ssa.num_components,
696 intrin->dest.ssa.bit_size);
703 assert(intrin->src[0].is_ssa);
704 nir_ssa_def *int_val = ballot_type_to_uint(b, intrin->src[0].ssa,
707 if (intrin->intrinsic != nir_intrinsic_ballot_bitfield_extract &&
708 intrin->intrinsic != nir_intrinsic_ballot_find_lsb) {
731 switch (intrin->intrinsic) {
733 assert(intrin->src[1].is_ssa);
734 nir_ssa_def *idx = intrin->src[1].ssa;
761 if (intrin->intrinsic == nir_intrinsic_ballot_bit_count_inclusive) {
767 assert(intrin->src[0].is_ssa);
768 nir_ssa_def *int_val = ballot_type_to_uint(b, intrin->src[0].ssa,
783 return lower_shuffle(b, intrin);
784 else if (options->lower_to_scalar && intrin->num_components > 1)
785 return lower_subgroup_op_to_scalar(b, intrin, options->lower_shuffle_to_32bit);
786 else if (options->lower_shuffle_to_32bit && intrin->src[0].ssa->bit_size == 64)
787 return lower_subgroup_op_to_32bit(b, intrin);
793 return lower_to_shuffle(b, intrin, options);
794 else if (options->lower_to_scalar && intrin->num_components > 1)
795 return lower_subgroup_op_to_scalar(b, intrin, options->lower_shuffle_to_32bit);
796 else if (options->lower_shuffle_to_32bit && intrin->src[0].ssa->bit_size == 64)
797 return lower_subgroup_op_to_32bit(b, intrin);
806 intrin->intrinsic == nir_intrinsic_quad_broadcast &&
807 !nir_src_is_const(intrin->src[1])))
808 return lower_dynamic_quad_broadcast(b, intrin, options);
809 else if (options->lower_to_scalar && intrin->num_components > 1)
810 return lower_subgroup_op_to_scalar(b, intrin, false);
817 nir_intrinsic_cluster_size(intrin) >= options->subgroup_size) {
818 nir_intrinsic_set_cluster_size(intrin, 0);
821 if (options->lower_to_scalar && intrin->num_components > 1)
822 ret = lower_subgroup_op_to_scalar(b, intrin, false);
827 if (options->lower_to_scalar && intrin->num_components > 1)
828 return lower_subgroup_op_to_scalar(b, intrin, false);