xref: /third_party/vixl/src/aarch64/logic-aarch64.cc

185 void Simulator::ld1(VectorFormat vform, LogicVRegister dst, uint64_t addr) {
186   dst.ClearForWrite(vform);
187   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
188     LoadLane(dst, vform, i, addr);
189     addr += LaneSizeInBytesFromFormat(vform);
194 void Simulator::ld1(VectorFormat vform,
198   LoadLane(dst, vform, index, addr);
202 void Simulator::ld1r(VectorFormat vform,
208   dst.ClearForWrite(vform);
209   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
211       LoadIntToLane(dst, vform, unpack_size, i, addr);
213       LoadUintToLane(dst, vform, unpack_size, i, addr);
219 void Simulator::ld1r(VectorFormat vform, LogicVRegister dst, uint64_t addr) {
220   ld1r(vform, vform, dst, addr);
224 void Simulator::ld2(VectorFormat vform,
228   dst1.ClearForWrite(vform);
229   dst2.ClearForWrite(vform);
230   int esize = LaneSizeInBytesFromFormat(vform);
232   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
233     LoadLane(dst1, vform, i, addr1);
234     LoadLane(dst2, vform, i, addr2);
241 void Simulator::ld2(VectorFormat vform,
246   dst1.ClearForWrite(vform);
247   dst2.ClearForWrite(vform);
248   uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
249   LoadLane(dst1, vform, index, addr1);
250   LoadLane(dst2, vform, index, addr2);
254 void Simulator::ld2r(VectorFormat vform,
258   dst1.ClearForWrite(vform);
259   dst2.ClearForWrite(vform);
260   uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
261   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
262     LoadLane(dst1, vform, i, addr);
263     LoadLane(dst2, vform, i, addr2);
268 void Simulator::ld3(VectorFormat vform,
273   dst1.ClearForWrite(vform);
274   dst2.ClearForWrite(vform);
275   dst3.ClearForWrite(vform);
276   int esize = LaneSizeInBytesFromFormat(vform);
279   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
280     LoadLane(dst1, vform, i, addr1);
281     LoadLane(dst2, vform, i, addr2);
282     LoadLane(dst3, vform, i, addr3);
290 void Simulator::ld3(VectorFormat vform,
296   dst1.ClearForWrite(vform);
297   dst2.ClearForWrite(vform);
298   dst3.ClearForWrite(vform);
299   uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
300   uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
301   LoadLane(dst1, vform, index, addr1);
302   LoadLane(dst2, vform, index, addr2);
303   LoadLane(dst3, vform, index, addr3);
307 void Simulator::ld3r(VectorFormat vform,
312   dst1.ClearForWrite(vform);
313   dst2.ClearForWrite(vform);
314   dst3.ClearForWrite(vform);
315   uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
316   uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
317   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
318     LoadLane(dst1, vform, i, addr);
319     LoadLane(dst2, vform, i, addr2);
320     LoadLane(dst3, vform, i, addr3);
325 void Simulator::ld4(VectorFormat vform,
331   dst1.ClearForWrite(vform);
332   dst2.ClearForWrite(vform);
333   dst3.ClearForWrite(vform);
334   dst4.ClearForWrite(vform);
335   int esize = LaneSizeInBytesFromFormat(vform);
339   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
340     LoadLane(dst1, vform, i, addr1);
341     LoadLane(dst2, vform, i, addr2);
342     LoadLane(dst3, vform, i, addr3);
343     LoadLane(dst4, vform, i, addr4);
352 void Simulator::ld4(VectorFormat vform,
359   dst1.ClearForWrite(vform);
360   dst2.ClearForWrite(vform);
361   dst3.ClearForWrite(vform);
362   dst4.ClearForWrite(vform);
363   uint64_t addr2 = addr1 + LaneSizeInBytesFromFormat(vform);
364   uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
365   uint64_t addr4 = addr3 + LaneSizeInBytesFromFormat(vform);
366   LoadLane(dst1, vform, index, addr1);
367   LoadLane(dst2, vform, index, addr2);
368   LoadLane(dst3, vform, index, addr3);
369   LoadLane(dst4, vform, index, addr4);
373 void Simulator::ld4r(VectorFormat vform,
379   dst1.ClearForWrite(vform);
380   dst2.ClearForWrite(vform);
381   dst3.ClearForWrite(vform);
382   dst4.ClearForWrite(vform);
383   uint64_t addr2 = addr + LaneSizeInBytesFromFormat(vform);
384   uint64_t addr3 = addr2 + LaneSizeInBytesFromFormat(vform);
385   uint64_t addr4 = addr3 + LaneSizeInBytesFromFormat(vform);
386   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
387     LoadLane(dst1, vform, i, addr);
388     LoadLane(dst2, vform, i, addr2);
389     LoadLane(dst3, vform, i, addr3);
390     LoadLane(dst4, vform, i, addr4);
395 void Simulator::st1(VectorFormat vform, LogicVRegister src, uint64_t addr) {
396   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
397     StoreLane(src, vform, i, addr);
398     addr += LaneSizeInBytesFromFormat(vform);
403 void Simulator::st1(VectorFormat vform,
407   StoreLane(src, vform, index, addr);
411 void Simulator::st2(VectorFormat vform,
415   int esize = LaneSizeInBytesFromFormat(vform);
417   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
418     StoreLane(src, vform, i, addr);
419     StoreLane(src2, vform, i, addr2);
426 void Simulator::st2(VectorFormat vform,
431   int esize = LaneSizeInBytesFromFormat(vform);
432   StoreLane(src, vform, index, addr);
433   StoreLane(src2, vform, index, addr + 1 * esize);
437 void Simulator::st3(VectorFormat vform,
442   int esize = LaneSizeInBytesFromFormat(vform);
445   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
446     StoreLane(src, vform, i, addr);
447     StoreLane(src2, vform, i, addr2);
448     StoreLane(src3, vform, i, addr3);
456 void Simulator::st3(VectorFormat vform,
462   int esize = LaneSizeInBytesFromFormat(vform);
463   StoreLane(src, vform, index, addr);
464   StoreLane(src2, vform, index, addr + 1 * esize);
465   StoreLane(src3, vform, index, addr + 2 * esize);
469 void Simulator::st4(VectorFormat vform,
475   int esize = LaneSizeInBytesFromFormat(vform);
479   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
480     StoreLane(src, vform, i, addr);
481     StoreLane(src2, vform, i, addr2);
482     StoreLane(src3, vform, i, addr3);
483     StoreLane(src4, vform, i, addr4);
492 void Simulator::st4(VectorFormat vform,
499   int esize = LaneSizeInBytesFromFormat(vform);
500   StoreLane(src, vform, index, addr);
501   StoreLane(src2, vform, index, addr + 1 * esize);
502   StoreLane(src3, vform, index, addr + 2 * esize);
503   StoreLane(src4, vform, index, addr + 3 * esize);
507 LogicVRegister Simulator::cmp(VectorFormat vform,
512   dst.ClearForWrite(vform);
513   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
514     int64_t sa = src1.Int(vform, i);
515     int64_t sb = src2.Int(vform, i);
516     uint64_t ua = src1.Uint(vform, i);
517     uint64_t ub = src2.Uint(vform, i);
545     dst.SetUint(vform, i, result ? MaxUintFromFormat(vform) : 0);
551 LogicVRegister Simulator::cmp(VectorFormat vform,
557   LogicVRegister imm_reg = dup_immediate(vform, temp, imm);
558   return cmp(vform, dst, src1, imm_reg, cond);
562 LogicVRegister Simulator::cmptst(VectorFormat vform,
566   dst.ClearForWrite(vform);
567   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
568     uint64_t ua = src1.Uint(vform, i);
569     uint64_t ub = src2.Uint(vform, i);
570     dst.SetUint(vform, i, ((ua & ub) != 0) ? MaxUintFromFormat(vform) : 0);
576 LogicVRegister Simulator::add(VectorFormat vform,
580   int lane_size = LaneSizeInBitsFromFormat(vform);
581   dst.ClearForWrite(vform);
583   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
585     uint64_t ua = src1.UintLeftJustified(vform, i);
586     uint64_t ub = src2.UintLeftJustified(vform, i);
601     dst.SetInt(vform, i, ur >> (64 - lane_size));
606 LogicVRegister Simulator::add_uint(VectorFormat vform,
610   int lane_size = LaneSizeInBitsFromFormat(vform);
612   dst.ClearForWrite(vform);
615   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
617     uint64_t ua = src1.UintLeftJustified(vform, i);
630     dst.SetInt(vform, i, ur >> (64 - lane_size));
635 LogicVRegister Simulator::addp(VectorFormat vform,
640   uzp1(vform, temp1, src1, src2);
641   uzp2(vform, temp2, src1, src2);
642   add(vform, dst, temp1, temp2);
643   if (IsSVEFormat(vform)) {
644     interleave_top_bottom(vform, dst, dst);
649 LogicVRegister Simulator::sdiv(VectorFormat vform,
653   VIXL_ASSERT((vform == kFormatVnS) || (vform == kFormatVnD));
655   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
656     int64_t val1 = src1.Int(vform, i);
657     int64_t val2 = src2.Int(vform, i);
658     int64_t min_int = (vform == kFormatVnD) ? kXMinInt : kWMinInt;
665     dst.SetInt(vform, i, quotient);
671 LogicVRegister Simulator::udiv(VectorFormat vform,
675   VIXL_ASSERT((vform == kFormatVnS) || (vform == kFormatVnD));
677   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
678     uint64_t val1 = src1.Uint(vform, i);
679     uint64_t val2 = src2.Uint(vform, i);
684     dst.SetUint(vform, i, quotient);
691 LogicVRegister Simulator::mla(VectorFormat vform,
697   mul(vform, temp, src1, src2);
698   add(vform, dst, srca, temp);
703 LogicVRegister Simulator::mls(VectorFormat vform,
709   mul(vform, temp, src1, src2);
710   sub(vform, dst, srca, temp);
715 LogicVRegister Simulator::mul(VectorFormat vform,
719   dst.ClearForWrite(vform);
721   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
722     dst.SetUint(vform, i, src1.Uint(vform, i) * src2.Uint(vform, i));
728 LogicVRegister Simulator::mul(VectorFormat vform,
734   VectorFormat indexform = VectorFormatFillQ(vform);
735   return mul(vform, dst, src1, dup_element(indexform, temp, src2, index));
739 LogicVRegister Simulator::smulh(VectorFormat vform,
743   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
745     int64_t val1 = src1.Int(vform, i);
746     int64_t val2 = src2.Int(vform, i);
747     switch (LaneSizeInBitsFromFormat(vform)) {
764     dst.SetInt(vform, i, dst_val);
770 LogicVRegister Simulator::umulh(VectorFormat vform,
774   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
776     uint64_t val1 = src1.Uint(vform, i);
777     uint64_t val2 = src2.Uint(vform, i);
778     switch (LaneSizeInBitsFromFormat(vform)) {
795     dst.SetUint(vform, i, dst_val);
801 LogicVRegister Simulator::mla(VectorFormat vform,
807   VectorFormat indexform = VectorFormatFillQ(vform);
808   return mla(vform, dst, dst, src1, dup_element(indexform, temp, src2, index));
812 LogicVRegister Simulator::mls(VectorFormat vform,
818   VectorFormat indexform = VectorFormatFillQ(vform);
819   return mls(vform, dst, dst, src1, dup_element(indexform, temp, src2, index));
822 LogicVRegister Simulator::sqdmull(VectorFormat vform,
829       VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
830   return sqdmull(vform, dst, src1, dup_element(indexform, temp, src2, index));
833 LogicVRegister Simulator::sqdmlal(VectorFormat vform,
840       VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
841   return sqdmlal(vform, dst, src1, dup_element(indexform, temp, src2, index));
844 LogicVRegister Simulator::sqdmlsl(VectorFormat vform,
851       VectorFormatHalfWidthDoubleLanes(VectorFormatFillQ(vform));
852   return sqdmlsl(vform, dst, src1, dup_element(indexform, temp, src2, index));
855 LogicVRegister Simulator::sqdmulh(VectorFormat vform,
861   VectorFormat indexform = VectorFormatFillQ(vform);
862   return sqdmulh(vform, dst, src1, dup_element(indexform, temp, src2, index));
866 LogicVRegister Simulator::sqrdmulh(VectorFormat vform,
872   VectorFormat indexform = VectorFormatFillQ(vform);
873   return sqrdmulh(vform, dst, src1, dup_element(indexform, temp, src2, index));
877 LogicVRegister Simulator::sqrdmlah(VectorFormat vform,
883   VectorFormat indexform = VectorFormatFillQ(vform);
884   return sqrdmlah(vform, dst, src1, dup_element(indexform, temp, src2, index));
888 LogicVRegister Simulator::sqrdmlsh(VectorFormat vform,
894   VectorFormat indexform = VectorFormatFillQ(vform);
895   return sqrdmlsh(vform, dst, src1, dup_element(indexform, temp, src2, index));
915 LogicVRegister Simulator::pmul(VectorFormat vform,
919   dst.ClearForWrite(vform);
920   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
921     dst.SetUint(vform,
923                 PolynomialMult(src1.Uint(vform, i),
924                                src2.Uint(vform, i),
925                                LaneSizeInBitsFromFormat(vform)));
931 LogicVRegister Simulator::pmull(VectorFormat vform,
935   dst.ClearForWrite(vform);
937   VectorFormat vform_src = VectorFormatHalfWidth(vform);
938   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
939     dst.SetUint(vform,
950 LogicVRegister Simulator::pmull2(VectorFormat vform,
954   VectorFormat vform_src = VectorFormatHalfWidthDoubleLanes(vform);
955   dst.ClearForWrite(vform);
956   int lane_count = LaneCountFromFormat(vform);
958     dst.SetUint(vform,
968 LogicVRegister Simulator::sub(VectorFormat vform,
972   int lane_size = LaneSizeInBitsFromFormat(vform);
973   dst.ClearForWrite(vform);
974   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
976     uint64_t ua = src1.UintLeftJustified(vform, i);
977     uint64_t ub = src2.UintLeftJustified(vform, i);
993     dst.SetInt(vform, i, ur >> (64 - lane_size));
998 LogicVRegister Simulator::sub_uint(VectorFormat vform,
1002   int lane_size = LaneSizeInBitsFromFormat(vform);
1004   dst.ClearForWrite(vform);
1007   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1009     uint64_t ua = src1.UintLeftJustified(vform, i);
1022     dst.SetInt(vform, i, ur >> (64 - lane_size));
1027 LogicVRegister Simulator::and_(VectorFormat vform,
1031   dst.ClearForWrite(vform);
1032   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1033     dst.SetUint(vform, i, src1.Uint(vform, i) & src2.Uint(vform, i));
1039 LogicVRegister Simulator::orr(VectorFormat vform,
1043   dst.ClearForWrite(vform);
1044   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1045     dst.SetUint(vform, i, src1.Uint(vform, i) | src2.Uint(vform, i));
1051 LogicVRegister Simulator::orn(VectorFormat vform,
1055   dst.ClearForWrite(vform);
1056   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1057     dst.SetUint(vform, i, src1.Uint(vform, i) | ~src2.Uint(vform, i));
1063 LogicVRegister Simulator::eor(VectorFormat vform,
1067   dst.ClearForWrite(vform);
1068   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1069     dst.SetUint(vform, i, src1.Uint(vform, i) ^ src2.Uint(vform, i));
1075 LogicVRegister Simulator::bic(VectorFormat vform,
1079   dst.ClearForWrite(vform);
1080   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1081     dst.SetUint(vform, i, src1.Uint(vform, i) & ~src2.Uint(vform, i));
1087 LogicVRegister Simulator::bic(VectorFormat vform,
1092   int lane_count = LaneCountFromFormat(vform);
1094     result[i] = src.Uint(vform, i) & ~imm;
1096   dst.ClearForWrite(vform);
1098     dst.SetUint(vform, i, result[i]);
1104 LogicVRegister Simulator::bif(VectorFormat vform,
1108   dst.ClearForWrite(vform);
1109   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1110     uint64_t operand1 = dst.Uint(vform, i);
1111     uint64_t operand2 = ~src2.Uint(vform, i);
1112     uint64_t operand3 = src1.Uint(vform, i);
1114     dst.SetUint(vform, i, result);
1120 LogicVRegister Simulator::bit(VectorFormat vform,
1124   dst.ClearForWrite(vform);
1125   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1126     uint64_t operand1 = dst.Uint(vform, i);
1127     uint64_t operand2 = src2.Uint(vform, i);
1128     uint64_t operand3 = src1.Uint(vform, i);
1130     dst.SetUint(vform, i, result);
1136 LogicVRegister Simulator::bsl(VectorFormat vform,
1141   dst.ClearForWrite(vform);
1142   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1143     uint64_t operand1 = src2.Uint(vform, i);
1144     uint64_t operand2 = src_mask.Uint(vform, i);
1145     uint64_t operand3 = src1.Uint(vform, i);
1147     dst.SetUint(vform, i, result);
1153 LogicVRegister Simulator::sminmax(VectorFormat vform,
1158   dst.ClearForWrite(vform);
1159   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1160     int64_t src1_val = src1.Int(vform, i);
1161     int64_t src2_val = src2.Int(vform, i);
1168     dst.SetInt(vform, i, dst_val);
1174 LogicVRegister Simulator::smax(VectorFormat vform,
1178   return sminmax(vform, dst, src1, src2, true);
1182 LogicVRegister Simulator::smin(VectorFormat vform,
1186   return sminmax(vform, dst, src1, src2, false);
1190 LogicVRegister Simulator::sminmaxp(VectorFormat vform,
1195   unsigned lanes = LaneCountFromFormat(vform);
1200       int64_t first_val = src->Int(vform, i);
1201       int64_t second_val = src->Int(vform, i + 1);
1213   dst.SetIntArray(vform, result);
1214   if (IsSVEFormat(vform)) {
1215     interleave_top_bottom(vform, dst, dst);
1221 LogicVRegister Simulator::smaxp(VectorFormat vform,
1225   return sminmaxp(vform, dst, src1, src2, true);
1229 LogicVRegister Simulator::sminp(VectorFormat vform,
1233   return sminmaxp(vform, dst, src1, src2, false);
1237 LogicVRegister Simulator::addp(VectorFormat vform,
1240   VIXL_ASSERT(vform == kFormatD);
1243   dst.ClearForWrite(vform);
1244   dst.SetUint(vform, 0, dst_val);
1249 LogicVRegister Simulator::addv(VectorFormat vform,
1253       ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
1257   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1258     dst_val += src.Int(vform, i);
1267 LogicVRegister Simulator::saddlv(VectorFormat vform,
1271       ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform) * 2);
1274   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1275     dst_val += src.Int(vform, i);
1284 LogicVRegister Simulator::uaddlv(VectorFormat vform,
1288       ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform) * 2);
1291   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1292     dst_val += src.Uint(vform, i);
1301 LogicVRegister Simulator::sminmaxv(VectorFormat vform,
1307   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1308     if (!pg.IsActive(vform, i)) continue;
1310     int64_t src_val = src.Int(vform, i);
1317   dst.ClearForWrite(ScalarFormatFromFormat(vform));
1318   dst.SetInt(vform, 0, dst_val);
1323 LogicVRegister Simulator::smaxv(VectorFormat vform,
1326   sminmaxv(vform, dst, GetPTrue(), src, true);
1331 LogicVRegister Simulator::sminv(VectorFormat vform,
1334   sminmaxv(vform, dst, GetPTrue(), src, false);
1339 LogicVRegister Simulator::smaxv(VectorFormat vform,
1343   VIXL_ASSERT(IsSVEFormat(vform));
1344   sminmaxv(vform, dst, pg, src, true);
1349 LogicVRegister Simulator::sminv(VectorFormat vform,
1353   VIXL_ASSERT(IsSVEFormat(vform));
1354   sminmaxv(vform, dst, pg, src, false);
1359 LogicVRegister Simulator::uminmax(VectorFormat vform,
1364   dst.ClearForWrite(vform);
1365   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1366     uint64_t src1_val = src1.Uint(vform, i);
1367     uint64_t src2_val = src2.Uint(vform, i);
1374     dst.SetUint(vform, i, dst_val);
1380 LogicVRegister Simulator::umax(VectorFormat vform,
1384   return uminmax(vform, dst, src1, src2, true);
1388 LogicVRegister Simulator::umin(VectorFormat vform,
1392   return uminmax(vform, dst, src1, src2, false);
1396 LogicVRegister Simulator::uminmaxp(VectorFormat vform,
1401   unsigned lanes = LaneCountFromFormat(vform);
1406       uint64_t first_val = src->Uint(vform, i);
1407       uint64_t second_val = src->Uint(vform, i + 1);
1419   dst.SetUintArray(vform, result);
1420   if (IsSVEFormat(vform)) {
1421     interleave_top_bottom(vform, dst, dst);
1427 LogicVRegister Simulator::umaxp(VectorFormat vform,
1431   return uminmaxp(vform, dst, src1, src2, true);
1435 LogicVRegister Simulator::uminp(VectorFormat vform,
1439   return uminmaxp(vform, dst, src1, src2, false);
1443 LogicVRegister Simulator::uminmaxv(VectorFormat vform,
1449   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1450     if (!pg.IsActive(vform, i)) continue;
1452     uint64_t src_val = src.Uint(vform, i);
1459   dst.ClearForWrite(ScalarFormatFromFormat(vform));
1460   dst.SetUint(vform, 0, dst_val);
1465 LogicVRegister Simulator::umaxv(VectorFormat vform,
1468   uminmaxv(vform, dst, GetPTrue(), src, true);
1473 LogicVRegister Simulator::uminv(VectorFormat vform,
1476   uminmaxv(vform, dst, GetPTrue(), src, false);
1481 LogicVRegister Simulator::umaxv(VectorFormat vform,
1485   VIXL_ASSERT(IsSVEFormat(vform));
1486   uminmaxv(vform, dst, pg, src, true);
1491 LogicVRegister Simulator::uminv(VectorFormat vform,
1495   VIXL_ASSERT(IsSVEFormat(vform));
1496   uminmaxv(vform, dst, pg, src, false);
1501 LogicVRegister Simulator::shl(VectorFormat vform,
1507   LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
1508   return ushl(vform, dst, src, shiftreg);
1512 LogicVRegister Simulator::sshll(VectorFormat vform,
1518   LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
1519   LogicVRegister extendedreg = sxtl(vform, temp2, src);
1520   return sshl(vform, dst, extendedreg, shiftreg);
1524 LogicVRegister Simulator::sshll2(VectorFormat vform,
1530   LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
1531   LogicVRegister extendedreg = sxtl2(vform, temp2, src);
1532   return sshl(vform, dst, extendedreg, shiftreg);
1536 LogicVRegister Simulator::shll(VectorFormat vform,
1539   int shift = LaneSizeInBitsFromFormat(vform) / 2;
1540   return sshll(vform, dst, src, shift);
1544 LogicVRegister Simulator::shll2(VectorFormat vform,
1547   int shift = LaneSizeInBitsFromFormat(vform) / 2;
1548   return sshll2(vform, dst, src, shift);
1552 LogicVRegister Simulator::ushll(VectorFormat vform,
1558   LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
1559   LogicVRegister extendedreg = uxtl(vform, temp2, src);
1560   return ushl(vform, dst, extendedreg, shiftreg);
1564 LogicVRegister Simulator::ushll2(VectorFormat vform,
1570   LogicVRegister shiftreg = dup_immediate(vform, temp1, shift);
1571   LogicVRegister extendedreg = uxtl2(vform, temp2, src);
1572   return ushl(vform, dst, extendedreg, shiftreg);
1575 std::pair<bool, uint64_t> Simulator::clast(VectorFormat vform,
1582   int last_active = GetLastActive(vform, pg);
1583   int lane_count = LaneCountFromFormat(vform);
1586   return std::make_pair(last_active >= 0, src.Uint(vform, index));
1589 LogicVRegister Simulator::compact(VectorFormat vform,
1594   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1595     if (pg.IsActive(vform, i)) {
1596       dst.SetUint(vform, j++, src.Uint(vform, i));
1599   for (; j < LaneCountFromFormat(vform); j++) {
1600     dst.SetUint(vform, j, 0);
1605 LogicVRegister Simulator::splice(VectorFormat vform,
1610   int lane_count = LaneCountFromFormat(vform);
1611   int first_active = GetFirstActive(vform, pg);
1612   int last_active = GetLastActive(vform, pg);
1620       result[dst_idx++] = src1.Uint(vform, i);
1626     result[i] = src2.Uint(vform, i - dst_idx);
1629   dst.SetUintArray(vform, result);
1634 LogicVRegister Simulator::sel(VectorFormat vform,
1640       LaneSizeInBitsFromFormat(vform) / kZRegBitsPerPRegBit;
1641   for (int lane = 0; lane < LaneCountFromFormat(vform); lane++) {
1643                               ? src1.Uint(vform, lane)
1644                               : src2.Uint(vform, lane);
1645     dst.SetUint(vform, lane, lane_value);
1665 LogicVRegister Simulator::sli(VectorFormat vform,
1669   dst.ClearForWrite(vform);
1670   int lane_count = LaneCountFromFormat(vform);
1672     uint64_t src_lane = src.Uint(vform, i);
1673     uint64_t dst_lane = dst.Uint(vform, i);
1675     uint64_t mask = MaxUintFromFormat(vform) << shift;
1676     dst.SetUint(vform, i, (dst_lane & ~mask) | shifted);
1682 LogicVRegister Simulator::sqshl(VectorFormat vform,
1688   LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
1689   return sshl(vform, dst, src, shiftreg).SignedSaturate(vform);
1693 LogicVRegister Simulator::uqshl(VectorFormat vform,
1699   LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
1700   return ushl(vform, dst, src, shiftreg).UnsignedSaturate(vform);
1704 LogicVRegister Simulator::sqshlu(VectorFormat vform,
1710   LogicVRegister shiftreg = dup_immediate(vform, temp, shift);
1711   return sshl(vform, dst, src, shiftreg).UnsignedSaturate(vform);
1715 LogicVRegister Simulator::sri(VectorFormat vform,
1719   dst.ClearForWrite(vform);
1720   int lane_count = LaneCountFromFormat(vform);
1722               (shift <= static_cast<int>(LaneSizeInBitsFromFormat(vform))));
1724     uint64_t src_lane = src.Uint(vform, i);
1725     uint64_t dst_lane = dst.Uint(vform, i);
1733       mask = MaxUintFromFormat(vform) >> shift;
1735     dst.SetUint(vform, i, (dst_lane & ~mask) | shifted);
1741 LogicVRegister Simulator::ushr(VectorFormat vform,
1747   LogicVRegister shiftreg = dup_immediate(vform, temp, -shift);
1748   return ushl(vform, dst, src, shiftreg);
1752 LogicVRegister Simulator::sshr(VectorFormat vform,
1758   LogicVRegister shiftreg = dup_immediate(vform, temp, -shift);
1759   return sshl(vform, dst, src, shiftreg);
1763 LogicVRegister Simulator::ssra(VectorFormat vform,
1768   LogicVRegister shifted_reg = sshr(vform, temp, src, shift);
1769   return add(vform, dst, dst, shifted_reg);
1773 LogicVRegister Simulator::usra(VectorFormat vform,
1778   LogicVRegister shifted_reg = ushr(vform, temp, src, shift);
1779   return add(vform, dst, dst, shifted_reg);
1783 LogicVRegister Simulator::srsra(VectorFormat vform,
1788   LogicVRegister shifted_reg = sshr(vform, temp, src, shift).Round(vform);
1789   return add(vform, dst, dst, shifted_reg);
1793 LogicVRegister Simulator::ursra(VectorFormat vform,
1798   LogicVRegister shifted_reg = ushr(vform, temp, src, shift).Round(vform);
1799   return add(vform, dst, dst, shifted_reg);
1803 LogicVRegister Simulator::cls(VectorFormat vform,
1806   int lane_size_in_bits = LaneSizeInBitsFromFormat(vform);
1807   int lane_count = LaneCountFromFormat(vform);
1813     result[i] = CountLeadingSignBits(src.Int(vform, i), lane_size_in_bits);
1816   dst.ClearForWrite(vform);
1818     dst.SetUint(vform, i, result[i]);
1824 LogicVRegister Simulator::clz(VectorFormat vform,
1827   int lane_size_in_bits = LaneSizeInBitsFromFormat(vform);
1828   int lane_count = LaneCountFromFormat(vform);
1834     result[i] = CountLeadingZeros(src.Uint(vform, i), lane_size_in_bits);
1837   dst.ClearForWrite(vform);
1839     dst.SetUint(vform, i, result[i]);
1845 LogicVRegister Simulator::cnot(VectorFormat vform,
1848   dst.ClearForWrite(vform);
1849   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1850     uint64_t value = (src.Uint(vform, i) == 0) ? 1 : 0;
1851     dst.SetUint(vform, i, value);
1857 LogicVRegister Simulator::cnt(VectorFormat vform,
1860   int lane_size_in_bits = LaneSizeInBitsFromFormat(vform);
1861   int lane_count = LaneCountFromFormat(vform);
1867     result[i] = CountSetBits(src.Uint(vform, i), lane_size_in_bits);
1870   dst.ClearForWrite(vform);
1872     dst.SetUint(vform, i, result[i]);
1892 LogicVRegister Simulator::sshl(VectorFormat vform,
1897   dst.ClearForWrite(vform);
1898   int esize = LaneSizeInBitsFromFormat(vform);
1899   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1900     int64_t shift_val = CalculateSignedShiftDistance(src2.Int(vform, i),
1904     int64_t lj_src_val = src1.IntLeftJustified(vform, i);
1919     int64_t src_val = src1.Int(vform, i);
1922       dst.SetInt(vform, i, 0);
1925       dst.SetInt(vform, i, src_is_negative ? -1 : 0);
1950       dst.SetUint(vform, i, usrc_val);
1957 LogicVRegister Simulator::ushl(VectorFormat vform,
1962   dst.ClearForWrite(vform);
1963   int esize = LaneSizeInBitsFromFormat(vform);
1964   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
1965     int64_t shift_val = CalculateSignedShiftDistance(src2.Int(vform, i),
1969     uint64_t lj_src_val = src1.UintLeftJustified(vform, i);
1976     uint64_t src_val = src1.Uint(vform, i);
1978       dst.SetUint(vform, i, 0);
1994       dst.SetUint(vform, i, src_val);
2000 LogicVRegister Simulator::sshr(VectorFormat vform,
2006   neg(vform, temp, src2).SignedSaturate(vform);
2007   sshl(vform, dst, src1, temp, false);
2011 LogicVRegister Simulator::ushr(VectorFormat vform,
2017   neg(vform, temp, src2).SignedSaturate(vform);
2018   ushl(vform, dst, src1, temp, false);
2022 LogicVRegister Simulator::neg(VectorFormat vform,
2025   dst.ClearForWrite(vform);
2026   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2028     int64_t sa = src.Int(vform, i);
2029     if (sa == MinIntFromFormat(vform)) {
2032     dst.SetInt(vform, i, (sa == INT64_MIN) ? sa : -sa);
2038 LogicVRegister Simulator::suqadd(VectorFormat vform,
2042   dst.ClearForWrite(vform);
2043   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2044     int64_t sa = src1.IntLeftJustified(vform, i);
2045     uint64_t ub = src2.UintLeftJustified(vform, i);
2051       dst.SetInt(vform, i, MaxIntFromFormat(vform));
2053       dst.SetUint(vform, i, src1.Int(vform, i) + src2.Uint(vform, i));
2060 LogicVRegister Simulator::usqadd(VectorFormat vform,
2064   dst.ClearForWrite(vform);
2065   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2066     uint64_t ua = src1.UintLeftJustified(vform, i);
2067     int64_t sb = src2.IntLeftJustified(vform, i);
2071       dst.SetUint(vform, i, MaxUintFromFormat(vform));  // Positive saturation.
2073       dst.SetUint(vform, i, 0);  // Negative saturation.
2075       dst.SetUint(vform, i, src1.Uint(vform, i) + src2.Int(vform, i));
2082 LogicVRegister Simulator::abs(VectorFormat vform,
2085   dst.ClearForWrite(vform);
2086   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2088     int64_t sa = src.Int(vform, i);
2089     if (sa == MinIntFromFormat(vform)) {
2093       dst.SetInt(vform, i, (sa == INT64_MIN) ? sa : -sa);
2095       dst.SetInt(vform, i, sa);
2102 LogicVRegister Simulator::andv(VectorFormat vform,
2106   VIXL_ASSERT(IsSVEFormat(vform));
2107   uint64_t result = GetUintMask(LaneSizeInBitsFromFormat(vform));
2108   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2109     if (!pg.IsActive(vform, i)) continue;
2111     result &= src.Uint(vform, i);
2114       ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
2121 LogicVRegister Simulator::eorv(VectorFormat vform,
2125   VIXL_ASSERT(IsSVEFormat(vform));
2127   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2128     if (!pg.IsActive(vform, i)) continue;
2130     result ^= src.Uint(vform, i);
2133       ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
2140 LogicVRegister Simulator::orv(VectorFormat vform,
2144   VIXL_ASSERT(IsSVEFormat(vform));
2146   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2147     if (!pg.IsActive(vform, i)) continue;
2149     result |= src.Uint(vform, i);
2152       ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
2159 LogicVRegister Simulator::saddv(VectorFormat vform,
2163   VIXL_ASSERT(IsSVEFormat(vform));
2164   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) <= kSRegSize);
2166   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2167     if (!pg.IsActive(vform, i)) continue;
2172     result += src.Int(vform, i);
2181 LogicVRegister Simulator::uaddv(VectorFormat vform,
2185   VIXL_ASSERT(IsSVEFormat(vform));
2187   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2188     if (!pg.IsActive(vform, i)) continue;
2190     result += src.Uint(vform, i);
2267 LogicVRegister Simulator::xtn(VectorFormat vform,
2270   return extractnarrow(vform, dst, true, src, true);
2274 LogicVRegister Simulator::sqxtn(VectorFormat vform,
2277   return extractnarrow(vform, dst, true, src, true).SignedSaturate(vform);
2281 LogicVRegister Simulator::sqxtun(VectorFormat vform,
2284   return extractnarrow(vform, dst, false, src, true).UnsignedSaturate(vform);
2288 LogicVRegister Simulator::uqxtn(VectorFormat vform,
2291   return extractnarrow(vform, dst, false, src, false).UnsignedSaturate(vform);
2295 LogicVRegister Simulator::absdiff(VectorFormat vform,
2300   dst.ClearForWrite(vform);
2301   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2302     bool src1_gt_src2 = is_signed ? (src1.Int(vform, i) > src2.Int(vform, i))
2303                                   : (src1.Uint(vform, i) > src2.Uint(vform, i));
2307       dst.SetUint(vform, i, src1.Uint(vform, i) - src2.Uint(vform, i));
2309       dst.SetUint(vform, i, src2.Uint(vform, i) - src1.Uint(vform, i));
2316 LogicVRegister Simulator::saba(VectorFormat vform,
2321   dst.ClearForWrite(vform);
2322   absdiff(vform, temp, src1, src2, true);
2323   add(vform, dst, dst, temp);
2328 LogicVRegister Simulator::uaba(VectorFormat vform,
2333   dst.ClearForWrite(vform);
2334   absdiff(vform, temp, src1, src2, false);
2335   add(vform, dst, dst, temp);
2340 LogicVRegister Simulator::not_(VectorFormat vform,
2343   dst.ClearForWrite(vform);
2344   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2345     dst.SetUint(vform, i, ~src.Uint(vform, i));
2351 LogicVRegister Simulator::rbit(VectorFormat vform,
2355   int lane_count = LaneCountFromFormat(vform);
2356   int lane_size_in_bits = LaneSizeInBitsFromFormat(vform);
2360     value = src.Uint(vform, i);
2369   dst.ClearForWrite(vform);
2371     dst.SetUint(vform, i, result[i]);
2377 LogicVRegister Simulator::rev(VectorFormat vform,
2380   VIXL_ASSERT(IsSVEFormat(vform));
2381   int lane_count = LaneCountFromFormat(vform);
2383     uint64_t t = src.Uint(vform, i);
2384     dst.SetUint(vform, i, src.Uint(vform, lane_count - i - 1));
2385     dst.SetUint(vform, lane_count - i - 1, t);
2391 LogicVRegister Simulator::rev_byte(VectorFormat vform,
2396   int lane_count = LaneCountFromFormat(vform);
2397   int lane_size = LaneSizeInBytesFromFormat(vform);
2401       result[i + lanes_per_loop - 1 - j] = src.Uint(vform, i + j);
2404   dst.ClearForWrite(vform);
2406     dst.SetUint(vform, i, result[i]);
2412 LogicVRegister Simulator::rev16(VectorFormat vform,
2415   return rev_byte(vform, dst, src, 2);
2419 LogicVRegister Simulator::rev32(VectorFormat vform,
2422   return rev_byte(vform, dst, src, 4);
2426 LogicVRegister Simulator::rev64(VectorFormat vform,
2429   return rev_byte(vform, dst, src, 8);
2432 LogicVRegister Simulator::addlp(VectorFormat vform,
2437   VectorFormat vformsrc = VectorFormatHalfWidthDoubleLanes(vform);
2441   int lane_count = LaneCountFromFormat(vform);
2451   dst.ClearForWrite(vform);
2454       result[i] += dst.Uint(vform, i);
2456     dst.SetUint(vform, i, result[i]);
2463 LogicVRegister Simulator::saddlp(VectorFormat vform,
2466   return addlp(vform, dst, src, true, false);
2470 LogicVRegister Simulator::uaddlp(VectorFormat vform,
2473   return addlp(vform, dst, src, false, false);
2477 LogicVRegister Simulator::sadalp(VectorFormat vform,
2480   return addlp(vform, dst, src, true, true);
2484 LogicVRegister Simulator::uadalp(VectorFormat vform,
2487   return addlp(vform, dst, src, false, true);
2490 LogicVRegister Simulator::ror(VectorFormat vform,
2494   int width = LaneSizeInBitsFromFormat(vform);
2495   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2496     uint64_t value = src.Uint(vform, i);
2497     dst.SetUint(vform, i, RotateRight(value, rotation, width));
2502 LogicVRegister Simulator::ext(VectorFormat vform,
2508   int lane_count = LaneCountFromFormat(vform);
2510     result[i] = src1.Uint(vform, i + index);
2513     result[lane_count - index + i] = src2.Uint(vform, i);
2515   dst.ClearForWrite(vform);
2517     dst.SetUint(vform, i, result[i]);
2522 LogicVRegister Simulator::rotate_elements_right(VectorFormat vform,
2526   if (index < 0) index += LaneCountFromFormat(vform);
2527   VIXL_ASSERT((index >= 0) && (index < LaneCountFromFormat(vform)));
2528   index *= LaneSizeInBytesFromFormat(vform);
2534 LogicVRegister Simulator::fadda(VectorFormat vform,
2539   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2540     if (!pg.IsActive(vform, i)) continue;
2545       ScalarFormatFromLaneSize(LaneSizeInBitsFromFormat(vform));
2551 LogicVRegister Simulator::fadda(VectorFormat vform,
2555   switch (LaneSizeInBitsFromFormat(vform)) {
2557       fadda<SimFloat16>(vform, acc, pg, src);
2560       fadda<float>(vform, acc, pg, src);
2563       fadda<double>(vform, acc, pg, src);
2572 LogicVRegister Simulator::fcadd(VectorFormat vform,
2577   int elements = LaneCountFromFormat(vform);
2600     dst.ClearForWrite(vform);
2608 LogicVRegister Simulator::fcadd(VectorFormat vform,
2613   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
2614     fcadd<SimFloat16>(vform, dst, src1, src2, rot);
2615   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
2616     fcadd<float>(vform, dst, src1, src2, rot);
2618     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
2619     fcadd<double>(vform, dst, src1, src2, rot);
2625 LogicVRegister Simulator::fcmla(VectorFormat vform,
2632   int elements = LaneCountFromFormat(vform);
2675     dst.ClearForWrite(vform);
2676     dst.SetFloat<T>(vform,
2679     dst.SetFloat<T>(vform,
2686 LogicVRegister Simulator::fcmla(VectorFormat vform,
2692   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
2693     fcmla<SimFloat16>(vform, dst, src1, src2, acc, -1, rot);
2694   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
2695     fcmla<float>(vform, dst, src1, src2, acc, -1, rot);
2697     fcmla<double>(vform, dst, src1, src2, acc, -1, rot);
2703 LogicVRegister Simulator::fcmla(VectorFormat vform,
2709   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
2711   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
2712     fcmla<float>(vform, dst, src1, src2, dst, index, rot);
2714     fcmla<double>(vform, dst, src1, src2, dst, index, rot);
2719 LogicVRegister Simulator::cadd(VectorFormat vform,
2729   uzp1(vform, src1_r, src1, zero);
2730   uzp2(vform, src1_i, src1, zero);
2731   uzp1(vform, src2_r, src2, zero);
2732   uzp2(vform, src2_i, src2, zero);
2736       sub(vform, src1_r, src1_r, src2_i).SignedSaturate(vform);
2737       add(vform, src1_i, src1_i, src2_r).SignedSaturate(vform);
2739       sub(vform, src1_r, src1_r, src2_i);
2740       add(vform, src1_i, src1_i, src2_r);
2745       add(vform, src1_r, src1_r, src2_i).SignedSaturate(vform);
2746       sub(vform, src1_i, src1_i, src2_r).SignedSaturate(vform);
2748       add(vform, src1_r, src1_r, src2_i);
2749       sub(vform, src1_i, src1_i, src2_r);
2753   zip1(vform, dst, src1_r, src1_i);
2757 LogicVRegister Simulator::cmla(VectorFormat vform,
2770     uzp1(vform, src1_a, src1, zero);
2771     uzp1(vform, src2_a, src2, zero);
2772     uzp2(vform, src2_b, src2, zero);
2774     uzp2(vform, src1_a, src1, zero);
2775     uzp2(vform, src2_a, src2, zero);
2776     uzp1(vform, src2_b, src2, zero);
2779   uzp1(vform, srca_r, srca, zero);
2780   uzp2(vform, srca_i, srca, zero);
2785   mul(vform, temp, src1_a, src2_a);
2787     sub(vform, srca_r, srca_r, temp);
2789     add(vform, srca_r, srca_r, temp);
2792   mul(vform, temp, src1_a, src2_b);
2794     sub(vform, srca_i, srca_i, temp);
2796     add(vform, srca_i, srca_i, temp);
2799   zip1(vform, dst, srca_r, srca_i);
2803 LogicVRegister Simulator::cmla(VectorFormat vform,
2811   dup_elements_to_segments(VectorFormatDoubleWidth(vform), temp, src2, index);
2812   return cmla(vform, dst, srca, src1, temp, rot);
2815 LogicVRegister Simulator::bgrp(VectorFormat vform,
2820   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2821     uint64_t value = src1.Uint(vform, i);
2822     uint64_t mask = src2.Uint(vform, i);
2827     for (unsigned j = 0; j < LaneSizeInBitsFromFormat(vform); j++) {
2843     dst.SetUint(vform, i, result_low);
2848 LogicVRegister Simulator::bdep(VectorFormat vform,
2852   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2853     uint64_t value = src1.Uint(vform, i);
2854     uint64_t mask = src2.Uint(vform, i);
2856     for (unsigned j = 0; j < LaneSizeInBitsFromFormat(vform); j++) {
2863     dst.SetUint(vform, i, result);
2868 LogicVRegister Simulator::histogram(VectorFormat vform,
2874   int elements_per_segment = kQRegSize / LaneSizeInBitsFromFormat(vform);
2877   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2879     uint64_t value = src1.Uint(vform, i);
2885       if (pg.IsActive(vform, j) &&
2886           (value == src2.Uint(vform, j + segment_offset))) {
2892   dst.SetUintArray(vform, result);
2896 LogicVRegister Simulator::dup_element(VectorFormat vform,
2900   if ((vform == kFormatVnQ) || (vform == kFormatVnO)) {
2904     int count = (vform == kFormatVnQ) ? 2 : 4;
2909     for (int i = 0; i < LaneCountFromFormat(vform) * count; i++) {
2913     int lane_count = LaneCountFromFormat(vform);
2914     uint64_t value = src.Uint(vform, src_index);
2915     dst.ClearForWrite(vform);
2917       dst.SetUint(vform, i, value);
2923 LogicVRegister Simulator::dup_elements_to_segments(VectorFormat vform,
2930   int segment_size = std::min(kQRegSize, RegisterSizeInBitsFromFormat(vform));
2931   VIXL_ASSERT(IsMultiple(segment_size, LaneSizeInBitsFromFormat(vform)));
2932   int lanes_per_segment = segment_size / LaneSizeInBitsFromFormat(vform);
2937   dst.ClearForWrite(vform);
2938   for (int j = 0; j < LaneCountFromFormat(vform); j += lanes_per_segment) {
2939     uint64_t value = src.Uint(vform, j + src_index);
2941       dst.SetUint(vform, j + i, value);
2948     VectorFormat vform,
2951   return dup_elements_to_segments(vform,
2957 LogicVRegister Simulator::dup_immediate(VectorFormat vform,
2960   int lane_count = LaneCountFromFormat(vform);
2961   uint64_t value = imm & MaxUintFromFormat(vform);
2962   dst.ClearForWrite(vform);
2964     dst.SetUint(vform, i, value);
2970 LogicVRegister Simulator::ins_element(VectorFormat vform,
2975   dst.SetUint(vform, dst_index, src.Uint(vform, src_index));
2980 LogicVRegister Simulator::ins_immediate(VectorFormat vform,
2984   uint64_t value = imm & MaxUintFromFormat(vform);
2985   dst.SetUint(vform, dst_index, value);
2990 LogicVRegister Simulator::index(VectorFormat vform,
2994   VIXL_ASSERT(IsSVEFormat(vform));
2996   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
2997     dst.SetUint(vform, i, value);
3004 LogicVRegister Simulator::insr(VectorFormat vform,
3007   VIXL_ASSERT(IsSVEFormat(vform));
3008   for (int i = LaneCountFromFormat(vform) - 1; i > 0; i--) {
3009     dst.SetUint(vform, i, dst.Uint(vform, i - 1));
3011   dst.SetUint(vform, 0, imm);
3016 LogicVRegister Simulator::mov(VectorFormat vform,
3019   dst.ClearForWrite(vform);
3020   for (int lane = 0; lane < LaneCountFromFormat(vform); lane++) {
3021     dst.SetUint(vform, lane, src.Uint(vform, lane));
3038 LogicVRegister Simulator::mov_merging(VectorFormat vform,
3042   return sel(vform, dst, pg, src, dst);
3045 LogicVRegister Simulator::mov_zeroing(VectorFormat vform,
3050   dup_immediate(vform, zero, 0);
3051   return sel(vform, dst, pg, src, zero);
3054 LogicVRegister Simulator::mov_alternating(VectorFormat vform,
3059   for (int i = start_at; i < LaneCountFromFormat(vform); i += 2) {
3060     dst.SetUint(vform, i, src.Uint(vform, i));
3078 LogicVRegister Simulator::movi(VectorFormat vform,
3081   int lane_count = LaneCountFromFormat(vform);
3082   dst.ClearForWrite(vform);
3084     dst.SetUint(vform, i, imm);
3090 LogicVRegister Simulator::mvni(VectorFormat vform,
3093   int lane_count = LaneCountFromFormat(vform);
3094   dst.ClearForWrite(vform);
3096     dst.SetUint(vform, i, ~imm);
3102 LogicVRegister Simulator::orr(VectorFormat vform,
3107   int lane_count = LaneCountFromFormat(vform);
3109     result[i] = src.Uint(vform, i) | imm;
3111   dst.ClearForWrite(vform);
3113     dst.SetUint(vform, i, result[i]);
3119 LogicVRegister Simulator::uxtl(VectorFormat vform,
3123   VectorFormat vform_half = VectorFormatHalfWidth(vform);
3124   int lane_count = LaneCountFromFormat(vform);
3127   dst.ClearForWrite(vform);
3129     dst.SetUint(vform, i, src.Uint(vform_half, src_offset + i));
3135 LogicVRegister Simulator::sxtl(VectorFormat vform,
3139   VectorFormat vform_half = VectorFormatHalfWidth(vform);
3140   int lane_count = LaneCountFromFormat(vform);
3143   dst.ClearForWrite(vform);
3144   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
3145     dst.SetInt(vform, i, src.Int(vform_half, src_offset + i));
3151 LogicVRegister Simulator::uxtl2(VectorFormat vform,
3154   return uxtl(vform, dst, src, /* is_2 = */ true);
3158 LogicVRegister Simulator::sxtl2(VectorFormat vform,
3161   return sxtl(vform, dst, src, /* is_2 = */ true);
3165 LogicVRegister Simulator::uxt(VectorFormat vform,
3169   int lane_count = LaneCountFromFormat(vform);
3172   dst.ClearForWrite(vform);
3174     dst.SetInt(vform, i, src.Uint(vform, i) & mask);
3180 LogicVRegister Simulator::sxt(VectorFormat vform,
3184   int lane_count = LaneCountFromFormat(vform);
3186   dst.ClearForWrite(vform);
3189         ExtractSignedBitfield64(from_size_in_bits - 1, 0, src.Uint(vform, i));
3190     dst.SetInt(vform, i, value);
3196 LogicVRegister Simulator::shrn(VectorFormat vform,
3201   VectorFormat vform_src = VectorFormatDoubleWidth(vform);
3202   VectorFormat vform_dst = vform;
3208 LogicVRegister Simulator::shrn2(VectorFormat vform,
3213   VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
3214   VectorFormat vformdst = vform;
3220 LogicVRegister Simulator::rshrn(VectorFormat vform,
3225   VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
3226   VectorFormat vformdst = vform;
3232 LogicVRegister Simulator::rshrn2(VectorFormat vform,
3237   VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
3238   VectorFormat vformdst = vform;
3243 LogicVRegister Simulator::Table(VectorFormat vform,
3252   int lane_count = LaneCountFromFormat(vform);
3258   // 8B or 16B vform for the destination, so infer the table format from the
3260   VectorFormat vform_tab = (vform == kFormat8B) ? kFormat16B : vform;
3268     uint64_t index = ind.Uint(vform, i);
3269     result[i] = zero_out_of_bounds ? 0 : dst.Uint(vform, i);
3272   dst.SetUintArray(vform, result);
3276 LogicVRegister Simulator::tbl(VectorFormat vform,
3280   return Table(vform, dst, ind, true, &tab);
3284 LogicVRegister Simulator::tbl(VectorFormat vform,
3289   return Table(vform, dst, ind, true, &tab, &tab2);
3293 LogicVRegister Simulator::tbl(VectorFormat vform,
3299   return Table(vform, dst, ind, true, &tab, &tab2, &tab3);
3303 LogicVRegister Simulator::tbl(VectorFormat vform,
3310   return Table(vform, dst, ind, true, &tab, &tab2, &tab3, &tab4);
3314 LogicVRegister Simulator::tbx(VectorFormat vform,
3318   return Table(vform, dst, ind, false, &tab);
3322 LogicVRegister Simulator::tbx(VectorFormat vform,
3327   return Table(vform, dst, ind, false, &tab, &tab2);
3331 LogicVRegister Simulator::tbx(VectorFormat vform,
3337   return Table(vform, dst, ind, false, &tab, &tab2, &tab3);
3341 LogicVRegister Simulator::tbx(VectorFormat vform,
3348   return Table(vform, dst, ind, false, &tab, &tab2, &tab3, &tab4);
3352 LogicVRegister Simulator::uqshrn(VectorFormat vform,
3356   return shrn(vform, dst, src, shift).UnsignedSaturate(vform);
3360 LogicVRegister Simulator::uqshrn2(VectorFormat vform,
3364   return shrn2(vform, dst, src, shift).UnsignedSaturate(vform);
3368 LogicVRegister Simulator::uqrshrn(VectorFormat vform,
3372   return rshrn(vform, dst, src, shift).UnsignedSaturate(vform);
3376 LogicVRegister Simulator::uqrshrn2(VectorFormat vform,
3380   return rshrn2(vform, dst, src, shift).UnsignedSaturate(vform);
3384 LogicVRegister Simulator::sqshrn(VectorFormat vform,
3389   VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
3390   VectorFormat vformdst = vform;
3396 LogicVRegister Simulator::sqshrn2(VectorFormat vform,
3401   VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
3402   VectorFormat vformdst = vform;
3408 LogicVRegister Simulator::sqrshrn(VectorFormat vform,
3413   VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
3414   VectorFormat vformdst = vform;
3420 LogicVRegister Simulator::sqrshrn2(VectorFormat vform,
3425   VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
3426   VectorFormat vformdst = vform;
3432 LogicVRegister Simulator::sqshrun(VectorFormat vform,
3437   VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
3438   VectorFormat vformdst = vform;
3444 LogicVRegister Simulator::sqshrun2(VectorFormat vform,
3449   VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
3450   VectorFormat vformdst = vform;
3456 LogicVRegister Simulator::sqrshrun(VectorFormat vform,
3461   VectorFormat vformsrc = VectorFormatDoubleWidth(vform);
3462   VectorFormat vformdst = vform;
3468 LogicVRegister Simulator::sqrshrun2(VectorFormat vform,
3473   VectorFormat vformsrc = VectorFormatDoubleWidth(VectorFormatHalfLanes(vform));
3474   VectorFormat vformdst = vform;
3480 LogicVRegister Simulator::uaddl(VectorFormat vform,
3485   uxtl(vform, temp1, src1);
3486   uxtl(vform, temp2, src2);
3487   add(vform, dst, temp1, temp2);
3492 LogicVRegister Simulator::uaddl2(VectorFormat vform,
3497   uxtl2(vform, temp1, src1);
3498   uxtl2(vform, temp2, src2);
3499   add(vform, dst, temp1, temp2);
3504 LogicVRegister Simulator::uaddw(VectorFormat vform,
3509   uxtl(vform, temp, src2);
3510   add(vform, dst, src1, temp);
3515 LogicVRegister Simulator::uaddw2(VectorFormat vform,
3520   uxtl2(vform, temp, src2);
3521   add(vform, dst, src1, temp);
3526 LogicVRegister Simulator::saddl(VectorFormat vform,
3531   sxtl(vform, temp1, src1);
3532   sxtl(vform, temp2, src2);
3533   add(vform, dst, temp1, temp2);
3538 LogicVRegister Simulator::saddl2(VectorFormat vform,
3543   sxtl2(vform, temp1, src1);
3544   sxtl2(vform, temp2, src2);
3545   add(vform, dst, temp1, temp2);
3550 LogicVRegister Simulator::saddw(VectorFormat vform,
3555   sxtl(vform, temp, src2);
3556   add(vform, dst, src1, temp);
3561 LogicVRegister Simulator::saddw2(VectorFormat vform,
3566   sxtl2(vform, temp, src2);
3567   add(vform, dst, src1, temp);
3572 LogicVRegister Simulator::usubl(VectorFormat vform,
3577   uxtl(vform, temp1, src1);
3578   uxtl(vform, temp2, src2);
3579   sub(vform, dst, temp1, temp2);
3584 LogicVRegister Simulator::usubl2(VectorFormat vform,
3589   uxtl2(vform, temp1, src1);
3590   uxtl2(vform, temp2, src2);
3591   sub(vform, dst, temp1, temp2);
3596 LogicVRegister Simulator::usubw(VectorFormat vform,
3601   uxtl(vform, temp, src2);
3602   sub(vform, dst, src1, temp);
3607 LogicVRegister Simulator::usubw2(VectorFormat vform,
3612   uxtl2(vform, temp, src2);
3613   sub(vform, dst, src1, temp);
3618 LogicVRegister Simulator::ssubl(VectorFormat vform,
3623   sxtl(vform, temp1, src1);
3624   sxtl(vform, temp2, src2);
3625   sub(vform, dst, temp1, temp2);
3630 LogicVRegister Simulator::ssubl2(VectorFormat vform,
3635   sxtl2(vform, temp1, src1);
3636   sxtl2(vform, temp2, src2);
3637   sub(vform, dst, temp1, temp2);
3642 LogicVRegister Simulator::ssubw(VectorFormat vform,
3647   sxtl(vform, temp, src2);
3648   sub(vform, dst, src1, temp);
3653 LogicVRegister Simulator::ssubw2(VectorFormat vform,
3658   sxtl2(vform, temp, src2);
3659   sub(vform, dst, src1, temp);
3664 LogicVRegister Simulator::uabal(VectorFormat vform,
3669   uxtl(vform, temp1, src1);
3670   uxtl(vform, temp2, src2);
3671   uaba(vform, dst, temp1, temp2);
3676 LogicVRegister Simulator::uabal2(VectorFormat vform,
3681   uxtl2(vform, temp1, src1);
3682   uxtl2(vform, temp2, src2);
3683   uaba(vform, dst, temp1, temp2);
3688 LogicVRegister Simulator::sabal(VectorFormat vform,
3693   sxtl(vform, temp1, src1);
3694   sxtl(vform, temp2, src2);
3695   saba(vform, dst, temp1, temp2);
3700 LogicVRegister Simulator::sabal2(VectorFormat vform,
3705   sxtl2(vform, temp1, src1);
3706   sxtl2(vform, temp2, src2);
3707   saba(vform, dst, temp1, temp2);
3712 LogicVRegister Simulator::uabdl(VectorFormat vform,
3717   uxtl(vform, temp1, src1);
3718   uxtl(vform, temp2, src2);
3719   absdiff(vform, dst, temp1, temp2, false);
3724 LogicVRegister Simulator::uabdl2(VectorFormat vform,
3729   uxtl2(vform, temp1, src1);
3730   uxtl2(vform, temp2, src2);
3731   absdiff(vform, dst, temp1, temp2, false);
3736 LogicVRegister Simulator::sabdl(VectorFormat vform,
3741   sxtl(vform, temp1, src1);
3742   sxtl(vform, temp2, src2);
3743   absdiff(vform, dst, temp1, temp2, true);
3748 LogicVRegister Simulator::sabdl2(VectorFormat vform,
3753   sxtl2(vform, temp1, src1);
3754   sxtl2(vform, temp2, src2);
3755   absdiff(vform, dst, temp1, temp2, true);
3760 LogicVRegister Simulator::umull(VectorFormat vform,
3766   uxtl(vform, temp1, src1, is_2);
3767   uxtl(vform, temp2, src2, is_2);
3768   mul(vform, dst, temp1, temp2);
3773 LogicVRegister Simulator::umull2(VectorFormat vform,
3777   return umull(vform, dst, src1, src2, /* is_2 = */ true);
3781 LogicVRegister Simulator::smull(VectorFormat vform,
3787   sxtl(vform, temp1, src1, is_2);
3788   sxtl(vform, temp2, src2, is_2);
3789   mul(vform, dst, temp1, temp2);
3794 LogicVRegister Simulator::smull2(VectorFormat vform,
3798   return smull(vform, dst, src1, src2, /* is_2 = */ true);
3802 LogicVRegister Simulator::umlsl(VectorFormat vform,
3808   uxtl(vform, temp1, src1, is_2);
3809   uxtl(vform, temp2, src2, is_2);
3810   mls(vform, dst, dst, temp1, temp2);
3815 LogicVRegister Simulator::umlsl2(VectorFormat vform,
3819   return umlsl(vform, dst, src1, src2, /* is_2 = */ true);
3823 LogicVRegister Simulator::smlsl(VectorFormat vform,
3829   sxtl(vform, temp1, src1, is_2);
3830   sxtl(vform, temp2, src2, is_2);
3831   mls(vform, dst, dst, temp1, temp2);
3836 LogicVRegister Simulator::smlsl2(VectorFormat vform,
3840   return smlsl(vform, dst, src1, src2, /* is_2 = */ true);
3844 LogicVRegister Simulator::umlal(VectorFormat vform,
3850   uxtl(vform, temp1, src1, is_2);
3851   uxtl(vform, temp2, src2, is_2);
3852   mla(vform, dst, dst, temp1, temp2);
3857 LogicVRegister Simulator::umlal2(VectorFormat vform,
3861   return umlal(vform, dst, src1, src2, /* is_2 = */ true);
3865 LogicVRegister Simulator::smlal(VectorFormat vform,
3871   sxtl(vform, temp1, src1, is_2);
3872   sxtl(vform, temp2, src2, is_2);
3873   mla(vform, dst, dst, temp1, temp2);
3878 LogicVRegister Simulator::smlal2(VectorFormat vform,
3882   return smlal(vform, dst, src1, src2, /* is_2 = */ true);
3886 LogicVRegister Simulator::sqdmlal(VectorFormat vform,
3892   LogicVRegister product = sqdmull(vform, temp, src1, src2, is_2);
3893   return add(vform, dst, dst, product).SignedSaturate(vform);
3897 LogicVRegister Simulator::sqdmlal2(VectorFormat vform,
3901   return sqdmlal(vform, dst, src1, src2, /* is_2 = */ true);
3905 LogicVRegister Simulator::sqdmlsl(VectorFormat vform,
3911   LogicVRegister product = sqdmull(vform, temp, src1, src2, is_2);
3912   return sub(vform, dst, dst, product).SignedSaturate(vform);
3916 LogicVRegister Simulator::sqdmlsl2(VectorFormat vform,
3920   return sqdmlsl(vform, dst, src1, src2, /* is_2 = */ true);
3924 LogicVRegister Simulator::sqdmull(VectorFormat vform,
3930   LogicVRegister product = smull(vform, temp, src1, src2, is_2);
3931   return add(vform, dst, product, product).SignedSaturate(vform);
3935 LogicVRegister Simulator::sqdmull2(VectorFormat vform,
3939   return sqdmull(vform, dst, src1, src2, /* is_2 = */ true);
3942 LogicVRegister Simulator::sqrdmulh(VectorFormat vform,
3947   int esize = LaneSizeInBitsFromFormat(vform);
3952   mul(vform, temp_lo, src1, src2);
3953   smulh(vform, temp_hi, src1, src2);
3957   shl(vform, temp_hi, temp_hi, 1);
3958   usra(vform, temp_hi, temp_lo, esize - 1);
3963     shl(vform, temp_lo, temp_lo, 1);
3964     usra(vform, temp_hi, temp_lo, esize - 1);
3969   dst.ClearForWrite(vform);
3970   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
3973     ptemp.SetActive(vform, i, true);
3974     if ((src1.Int(vform, i) == MinIntFromFormat(vform)) &&
3975         (src2.Int(vform, i) == MinIntFromFormat(vform))) {
3976       ptemp.SetActive(vform, i, false);
3978     dst.SetInt(vform, i, MaxIntFromFormat(vform));
3981   mov_merging(vform, dst, not_sat, temp_hi);
3986 LogicVRegister Simulator::dot(VectorFormat vform,
3993       VectorFormatHalfWidthDoubleLanes(VectorFormatHalfWidthDoubleLanes(vform));
3995   dst.ClearForWrite(vform);
3996   for (int e = 0; e < LaneCountFromFormat(vform); e++) {
4013     dst.SetUint(vform, e, result + dst.Uint(vform, e));
4019 LogicVRegister Simulator::sdot(VectorFormat vform,
4023   return dot(vform, dst, src1, src2, true, true);
4027 LogicVRegister Simulator::udot(VectorFormat vform,
4031   return dot(vform, dst, src1, src2, false, false);
4034 LogicVRegister Simulator::usdot(VectorFormat vform,
4038   return dot(vform, dst, src1, src2, false, true);
4041 LogicVRegister Simulator::cdot(VectorFormat vform,
4049       VectorFormatHalfWidthDoubleLanes(VectorFormatHalfWidthDoubleLanes(vform));
4055   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
4056     int64_t result = acc.Int(vform, i);
4064     dst.SetInt(vform, i, result);
4069 LogicVRegister Simulator::sqrdcmlah(VectorFormat vform,
4082     uzp1(vform, src1_a, src1, zero);
4083     uzp1(vform, src2_a, src2, zero);
4084     uzp2(vform, src2_b, src2, zero);
4086     uzp2(vform, src1_a, src1, zero);
4087     uzp2(vform, src2_a, src2, zero);
4088     uzp1(vform, src2_b, src2, zero);
4091   uzp1(vform, srca_r, srca, zero);
4092   uzp2(vform, srca_i, srca, zero);
4098   sqrdmlash(vform, srca_r, src1_a, src2_a, round, sub_r);
4099   sqrdmlash(vform, srca_i, src1_a, src2_b, round, sub_i);
4100   zip1(vform, dst, srca_r, srca_i);
4104 LogicVRegister Simulator::sqrdcmlah(VectorFormat vform,
4112   dup_elements_to_segments(VectorFormatDoubleWidth(vform), temp, src2, index);
4113   return sqrdcmlah(vform, dst, srca, src1, temp, rot);
4116 LogicVRegister Simulator::sqrdmlash_d(VectorFormat vform,
4128   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
4138   dst.ClearForWrite(vform);
4139   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
4141     accum.first = dst.Int(vform, i) >> 1;
4142     accum.second = dst.Int(vform, i) << (esize - 1);
4144     vixl_uint128_t product = Mul64(src1.Int(vform, i), src2.Int(vform, i));
4161         (accum.second > static_cast<uint64_t>(MaxIntFromFormat(vform)))) {
4162       accum.second = MaxIntFromFormat(vform);
4164                            static_cast<uint64_t>(MinIntFromFormat(vform)))) {
4165       accum.second = MinIntFromFormat(vform);
4168     dst.SetInt(vform, i, accum.second);
4174 LogicVRegister Simulator::sqrdmlash(VectorFormat vform,
4186   if (vform == kFormatVnD) {
4187     return sqrdmlash_d(vform, dst, src1, src2, round, sub_op);
4190   int esize = LaneSizeInBitsFromFormat(vform);
4194   dst.ClearForWrite(vform);
4195   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
4196     accum = dst.Int(vform, i) << (esize - 1);
4198       accum -= src1.Int(vform, i) * src2.Int(vform, i);
4200       accum += src1.Int(vform, i) * src2.Int(vform, i);
4205     if (accum > MaxIntFromFormat(vform)) {
4206       accum = MaxIntFromFormat(vform);
4207     } else if (accum < MinIntFromFormat(vform)) {
4208       accum = MinIntFromFormat(vform);
4210     dst.SetInt(vform, i, accum);
4216 LogicVRegister Simulator::sqrdmlah(VectorFormat vform,
4221   return sqrdmlash(vform, dst, src1, src2, round, false);
4225 LogicVRegister Simulator::sqrdmlsh(VectorFormat vform,
4230   return sqrdmlash(vform, dst, src1, src2, round, true);
4234 LogicVRegister Simulator::sqdmulh(VectorFormat vform,
4238   return sqrdmulh(vform, dst, src1, src2, false);
4242 LogicVRegister Simulator::addhn(VectorFormat vform,
4247   add(VectorFormatDoubleWidth(vform), temp, src1, src2);
4248   shrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
4253 LogicVRegister Simulator::addhn2(VectorFormat vform,
4258   add(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
4259   shrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
4264 LogicVRegister Simulator::raddhn(VectorFormat vform,
4269   add(VectorFormatDoubleWidth(vform), temp, src1, src2);
4270   rshrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
4275 LogicVRegister Simulator::raddhn2(VectorFormat vform,
4280   add(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
4281   rshrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
4286 LogicVRegister Simulator::subhn(VectorFormat vform,
4291   sub(VectorFormatDoubleWidth(vform), temp, src1, src2);
4292   shrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
4297 LogicVRegister Simulator::subhn2(VectorFormat vform,
4302   sub(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
4303   shrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
4308 LogicVRegister Simulator::rsubhn(VectorFormat vform,
4313   sub(VectorFormatDoubleWidth(vform), temp, src1, src2);
4314   rshrn(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
4319 LogicVRegister Simulator::rsubhn2(VectorFormat vform,
4324   sub(VectorFormatDoubleWidth(VectorFormatHalfLanes(vform)), temp, src1, src2);
4325   rshrn2(vform, dst, temp, LaneSizeInBitsFromFormat(vform));
4330 LogicVRegister Simulator::trn1(VectorFormat vform,
4335   int lane_count = LaneCountFromFormat(vform);
4338     result[2 * i] = src1.Uint(vform, 2 * i);
4339     result[(2 * i) + 1] = src2.Uint(vform, 2 * i);
4342   dst.ClearForWrite(vform);
4344     dst.SetUint(vform, i, result[i]);
4350 LogicVRegister Simulator::trn2(VectorFormat vform,
4355   int lane_count = LaneCountFromFormat(vform);
4358     result[2 * i] = src1.Uint(vform, (2 * i) + 1);
4359     result[(2 * i) + 1] = src2.Uint(vform, (2 * i) + 1);
4362   dst.ClearForWrite(vform);
4364     dst.SetUint(vform, i, result[i]);
4370 LogicVRegister Simulator::zip1(VectorFormat vform,
4375   int lane_count = LaneCountFromFormat(vform);
4378     result[2 * i] = src1.Uint(vform, i);
4379     result[(2 * i) + 1] = src2.Uint(vform, i);
4382   dst.ClearForWrite(vform);
4384     dst.SetUint(vform, i, result[i]);
4390 LogicVRegister Simulator::zip2(VectorFormat vform,
4395   int lane_count = LaneCountFromFormat(vform);
4398     result[2 * i] = src1.Uint(vform, pairs + i);
4399     result[(2 * i) + 1] = src2.Uint(vform, pairs + i);
4402   dst.ClearForWrite(vform);
4404     dst.SetUint(vform, i, result[i]);
4410 LogicVRegister Simulator::uzp1(VectorFormat vform,
4415   int lane_count = LaneCountFromFormat(vform);
4417     result[i] = src1.Uint(vform, i);
4418     result[lane_count + i] = src2.Uint(vform, i);
4421   dst.ClearForWrite(vform);
4423     dst.SetUint(vform, i, result[2 * i]);
4429 LogicVRegister Simulator::uzp2(VectorFormat vform,
4434   int lane_count = LaneCountFromFormat(vform);
4436     result[i] = src1.Uint(vform, i);
4437     result[lane_count + i] = src2.Uint(vform, i);
4440   dst.ClearForWrite(vform);
4442     dst.SetUint(vform, i, result[(2 * i) + 1]);
4447 LogicVRegister Simulator::interleave_top_bottom(VectorFormat vform,
4460   int lane_count = LaneCountFromFormat(vform);
4462     result[i] = src.Uint(vform, i / 2);
4463     result[i + 1] = src.Uint(vform, (lane_count / 2) + (i / 2));
4465   dst.SetUintArray(vform, result);
4996   LogicVRegister Simulator::FN(VectorFormat vform,               \
5000     dst.ClearForWrite(vform);                                    \
5001     for (int i = 0; i < LaneCountFromFormat(vform); i++) {       \
5013       dst.SetFloat(vform, i, result);                            \
5018   LogicVRegister Simulator::FN(VectorFormat vform,               \
5022     if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {          \
5023       FN<SimFloat16>(vform, dst, src1, src2);                    \
5024     } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {   \
5025       FN<float>(vform, dst, src1, src2);                         \
5027       VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize); \
5028       FN<double>(vform, dst, src1, src2);                        \
5036 LogicVRegister Simulator::fnmul(VectorFormat vform,
5041   LogicVRegister product = fmul(vform, temp, src1, src2);
5042   return fneg(vform, dst, product);
5047 LogicVRegister Simulator::frecps(VectorFormat vform,
5051   dst.ClearForWrite(vform);
5052   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5056     dst.SetFloat(vform, i, IsNaN(result) ? result : FPRecipStepFused(op1, op2));
5062 LogicVRegister Simulator::frecps(VectorFormat vform,
5066   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5067     frecps<SimFloat16>(vform, dst, src1, src2);
5068   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5069     frecps<float>(vform, dst, src1, src2);
5071     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5072     frecps<double>(vform, dst, src1, src2);
5079 LogicVRegister Simulator::frsqrts(VectorFormat vform,
5083   dst.ClearForWrite(vform);
5084   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5088     dst.SetFloat(vform, i, IsNaN(result) ? result : FPRSqrtStepFused(op1, op2));
5094 LogicVRegister Simulator::frsqrts(VectorFormat vform,
5098   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5099     frsqrts<SimFloat16>(vform, dst, src1, src2);
5100   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5101     frsqrts<float>(vform, dst, src1, src2);
5103     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5104     frsqrts<double>(vform, dst, src1, src2);
5111 LogicVRegister Simulator::fcmp(VectorFormat vform,
5116   dst.ClearForWrite(vform);
5117   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5156     dst.SetUint(vform, i, result ? MaxUintFromFormat(vform) : 0);
5162 LogicVRegister Simulator::fcmp(VectorFormat vform,
5167   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5168     fcmp<SimFloat16>(vform, dst, src1, src2, cond);
5169   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5170     fcmp<float>(vform, dst, src1, src2, cond);
5172     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5173     fcmp<double>(vform, dst, src1, src2, cond);
5179 LogicVRegister Simulator::fcmp_zero(VectorFormat vform,
5184   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5186         dup_immediate(vform, temp, Float16ToRawbits(SimFloat16(0.0)));
5187     fcmp<SimFloat16>(vform, dst, src, zero_reg, cond);
5188   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5189     LogicVRegister zero_reg = dup_immediate(vform, temp, FloatToRawbits(0.0));
5190     fcmp<float>(vform, dst, src, zero_reg, cond);
5192     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5193     LogicVRegister zero_reg = dup_immediate(vform, temp, DoubleToRawbits(0.0));
5194     fcmp<double>(vform, dst, src, zero_reg, cond);
5200 LogicVRegister Simulator::fabscmp(VectorFormat vform,
5206   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5207     LogicVRegister abs_src1 = fabs_<SimFloat16>(vform, temp1, src1);
5208     LogicVRegister abs_src2 = fabs_<SimFloat16>(vform, temp2, src2);
5209     fcmp<SimFloat16>(vform, dst, abs_src1, abs_src2, cond);
5210   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5211     LogicVRegister abs_src1 = fabs_<float>(vform, temp1, src1);
5212     LogicVRegister abs_src2 = fabs_<float>(vform, temp2, src2);
5213     fcmp<float>(vform, dst, abs_src1, abs_src2, cond);
5215     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5216     LogicVRegister abs_src1 = fabs_<double>(vform, temp1, src1);
5217     LogicVRegister abs_src2 = fabs_<double>(vform, temp2, src2);
5218     fcmp<double>(vform, dst, abs_src1, abs_src2, cond);
5225 LogicVRegister Simulator::fmla(VectorFormat vform,
5230   dst.ClearForWrite(vform);
5231   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5236     dst.SetFloat(vform, i, result);
5242 LogicVRegister Simulator::fmla(VectorFormat vform,
5247   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5248     fmla<SimFloat16>(vform, dst, srca, src1, src2);
5249   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5250     fmla<float>(vform, dst, srca, src1, src2);
5252     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5253     fmla<double>(vform, dst, srca, src1, src2);
5260 LogicVRegister Simulator::fmls(VectorFormat vform,
5265   dst.ClearForWrite(vform);
5266   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5277 LogicVRegister Simulator::fmls(VectorFormat vform,
5282   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5283     fmls<SimFloat16>(vform, dst, srca, src1, src2);
5284   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5285     fmls<float>(vform, dst, srca, src1, src2);
5287     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5288     fmls<double>(vform, dst, srca, src1, src2);
5294 LogicVRegister Simulator::fmlal(VectorFormat vform,
5298   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
5299   dst.ClearForWrite(vform);
5300   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5311 LogicVRegister Simulator::fmlal2(VectorFormat vform,
5315   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
5316   dst.ClearForWrite(vform);
5317   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5318     int src = i + LaneCountFromFormat(vform);
5329 LogicVRegister Simulator::fmlsl(VectorFormat vform,
5333   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
5334   dst.ClearForWrite(vform);
5335   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5346 LogicVRegister Simulator::fmlsl2(VectorFormat vform,
5350   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
5351   dst.ClearForWrite(vform);
5352   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5353     int src = i + LaneCountFromFormat(vform);
5364 LogicVRegister Simulator::fmlal(VectorFormat vform,
5369   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
5370   dst.ClearForWrite(vform);
5372   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5382 LogicVRegister Simulator::fmlal2(VectorFormat vform,
5387   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
5388   dst.ClearForWrite(vform);
5390   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5391     int src = i + LaneCountFromFormat(vform);
5401 LogicVRegister Simulator::fmlsl(VectorFormat vform,
5406   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
5407   dst.ClearForWrite(vform);
5409   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5419 LogicVRegister Simulator::fmlsl2(VectorFormat vform,
5424   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
5425   dst.ClearForWrite(vform);
5427   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5428     int src = i + LaneCountFromFormat(vform);
5439 LogicVRegister Simulator::fneg(VectorFormat vform,
5442   dst.ClearForWrite(vform);
5443   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5452 LogicVRegister Simulator::fneg(VectorFormat vform,
5455   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5456     fneg<SimFloat16>(vform, dst, src);
5457   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5458     fneg<float>(vform, dst, src);
5460     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5461     fneg<double>(vform, dst, src);
5468 LogicVRegister Simulator::fabs_(VectorFormat vform,
5471   dst.ClearForWrite(vform);
5472   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5483 LogicVRegister Simulator::fabs_(VectorFormat vform,
5486   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5487     fabs_<SimFloat16>(vform, dst, src);
5488   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5489     fabs_<float>(vform, dst, src);
5491     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5492     fabs_<double>(vform, dst, src);
5498 LogicVRegister Simulator::fabd(VectorFormat vform,
5503   fsub(vform, temp, src1, src2);
5504   fabs_(vform, dst, temp);
5509 LogicVRegister Simulator::fsqrt(VectorFormat vform,
5512   dst.ClearForWrite(vform);
5513   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5514     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5518   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5519     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5524     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5525     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5535   LogicVRegister Simulator::FNP(VectorFormat vform,                            \
5540     uzp1(vform, temp1, src1, src2);                                            \
5541     uzp2(vform, temp2, src1, src2);                                            \
5542     FN(vform, dst, temp1, temp2);                                              \
5543     if (IsSVEFormat(vform)) {                                                  \
5544       interleave_top_bottom(vform, dst, dst);                                  \
5549   LogicVRegister Simulator::FNP(VectorFormat vform,                            \
5552     if (vform == kFormatH) {                                                   \
5553       SimFloat16 result(OP(SimFloat16(RawbitsToFloat16(src.Uint(vform, 0))),   \
5554                            SimFloat16(RawbitsToFloat16(src.Uint(vform, 1))))); \
5555       dst.SetUint(vform, 0, Float16ToRawbits(result));                         \
5556     } else if (vform == kFormatS) {                                            \
5560       VIXL_ASSERT(vform == kFormatD);                                          \
5564     dst.ClearForWrite(vform);                                                  \
5571 LogicVRegister Simulator::FPPairedAcrossHelper(VectorFormat vform,
5576   int lane_count = LaneCountFromFormat(vform);
5593   dst.ClearForWrite(ScalarFormatFromFormat(vform));
5599     VectorFormat vform,
5606   switch (LaneSizeInBitsFromFormat(vform)) {
5608       return FPPairedAcrossHelper<SimFloat16>(vform,
5614       return FPPairedAcrossHelper<float>(vform, dst, src, fn32, inactive_value);
5616       VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5617       return FPPairedAcrossHelper<double>(vform,
5625 LogicVRegister Simulator::faddv(VectorFormat vform,
5628   return FPPairedAcrossHelper(vform,
5637 LogicVRegister Simulator::fmaxv(VectorFormat vform,
5640   int lane_size = LaneSizeInBitsFromFormat(vform);
5643   return FPPairedAcrossHelper(vform,
5653 LogicVRegister Simulator::fminv(VectorFormat vform,
5656   int lane_size = LaneSizeInBitsFromFormat(vform);
5659   return FPPairedAcrossHelper(vform,
5669 LogicVRegister Simulator::fmaxnmv(VectorFormat vform,
5672   int lane_size = LaneSizeInBitsFromFormat(vform);
5674   return FPPairedAcrossHelper(vform,
5684 LogicVRegister Simulator::fminnmv(VectorFormat vform,
5687   int lane_size = LaneSizeInBitsFromFormat(vform);
5689   return FPPairedAcrossHelper(vform,
5699 LogicVRegister Simulator::fmul(VectorFormat vform,
5704   dst.ClearForWrite(vform);
5706   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5708     fmul<SimFloat16>(vform, dst, src1, index_reg);
5709   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5711     fmul<float>(vform, dst, src1, index_reg);
5713     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5715     fmul<double>(vform, dst, src1, index_reg);
5721 LogicVRegister Simulator::fmla(VectorFormat vform,
5726   dst.ClearForWrite(vform);
5728   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5730     fmla<SimFloat16>(vform, dst, dst, src1, index_reg);
5731   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5733     fmla<float>(vform, dst, dst, src1, index_reg);
5735     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5737     fmla<double>(vform, dst, dst, src1, index_reg);
5743 LogicVRegister Simulator::fmls(VectorFormat vform,
5748   dst.ClearForWrite(vform);
5750   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5752     fmls<SimFloat16>(vform, dst, dst, src1, index_reg);
5753   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5755     fmls<float>(vform, dst, dst, src1, index_reg);
5757     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5759     fmls<double>(vform, dst, dst, src1, index_reg);
5765 LogicVRegister Simulator::fmulx(VectorFormat vform,
5770   dst.ClearForWrite(vform);
5772   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5774     fmulx<SimFloat16>(vform, dst, src1, index_reg);
5775   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5777     fmulx<float>(vform, dst, src1, index_reg);
5779     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5781     fmulx<double>(vform, dst, src1, index_reg);
5787 LogicVRegister Simulator::frint(VectorFormat vform,
5793   dst.ClearForWrite(vform);
5794   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
5796     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5804   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5805     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5815     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5816     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5835   VectorFormat vform = SVEFormatFromLaneSizeInBits(
5838   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5839     if (!pg.IsActive(vform, i)) continue;
5843                                                       src.Uint(vform, i));
5850     dst.SetUint(vform, i, dst_raw_bits);
5856 LogicVRegister Simulator::fcvts(VectorFormat vform,
5864   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= dst_data_size_in_bits);
5865   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= src_data_size_in_bits);
5867   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5868     if (!pg.IsActive(vform, i)) continue;
5872                                                src.Uint(vform, i));
5878         dst.SetInt(vform, i, FPToInt16(result, round));
5881         dst.SetInt(vform, i, FPToInt32(result, round));
5884         dst.SetInt(vform, i, FPToInt64(result, round));
5895 LogicVRegister Simulator::fcvts(VectorFormat vform,
5900   dst.ClearForWrite(vform);
5901   return fcvts(vform,
5902                LaneSizeInBitsFromFormat(vform),
5903                LaneSizeInBitsFromFormat(vform),
5911 LogicVRegister Simulator::fcvtu(VectorFormat vform,
5919   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= dst_data_size_in_bits);
5920   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= src_data_size_in_bits);
5922   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
5923     if (!pg.IsActive(vform, i)) continue;
5927                                                src.Uint(vform, i));
5933         dst.SetUint(vform, i, FPToUInt16(result, round));
5936         dst.SetUint(vform, i, FPToUInt32(result, round));
5939         dst.SetUint(vform, i, FPToUInt64(result, round));
5950 LogicVRegister Simulator::fcvtu(VectorFormat vform,
5955   dst.ClearForWrite(vform);
5956   return fcvtu(vform,
5957                LaneSizeInBitsFromFormat(vform),
5958                LaneSizeInBitsFromFormat(vform),
5966 LogicVRegister Simulator::fcvtl(VectorFormat vform,
5969   if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5970     for (int i = LaneCountFromFormat(vform) - 1; i >= 0; i--) {
5977     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
5978     for (int i = LaneCountFromFormat(vform) - 1; i >= 0; i--) {
5986 LogicVRegister Simulator::fcvtl2(VectorFormat vform,
5989   int lane_count = LaneCountFromFormat(vform);
5990   if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
5999     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
6008 LogicVRegister Simulator::fcvtn(VectorFormat vform,
6013   dst.ClearForWrite(vform);
6014   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
6015     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6022     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
6023     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6031 LogicVRegister Simulator::fcvtn2(VectorFormat vform,
6034   int lane_count = LaneCountFromFormat(vform) / 2;
6035   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
6042     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
6052 LogicVRegister Simulator::fcvtxn(VectorFormat vform,
6057   int input_lane_count = LaneCountFromFormat(vform);
6058   if (IsSVEFormat(vform)) {
6063   dst.ClearForWrite(vform);
6064   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
6073 LogicVRegister Simulator::fcvtxn2(VectorFormat vform,
6076   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kSRegSize);
6077   int lane_count = LaneCountFromFormat(vform) / 2;
6182 LogicVRegister Simulator::frsqrte(VectorFormat vform,
6185   dst.ClearForWrite(vform);
6186   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
6187     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6189       dst.SetFloat(vform, i, FPRecipSqrtEstimate<SimFloat16>(input));
6191   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
6192     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6194       dst.SetFloat(vform, i, FPRecipSqrtEstimate<float>(input));
6197     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
6198     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6200       dst.SetFloat(vform, i, FPRecipSqrtEstimate<double>(input));
6326 LogicVRegister Simulator::frecpe(VectorFormat vform,
6330   dst.ClearForWrite(vform);
6331   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
6332     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6334       dst.SetFloat(vform, i, FPRecipEstimate<SimFloat16>(input, round));
6336   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
6337     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6339       dst.SetFloat(vform, i, FPRecipEstimate<float>(input, round));
6342     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
6343     for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6345       dst.SetFloat(vform, i, FPRecipEstimate<double>(input, round));
6352 LogicVRegister Simulator::ursqrte(VectorFormat vform,
6355   dst.ClearForWrite(vform);
6359   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6360     operand = src.Uint(vform, i);
6368     dst.SetUint(vform, i, result);
6385 LogicVRegister Simulator::urecpe(VectorFormat vform,
6388   dst.ClearForWrite(vform);
6392   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6393     operand = src.Uint(vform, i);
6401     dst.SetUint(vform, i, result);
6421 LogicPRegister Simulator::ptrue(VectorFormat vform,
6424   int count = GetPredicateConstraintLaneCount(vform, pattern);
6425   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6426     dst.SetActive(vform, i, i < count);
6431 LogicPRegister Simulator::pnext(VectorFormat vform,
6435   int next = GetLastActive(vform, src) + 1;
6436   while (next < LaneCountFromFormat(vform)) {
6437     if (pg.IsActive(vform, next)) break;
6441   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6442     dst.SetActive(vform, i, (i == next));
6448 LogicVRegister Simulator::frecpx(VectorFormat vform,
6451   dst.ClearForWrite(vform);
6452   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6484 LogicVRegister Simulator::frecpx(VectorFormat vform,
6487   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
6488     frecpx<SimFloat16>(vform, dst, src);
6489   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
6490     frecpx<float>(vform, dst, src);
6492     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
6493     frecpx<double>(vform, dst, src);
6498 LogicVRegister Simulator::flogb(VectorFormat vform,
6501   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6503     switch (vform) {
6519         dst.SetInt(vform, i, MaxIntFromFormat(vform));
6523         dst.SetInt(vform, i, MinIntFromFormat(vform));
6532         dst.SetInt(vform, i, -1023 - mant_zero_count);
6537         dst.SetInt(vform, i, static_cast<int64_t>(DoubleExp(op)) - 1023);
6544 LogicVRegister Simulator::ftsmul(VectorFormat vform,
6552   shl(vform, maybe_neg_src1, src2, LaneSizeInBitsFromFormat(vform) - 1);
6553   eor(vform, maybe_neg_src1, maybe_neg_src1, src1);
6558   fmul(vform, dst, src1, maybe_neg_src1);
6563 LogicVRegister Simulator::ftssel(VectorFormat vform,
6567   unsigned lane_bits = LaneSizeInBitsFromFormat(vform);
6580   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6583     uint64_t op = src1.Uint(vform, i);
6588     uint64_t q = src2.Uint(vform, i);
6592     dst.SetUint(vform, i, op);
6599 LogicVRegister Simulator::FTMaddHelper(VectorFormat vform,
6610   dup_immediate(vform, cf, coeff_pos);
6611   dup_immediate(vform, cfn, coeff_neg);
6618                              vform,
6625   mov_merging(vform, cf, is_neg, cfn);
6628   fabs_<T>(vform, temp, src2);
6629   fmla<T>(vform, cf, cf, src1, temp);
6630   mov(vform, dst, cf);
6635 LogicVRegister Simulator::ftmad(VectorFormat vform,
6694   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
6695     FTMaddHelper<SimFloat16>(vform,
6701   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
6702     FTMaddHelper<float>(vform,
6709     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
6710     FTMaddHelper<double>(vform,
6720 LogicVRegister Simulator::fexpa(VectorFormat vform,
6762   unsigned lane_size = LaneSizeInBitsFromFormat(vform);
6786   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6787     uint64_t op = src.Uint(vform, i);
6790     dst.SetUint(vform, i, result);
6796 LogicVRegister Simulator::fscale(VectorFormat vform,
6801   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6804       int64_t scale = src2.Int(vform, i);
6829 LogicVRegister Simulator::fscale(VectorFormat vform,
6833   if (LaneSizeInBitsFromFormat(vform) == kHRegSize) {
6834     fscale<SimFloat16>(vform, dst, src1, src2);
6835   } else if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
6836     fscale<float>(vform, dst, src1, src2);
6838     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
6839     fscale<double>(vform, dst, src1, src2);
6844 LogicVRegister Simulator::scvtf(VectorFormat vform,
6852   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= dst_data_size_in_bits);
6853   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= src_data_size_in_bits);
6854   dst.ClearForWrite(vform);
6856   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6857     if (!pg.IsActive(vform, i)) continue;
6861                                             src.Uint(vform, i));
6866         dst.SetUint(vform, i, Float16ToRawbits(result));
6871         dst.SetUint(vform, i, FloatToRawbits(result));
6876         dst.SetUint(vform, i, DoubleToRawbits(result));
6888 LogicVRegister Simulator::scvtf(VectorFormat vform,
6893   return scvtf(vform,
6894                LaneSizeInBitsFromFormat(vform),
6895                LaneSizeInBitsFromFormat(vform),
6903 LogicVRegister Simulator::ucvtf(VectorFormat vform,
6911   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= dst_data_size_in_bits);
6912   VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) >= src_data_size_in_bits);
6913   dst.ClearForWrite(vform);
6915   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
6916     if (!pg.IsActive(vform, i)) continue;
6920                                                src.Uint(vform, i));
6925         dst.SetUint(vform, i, Float16ToRawbits(result));
6930         dst.SetUint(vform, i, FloatToRawbits(result));
6935         dst.SetUint(vform, i, DoubleToRawbits(result));
6947 LogicVRegister Simulator::ucvtf(VectorFormat vform,
6952   return ucvtf(vform,
6953                LaneSizeInBitsFromFormat(vform),
6954                LaneSizeInBitsFromFormat(vform),
6962 LogicVRegister Simulator::unpk(VectorFormat vform,
6967   VectorFormat vform_half = VectorFormatHalfWidth(vform);
6968   const int lane_count = LaneCountFromFormat(vform);
6978         dst.SetInt(vform, i, result[i]);
6988         dst.SetUint(vform, i, result[i]);
6999                                                      VectorFormat vform,
7006   for (int lane = 0; lane < LaneCountFromFormat(vform); lane++) {
7008     if (mask.IsActive(vform, lane)) {
7011       int d_lane = (lane * LaneSizeInBitsFromFormat(vform)) / kDRegSize;
7019           op1 = src1.Int(vform, lane);
7021                                  : src2.Int(vform, lane);
7027           op1 = src1.Uint(vform, lane);
7029                                  : src2.Uint(vform, lane);
7070     dst.SetActive(vform, lane, result);
7073   if (flags == SetFlags) PredTest(vform, mask, dst);
7079                                                 VectorFormat vform,
7084   unsigned lane_size = LaneSizeInBitsFromFormat(vform);
7085   VectorFormat shift_vform = is_wide_elements ? kFormatVnD : vform;
7087   for (int lane = 0; lane < LaneCountFromFormat(vform); lane++) {
7099     uint64_t value = src1.Uint(vform, lane);
7104     dst.SetUint(vform, lane, result);
7110 LogicVRegister Simulator::asrd(VectorFormat vform,
7115                               LaneSizeInBitsFromFormat(vform)));
7117   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7118     int64_t value = src1.Int(vform, i);
7129     dst.SetInt(vform, i, value);
7136     VectorFormat vform,
7140   VIXL_ASSERT(IsSVEFormat(vform));
7141   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7142     uint64_t op1 = zn.Uint(vform, i);
7143     uint64_t op2 = zm.Uint(vform, i);
7161     zd.SetUint(vform, i, result);
7214     VectorFormat vform,
7217   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7218     uint64_t op1 = zd.Uint(vform, i);
7233     zd.SetUint(vform, i, result);
7239 void Simulator::SVEStructuredStoreHelper(VectorFormat vform,
7245   int esize_in_bytes_log2 = LaneSizeInBytesLog2FromFormat(vform);
7272   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7273     if (!pg.IsActive(vform, i)) continue;
7282     PrintRegisterFormat format = GetPrintRegisterFormat(vform);
7301 void Simulator::SVEStructuredLoadHelper(VectorFormat vform,
7306   int esize_in_bytes_log2 = LaneSizeInBytesLog2FromFormat(vform);
7326   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7330       if (!pg.IsActive(vform, i)) {
7331         zt[r].SetUint(vform, i, 0);
7336         LoadIntToLane(zt[r], vform, msize_in_bytes, i, element_address);
7338         LoadUintToLane(zt[r], vform, msize_in_bytes, i, element_address);
7344     PrintRegisterFormat format = GetPrintRegisterFormat(vform);
7432 void Simulator::SVEFaultTolerantLoadHelper(VectorFormat vform,
7438   int esize_in_bytes = LaneSizeInBytesFromFormat(vform);
7452   int fake_fault_at_lane = rnd % (LaneCountFromFormat(vform) * 8);
7454   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7457     if (pg.IsActive(vform, i)) {
7468       } else if (ffr.IsActive(vform, i)) {
7479           for (int j = i; j < LaneCountFromFormat(vform); j++) {
7480             ffr.SetActive(vform, j, false);
7493     if (ffr.IsActive(vform, i)) {
7496         zt.SetInt(vform, i, ExtractSignedBitfield64(msb, 0, value));
7498         zt.SetUint(vform, i, ExtractUnsignedBitfield64(msb, 0, value));
7504     PrintRegisterFormat format = GetPrintRegisterFormat(vform);
7518                                                     VectorFormat vform,
7528                              vform,
7533     SVEFaultTolerantLoadHelper(vform,
7540     SVEStructuredLoadHelper(vform,
7548 int Simulator::GetFirstActive(VectorFormat vform,
7550   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7551     if (pg.IsActive(vform, i)) return i;
7556 int Simulator::GetLastActive(VectorFormat vform,
7558   for (int i = LaneCountFromFormat(vform) - 1; i >= 0; i--) {
7559     if (pg.IsActive(vform, i)) return i;
7564 int Simulator::CountActiveLanes(VectorFormat vform,
7567   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7568     count += pg.IsActive(vform, i) ? 1 : 0;
7573 int Simulator::CountActiveAndTrueLanes(VectorFormat vform,
7577   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7578     count += (pg.IsActive(vform, i) && pn.IsActive(vform, i)) ? 1 : 0;
7583 int Simulator::GetPredicateConstraintLaneCount(VectorFormat vform,
7585   VIXL_ASSERT(IsSVEFormat(vform));
7586   int all = LaneCountFromFormat(vform);
7625 LogicPRegister Simulator::match(VectorFormat vform,
7634   int lanes_per_segment = kQRegSize / LaneSizeInBitsFromFormat(vform);
7636     dup_elements_to_segments(vform, ztemp, needles, i);
7638                                vform,
7648     ptrue(vform, ptemp, SVE_ALL);
7699 LogicVRegister Simulator::pack_odd_elements(VectorFormat vform,
7704   return uzp2(vform, dst, src, zero);
7707 LogicVRegister Simulator::pack_even_elements(VectorFormat vform,
7712   return uzp1(vform, dst, src, zero);
7715 LogicVRegister Simulator::adcl(VectorFormat vform,
7720   unsigned reg_size = LaneSizeInBitsFromFormat(vform);
7723   for (int i = 0; i < LaneCountFromFormat(vform); i += 2) {
7724     uint64_t left = src1.Uint(vform, i + (top ? 1 : 0));
7725     uint64_t right = dst.Uint(vform, i);
7726     unsigned carry_in = src2.Uint(vform, i + 1) & 1;
7731     dst.SetUint(vform, i, val_and_flags.first);
7735     dst.SetUint(vform, i + 1, carry_out);
7811 LogicVRegister Simulator::fmatmul(VectorFormat vform,
7831   for (int i = 0; i < LaneCountFromFormat(vform); i++) {
7836     srcdst.SetFloat<T>(vform, i, value);
7841 LogicVRegister Simulator::fmatmul(VectorFormat vform,
7845   if (LaneSizeInBitsFromFormat(vform) == kSRegSize) {
7846     fmatmul<float>(vform, dst, src1, src2);
7848     VIXL_ASSERT(LaneSizeInBitsFromFormat(vform) == kDRegSize);
7849     fmatmul<double>(vform, dst, src1, src2);

Indexes created Thu Nov 07 10:32:03 CST 2024