Lines Matching defs:masm
57 // Initialise(&masm, p0, 0x1234); // Sets p0 = 0b'0001'0010'0011'0100
58 void Initialise(MacroAssembler* masm,
65 UseScratchRegisterScope temps(masm);
70 masm->Adr(temp, &data);
71 masm->Ldr(pd, SVEMemOperand(temp));
72 masm->B(&done);
74 ExactAssemblyScope total(masm, kPRegMaxSizeInBytes);
75 masm->bind(&data);
76 masm->dc64(value0);
77 masm->dc64(value1);
78 masm->dc64(value2);
79 masm->dc64(value3);
81 masm->Bind(&done);
83 void Initialise(MacroAssembler* masm,
88 Initialise(masm, pd, 0, value2, value1, value0);
90 void Initialise(MacroAssembler* masm,
94 Initialise(masm, pd, 0, 0, value1, value0);
96 void Initialise(MacroAssembler* masm, const PRegister& pd, uint64_t value0) {
97 Initialise(masm, pd, 0, 0, 0, value0);
106 // Initialise(&masm, p0.VnS(), values); // Sets p0 = 0b'0000'0001'0010
115 void Initialise(MacroAssembler* masm,
137 Initialise(masm, pd, chunks[3], chunks[2], chunks[1], chunks[0]);
156 InsrHelper(&masm, z1.VnH(), z1_inputs);
204 Initialise(&masm, p0.VnB(), p0_inputs);
207 Initialise(&masm, p1.VnH(), p1_inputs);
210 Initialise(&masm, p2.VnS(), p2_inputs);
213 Initialise(&masm, p3.VnD(), p3_inputs);
221 Initialise(&masm, p4.VnH(), p4_inputs);
225 Initialise(&masm, p5.VnS(), p5_inputs);
229 Initialise(&masm, p6.VnD(), p6_inputs);
236 Initialise(&masm, p7.VnD(), p7_inputs);
385 InsrHelper(&masm, zd, zd_inputs);
386 InsrHelper(&masm, za, za_inputs);
387 InsrHelper(&masm, zn, zn_inputs);
388 InsrHelper(&masm, zm, zm_inputs);
395 Initialise(&masm, p0.WithLaneSize(lane_size_in_bits), p0_inputs);
396 Initialise(&masm, p1.WithLaneSize(lane_size_in_bits), p1_inputs);
397 Initialise(&masm, p2.WithLaneSize(lane_size_in_bits), p2_inputs);
398 Initialise(&masm, p3.WithLaneSize(lane_size_in_bits), p3_inputs);
486 InsrHelper(&masm, z8.VnD(), z8_inputs);
488 InsrHelper(&masm, z15.VnD(), z15_inputs);
518 Initialise(&masm, p2.VnB(), p2_inputs);
519 Initialise(&masm, p3.VnB(), p3_inputs);
602 Initialise(&masm, p2.VnB(), p2_inputs);
603 Initialise(&masm, p3.VnB(), p3_inputs);
686 Initialise(&masm, p2.VnB(), p2_inputs);
687 Initialise(&masm, p3.VnB(), p3_inputs);
764 Initialise(&masm, p2.VnB(), p2_inputs);
765 Initialise(&masm, p3.VnB(), p3_inputs);
842 Initialise(&masm, p2.VnB(), p2_inputs);
843 Initialise(&masm, p3.VnB(), p3_inputs);
982 Initialise(&masm, p2.VnB(), p2b_inputs);
983 Initialise(&masm, p3.VnB(), p3b_inputs);
984 Initialise(&masm, p4.VnB(), p4b_inputs);
985 Initialise(&masm, p5.VnB(), p5b_inputs);
986 Initialise(&masm, p6.VnB(), p6b_inputs);
1007 Initialise(&masm, p2.VnH(), p2h_inputs);
1008 Initialise(&masm, p3.VnH(), p3h_inputs);
1017 Initialise(&masm, p2.VnS(), p2s_inputs);
1018 Initialise(&masm, p3.VnS(), p3s_inputs);
1027 Initialise(&masm, p2.VnD(), p2d_inputs);
1028 Initialise(&masm, p3.VnD(), p3d_inputs);
1080 Initialise(&masm, p10.VnB(), p10_inputs);
1081 Initialise(&masm, p11.VnB(), p11_inputs);
1082 Initialise(&masm, p12.VnB(), p12_inputs);
1138 InsrHelper(&masm, z10.VnB(), z10_inputs);
1139 InsrHelper(&masm, z11.VnB(), z11_inputs);
1140 Initialise(&masm, p0.VnB(), p0_inputs);
1148 InsrHelper(&masm, z12.VnD(), z12_inputs);
1149 InsrHelper(&masm, z13.VnD(), z13_inputs);
1150 Initialise(&masm, p1.VnD(), p1_inputs);
1159 InsrHelper(&masm, z14.VnH(), z14_inputs);
1160 InsrHelper(&masm, z15.VnH(), z15_inputs);
1161 Initialise(&masm, p2.VnH(), p2_inputs);
1172 InsrHelper(&masm, z16.VnS(), z16_inputs);
1173 InsrHelper(&masm, z17.VnS(), z17_inputs);
1174 Initialise(&masm, p3.VnS(), p3_inputs);
1235 InsrHelper(&masm, z13.VnB(), src1_inputs_1);
1236 InsrHelper(&masm, z19.VnD(), src2_inputs_1);
1237 Initialise(&masm, p0.VnB(), mask_inputs_1);
1247 InsrHelper(&masm, z13.VnH(), src1_inputs_2);
1248 InsrHelper(&masm, z19.VnD(), src2_inputs_2);
1249 Initialise(&masm, p0.VnH(), mask_inputs_2);
1259 InsrHelper(&masm, z13.VnS(), src1_inputs_3);
1260 InsrHelper(&masm, z19.VnD(), src2_inputs_3);
1261 Initialise(&masm, p0.VnS(), mask_inputs_3);
1271 InsrHelper(&masm, z13.VnB(), src1_inputs_4);
1272 InsrHelper(&masm, z19.VnD(), src2_inputs_4);
1273 Initialise(&masm, p0.VnB(), mask_inputs_4);
1283 InsrHelper(&masm, z13.VnS(), src1_inputs_5);
1284 InsrHelper(&masm, z19.VnD(), src2_inputs_5);
1285 Initialise(&masm, p0.VnS(), mask_inputs_5);
1352 InsrHelper(&masm, z1.VnD(), z21_inputs);
1353 InsrHelper(&masm, z2.VnS(), z22_inputs);
1354 InsrHelper(&masm, z3.VnH(), z23_inputs);
1355 InsrHelper(&masm, z4.VnB(), z24_inputs);
1362 InsrHelper(&masm, z5.VnD(), z21_inputs);
1363 InsrHelper(&masm, z6.VnS(), z22_inputs);
1364 InsrHelper(&masm, z7.VnH(), z23_inputs);
1365 InsrHelper(&masm, z8.VnB(), z24_inputs);
1372 InsrHelper(&masm, z9.VnD(), z21_inputs);
1373 InsrHelper(&masm, z10.VnS(), z22_inputs);
1374 InsrHelper(&masm, z11.VnH(), z23_inputs);
1375 InsrHelper(&masm, z12.VnB(), z24_inputs);
1385 ExactAssemblyScope guard(&masm, 4 * kInstructionSize);
1511 Initialise(&masm, p0.VnB(), p0_inputs);
1566 Initialise(&masm, p0.VnB(), p0_inputs);
1679 Initialise(&masm, p0.VnB(), p0_inputs);
1789 Initialise(&masm, p0.VnB(), p0_inputs);
1794 InsrHelper(&masm, z0.VnD(), z0_inputs);
1797 InsrHelper(&masm, z1.VnD(), z1_inputs);
1800 InsrHelper(&masm, z2.VnS(), z2_inputs);
1803 InsrHelper(&masm, z3.VnH(), z3_inputs);
1937 Initialise(&masm, p0.VnB(), p0_inputs);
1942 InsrHelper(&masm, z0.VnD(), z0_inputs);
1945 InsrHelper(&masm, z1.VnD(), z1_inputs);
1948 InsrHelper(&masm, z2.VnS(), z2_inputs);
1951 InsrHelper(&masm, z3.VnH(), z3_inputs);
2085 Initialise(&masm, p0.VnB(), p0_inputs);
2090 InsrHelper(&masm, z0.VnD(), z0_inputs);
2093 InsrHelper(&masm, z1.VnD(), z1_inputs);
2096 InsrHelper(&masm, z2.VnS(), z2_inputs);
2099 InsrHelper(&masm, z3.VnH(), z3_inputs);
2512 InsrHelper(&masm, z13.VnB(), z13_inputs);
2513 Initialise(&masm, p0.VnB(), mask_inputs1);
2521 InsrHelper(&masm, z14.VnH(), z14_inputs);
2522 Initialise(&masm, p0.VnH(), mask_inputs2);
2530 InsrHelper(&masm, z15.VnS(), z15_inputs);
2531 Initialise(&masm, p0.VnS(), mask_inputs3);
2543 InsrHelper(&masm, z16.VnD(), z16_inputs);
2544 Initialise(&masm, p0.VnD(), mask_inputs4);
2610 InsrHelper(&masm, z13.VnB(), src1_inputs);
2611 Initialise(&masm, p0.VnB(), mask_inputs1);
2619 InsrHelper(&masm, z13.VnH(), src2_inputs);
2620 Initialise(&masm, p0.VnH(), mask_inputs2);
2628 InsrHelper(&masm, z13.VnS(), src3_inputs);
2629 Initialise(&masm, p0.VnS(), mask_inputs3);
2637 InsrHelper(&masm, z13.VnD(), src4_inputs);
2638 Initialise(&masm, p0.VnD(), mask_inputs4);
2776 Initialise(&masm, pg.WithLaneSize(lane_size_in_bits), pg_inputs);
2777 Initialise(&masm, pn.WithLaneSize(lane_size_in_bits), pn_inputs);
2786 (masm.*macro)(p0.WithLaneSize(lane_size_in_bits),
2793 (masm.*macro)(p1.WithLaneSize(lane_size_in_bits),
2800 UseScratchRegisterScope temps(&masm);
2804 (masm.*macro)(p2.WithLaneSize(lane_size_in_bits),
2942 Initialise(&masm, p0.VnB(), in_b);
2943 Initialise(&masm, p1.VnH(), in_h);
2944 Initialise(&masm, p2.VnS(), in_s);
2945 Initialise(&masm, p3.VnD(), in_d);
3283 Initialise(&masm, p0.VnB(), in_b);
3284 Initialise(&masm, p1.VnH(), in_h);
3285 Initialise(&masm, p2.VnS(), in_s);
3286 Initialise(&masm, p3.VnD(), in_d);
3390 ExactAssemblyScope guard(&masm, 12 * kInstructionSize);
3392 (masm.*assemble)(p[9], 0xe);
3396 (masm.*assemble)(p[10], 0x16);
3400 (masm.*assemble)(p[11], 0x1a);
3404 (masm.*assemble)(p[12], 0x1c);
3539 Initialise(&masm, p2.VnB(), in2);
3540 Initialise(&masm, p3.VnB(), in3);
3541 Initialise(&masm, p4.VnB(), in4);
3629 Initialise(&masm, p0.VnB(), p0_inputs);
3698 void GenerateCntSequence(MacroAssembler* masm,
3703 masm->Mov(x0, acc_value);
3704 masm->Mov(x1, acc_value);
3705 masm->Mov(x2, acc_value);
3706 masm->Mov(x3, acc_value);
3707 masm->Mov(x4, acc_value);
3708 masm->Mov(x5, acc_value);
3709 masm->Mov(x6, acc_value);
3710 masm->Mov(x7, acc_value);
3711 masm->Mov(x8, acc_value);
3712 masm->Mov(x9, acc_value);
3713 masm->Mov(x10, acc_value);
3714 masm->Mov(x11, acc_value);
3715 masm->Mov(x12, acc_value);
3716 masm->Mov(x13, acc_value);
3717 masm->Mov(x14, acc_value);
3718 masm->Mov(x15, acc_value);
3719 masm->Mov(x18, acc_value);
3720 masm->Mov(x19, acc_value);
3721 masm->Mov(x20, acc_value);
3722 masm->Mov(x21, acc_value);
3724 (masm->*cnt)(Register(0, sizeof(T) * kBitsPerByte), SVE_POW2, multiplier);
3725 (masm->*cnt)(Register(1, sizeof(T) * kBitsPerByte), SVE_VL1, multiplier);
3726 (masm->*cnt)(Register(2, sizeof(T) * kBitsPerByte), SVE_VL2, multiplier);
3727 (masm->*cnt)(Register(3, sizeof(T) * kBitsPerByte), SVE_VL3, multiplier);
3728 (masm->*cnt)(Register(4, sizeof(T) * kBitsPerByte), SVE_VL4, multiplier);
3729 (masm->*cnt)(Register(5, sizeof(T) * kBitsPerByte), SVE_VL5, multiplier);
3730 (masm->*cnt)(Register(6, sizeof(T) * kBitsPerByte), SVE_VL6, multiplier);
3731 (masm->*cnt)(Register(7, sizeof(T) * kBitsPerByte), SVE_VL7, multiplier);
3732 (masm->*cnt)(Register(8, sizeof(T) * kBitsPerByte), SVE_VL8, multiplier);
3733 (masm->*cnt)(Register(9, sizeof(T) * kBitsPerByte), SVE_VL16, multiplier);
3734 (masm->*cnt)(Register(10, sizeof(T) * kBitsPerByte), SVE_VL32, multiplier);
3735 (masm->*cnt)(Register(11, sizeof(T) * kBitsPerByte), SVE_VL64, multiplier);
3736 (masm->*cnt)(Register(12, sizeof(T) * kBitsPerByte), SVE_VL128, multiplier);
3737 (masm->*cnt)(Register(13, sizeof(T) * kBitsPerByte), SVE_VL256, multiplier);
3738 (masm->*cnt)(Register(14, sizeof(T) * kBitsPerByte), 16, multiplier);
3739 (masm->*cnt)(Register(15, sizeof(T) * kBitsPerByte), 23, multiplier);
3740 (masm->*cnt)(Register(18, sizeof(T) * kBitsPerByte), 28, multiplier);
3741 (masm->*cnt)(Register(19, sizeof(T) * kBitsPerByte), SVE_MUL4, multiplier);
3742 (masm->*cnt)(Register(20, sizeof(T) * kBitsPerByte), SVE_MUL3, multiplier);
3743 (masm->*cnt)(Register(21, sizeof(T) * kBitsPerByte), SVE_ALL, multiplier);
3761 GenerateCntSequence(&masm, cnt, acc_value, multiplier);
3924 GenerateCntSequence(&masm, cnt, acc_value, multiplier);
4195 (masm.*cnt)(x0, w0, SVE_POW2, multiplier);
4196 (masm.*cnt)(x1, w1, SVE_VL1, multiplier);
4197 (masm.*cnt)(x2, w2, SVE_VL2, multiplier);
4198 (masm.*cnt)(x3, w3, SVE_VL3, multiplier);
4199 (masm.*cnt)(x4, w4, SVE_VL4, multiplier);
4200 (masm.*cnt)(x5, w5, SVE_VL5, multiplier);
4201 (masm.*cnt)(x6, w6, SVE_VL6, multiplier);
4202 (masm.*cnt)(x7, w7, SVE_VL7, multiplier);
4203 (masm.*cnt)(x8, w8, SVE_VL8, multiplier);
4204 (masm.*cnt)(x9, w9, SVE_VL16, multiplier);
4205 (masm.*cnt)(x10, w10, SVE_VL32, multiplier);
4206 (masm.*cnt)(x11, w11, SVE_VL64, multiplier);
4207 (masm.*cnt)(x12, w12, SVE_VL128, multiplier);
4208 (masm.*cnt)(x13, w13, SVE_VL256, multiplier);
4209 (masm.*cnt)(x14, w14, 16, multiplier);
4210 (masm.*cnt)(x15, w15, 23, multiplier);
4211 (masm.*cnt)(x18, w18, 28, multiplier);
4212 (masm.*cnt)(x19, w19, SVE_MUL4, multiplier);
4213 (masm.*cnt)(x20, w20, SVE_MUL3, multiplier);
4214 (masm.*cnt)(x21, w21, SVE_ALL, multiplier);
4350 InsrHelper(&masm, ZRegister(i, kDRegSize), acc_inputs);
4357 (masm.*fn)(z16.WithLaneSize(lane_size_in_bits), SVE_POW2, multiplier);
4358 (masm.*fn)(z17.WithLaneSize(lane_size_in_bits), SVE_VL1, multiplier);
4359 (masm.*fn)(z18.WithLaneSize(lane_size_in_bits), SVE_VL2, multiplier);
4360 (masm.*fn)(z19.WithLaneSize(lane_size_in_bits), SVE_VL3, multiplier);
4361 (masm.*fn)(z20.WithLaneSize(lane_size_in_bits), SVE_VL4, multiplier);
4362 (masm.*fn)(z21.WithLaneSize(lane_size_in_bits), SVE_VL7, multiplier);
4363 (masm.*fn)(z22.WithLaneSize(lane_size_in_bits), SVE_VL8, multiplier);
4364 (masm.*fn)(z23.WithLaneSize(lane_size_in_bits), SVE_VL16, multiplier);
4365 (masm.*fn)(z24.WithLaneSize(lane_size_in_bits), SVE_VL64, multiplier);
4366 (masm.*fn)(z25.WithLaneSize(lane_size_in_bits), SVE_VL256, multiplier);
4367 (masm.*fn)(z26.WithLaneSize(lane_size_in_bits), 16, multiplier);
4368 (masm.*fn)(z27.WithLaneSize(lane_size_in_bits), 28, multiplier);
4369 (masm.*fn)(z28.WithLaneSize(lane_size_in_bits), SVE_MUL3, multiplier);
4370 (masm.*fn)(z29.WithLaneSize(lane_size_in_bits), SVE_MUL4, multiplier);
4371 (masm.*fn)(z30.WithLaneSize(lane_size_in_bits), SVE_ALL, multiplier);
4374 (masm.*cnt)(x0, SVE_POW2, multiplier);
4375 (masm.*cnt)(x1, SVE_VL1, multiplier);
4376 (masm.*cnt)(x2, SVE_VL2, multiplier);
4377 (masm.*cnt)(x3, SVE_VL3, multiplier);
4378 (masm.*cnt)(x4, SVE_VL4, multiplier);
4379 (masm.*cnt)(x5, SVE_VL7, multiplier);
4380 (masm.*cnt)(x6, SVE_VL8, multiplier);
4381 (masm.*cnt)(x7, SVE_VL16, multiplier);
4382 (masm.*cnt)(x8, SVE_VL64, multiplier);
4383 (masm.*cnt)(x9, SVE_VL256, multiplier);
4384 (masm.*cnt)(x10, 16, multiplier);
4385 (masm.*cnt)(x11, 28, multiplier);
4386 (masm.*cnt)(x12, SVE_MUL3, multiplier);
4387 (masm.*cnt)(x13, SVE_MUL4, multiplier);
4388 (masm.*cnt)(x14, SVE_ALL, multiplier);
4395 (masm.*addsub)(zsrcdst, zsrcdst, zscratch);
4487 InsrHelper(&masm, src_a, zn_inputs);
4488 InsrHelper(&masm, src_b, zm_inputs);
4490 Initialise(&masm, p0.WithLaneSize(lane_size_in_bits), pg_inputs);
4498 (masm.*macro)(zd_1, p0.Merging(), zd_1, src_b);
4505 (masm.*macro)(zd_2, p0.Merging(), src_a, zd_2);
4514 (masm.*macro)(zd_3, p0.Merging(), src_a, src_b);
5048 InsrHelper(&masm, zn, zn_inputs);
5049 InsrHelper(&masm, zm, zm_inputs);
5052 (masm.*macro)(zd, zn, zm);
5615 InsrHelper(&masm, z9.VnD(), inputs);
5744 InsrHelper(&masm, z9.VnD(), z9_inputs);
5883 Initialise(&masm, p0.VnB(), pg_in);
5889 InsrHelper(&masm, z31.VnD(), in);
5991 Initialise(&masm, p0.VnB(), pg_in);
5997 InsrHelper(&masm, z31.VnD(), in);
6079 Initialise(&masm, p0.VnB(), pg_in);
6085 InsrHelper(&masm, z31.VnD(), in);
6161 Initialise(&masm, p0.VnB(), pg_in);
6167 InsrHelper(&masm, z31.VnD(), in);
6247 Initialise(&masm, p0.VnB(), pg_in);
6253 InsrHelper(&masm, z31.VnD(), in);
6330 Initialise(&masm, p0.VnB(), pg_in);
6333 InsrHelper(&masm, z31.VnD(), in);
6338 InsrHelper(&masm, z31.VnD(), in);
6419 Initialise(&masm, pg.VnB(), pg_in);
6526 Initialise(&masm, pg.VnB(), pg_in);
6633 Initialise(&masm, pg.VnB(), pg_in);
6771 InsrHelper(&masm, z9.VnD(), table_inputs);
6778 InsrHelper(&masm, ind_b, index_b);
6779 InsrHelper(&masm, ind_h, index_h);
6780 InsrHelper(&masm, ind_s, index_s);
6781 InsrHelper(&masm, ind_d, index_d);
6953 Initialise(&masm,
7269 Initialise(&masm,
7437 Initialise(&masm,
7653 Initialise(&masm,
7844 Initialise(&masm,
8091 Initialise(&masm,
8326 Initialise(&masm,
8731 static void ScalarLoadHelper(MacroAssembler* masm,
8739 masm->Ldrsb(dst, MemOperand(addr));
8742 masm->Ldrsh(dst, MemOperand(addr));
8745 masm->Ldrsw(dst, MemOperand(addr));
8754 masm->Ldrb(dst, MemOperand(addr));
8757 masm->Ldrh(dst, MemOperand(addr));
8760 masm->Ldr(dst.W(), MemOperand(addr));
8763 masm->Ldr(dst, MemOperand(addr));
8776 static void ScalarLoadHelper(MacroAssembler* masm,
8786 masm->Index(lane_numbers, 0, 1);
8787 masm->Dup(zt_ref, 0);
8789 masm->Mov(x29, addresses[N - i - 1]);
8791 ScalarLoadHelper(masm, rt, x29, msize_in_bits, is_signed);
8794 masm->Cmpeq(p7.WithLaneSize(esize_in_bits), pg, lane_numbers, i);
8795 masm->Cpy(zt_ref, p7.Merging(), rt);
8866 (masm.*ldff1)(z0.WithLaneSize(esize_in_bits),
8931 (masm.*
8942 (masm.*
8951 (masm.*ld1)(z16.WithLaneSize(esize_in_bits),
9875 Initialise(&masm,
9885 ScalarLoadHelper(&masm,
9894 InsrHelper(&masm, zn, offsets);
9900 (masm.*ld1)(zt, pg, SVEMemOperand(x0, zn, mod, shift));
9908 (masm.*ldff1)(zt_ff, pg, SVEMemOperand(x0, zn, mod, shift));
9976 Initialise(&masm,
9988 InsrHelper(&masm, zn, addresses);
9989 (masm.*sve_ld1)(zt_addresses, pg, SVEMemOperand(zn));
9992 InsrHelper(&masm, zn, offsets);
9993 (masm.*sve_ld1)(zt_offsets, pg, SVEMemOperand(zn, data));
9995 InsrHelper(&masm, zn, maxed_offsets);
9996 (masm.*sve_ld1)(zt_maxed, pg, SVEMemOperand(zn, maxed_offsets_imm));
9999 ScalarLoadHelper(&masm,
10791 InsrHelper(&masm, zd1, zn_inputs);
10795 InsrHelper(&masm, zn, zn_inputs);
10803 UseScratchRegisterScope temps(&masm);
10808 (masm.*macro)(zd1, zd1, imm);
10809 (masm.*macro)(zd2, zn, imm);
11313 InsrHelper(&masm, z31.VnD(), in);
11320 Initialise(&masm, p0.VnB(), pg_in);
11401 InsrHelper(&masm, z31.VnD(), in);
11408 Initialise(&masm, p0.VnB(), pg_in);
11470 InsrHelper(&masm, z31.VnD(), in);
11477 Initialise(&masm, p0.VnB(), pg_in);
11542 InsrHelper(&masm, z31.VnD(), in);
11549 Initialise(&masm, p0.VnB(), pg_in);
11650 InsrHelper(&masm, zd, zd_inputs);
11651 InsrHelper(&masm, za, za_inputs);
11652 InsrHelper(&masm, zn, zn_inputs);
11653 InsrHelper(&masm, zm, zm_inputs);
11907 static void IntSegmentPatternHelper(MacroAssembler* masm,
11911 UseScratchRegisterScope temps(masm);
11913 masm->Index(ztmp, 0, 1);
11914 masm->Asr(ztmp, ztmp, kQRegSizeInBytesLog2 - dst.GetLaneSizeInBytesLog2());
11915 masm->Add(dst, src, ztmp);
11944 IntSegmentPatternHelper(&masm, z27.VnB(), z27.VnB());
12045 IntSegmentPatternHelper(&masm, z0.VnH(), z0.VnH());
12120 InsrHelper(&masm, zn, zn_rawbits);
12121 InsrHelper(&masm, zm, zm_rawbits);
12123 (masm.*macro)(zd, zn, zm);
12308 InsrHelper(&masm, zn, zn_inputs_rawbits);
12309 InsrHelper(&masm, zm, zm_inputs_rawbits);
12310 InsrHelper(&masm, zd, zd_inputs_rawbits);
12313 Initialise(&masm, pg, pg_inputs);
12319 (masm.*macro)(dn_result, pg.Merging(), dn_result, zm, nan_option);
12321 (masm.*macro_nonan)(dn_result, pg.Merging(), dn_result, zm);
12331 (masm.*macro)(dm_result, pg.Merging(), zn, dm_result, nan_option);
12333 (masm.*macro_nonan)(dm_result, pg.Merging(), zn, dm_result);
12345 (masm.*macro)(d_result, pg.Merging(), zn, zm, nan_option);
12347 (masm.*macro_nonan)(d_result, pg.Merging(), zn, zm);
12474 Initialise(&masm, p0.VnB(), pg_in);
12477 InsrHelper(&masm, z30.VnD(), in0);
12478 InsrHelper(&masm, z31.VnD(), in1);
12696 InsrHelper(&masm, zn, zn_inputs);
12811 InsrHelper(&masm, zn, zn_inputs);
12812 InsrHelper(&masm, zm, zm_inputs);
12814 (masm.*macro)(zd, zn, zm);
12820 (masm.*macro)(zd_max_shift_amount, zn, zm_max_shift_amount);
12826 (masm.*macro)(zd_out_of_range, zn, zm_out_of_range);
13275 Initialise(&masm, p1.VnB(), inputs);
13322 Initialise(&masm, p0.VnB(), pg);
13323 Initialise(&masm, p1.VnB(), pd);
13382 Initialise(&masm, pg.VnB(), pg_inputs);
13383 Initialise(&masm, pn.VnB(), pn_inputs);
13384 Initialise(&masm, pm.VnB(), pm_inputs);
13390 (masm.*macro_set_flags)(p0.VnB(), pg.Zeroing(), pn.VnB(), pm.VnB());
13393 (masm.*macro)(p1.VnB(), pg.Zeroing(), pn.VnB(), pm.VnB());
13591 InsrHelper(&masm, z0.VnD(), inputs);
13595 Initialise(&masm, p2.VnB(), pred);
13637 InsrHelper(&masm, z0.VnD(), inputs);
13641 Initialise(&masm, p2.VnB(), pred);
13685 InsrHelper(&masm, z0.VnD(), in);
13686 InsrHelper(&masm, z1.VnD(), q);
13716 InsrHelper(&masm, z0.VnD(), in0);
13717 InsrHelper(&masm, z1.VnD(), in1);
13718 InsrHelper(&masm, z2.VnD(), in2);
13719 InsrHelper(&masm, z3.VnD(), in3);
13720 InsrHelper(&masm, z4.VnD(), in4);
13721 InsrHelper(&masm, z5.VnD(), in5);
13753 Initialise(&masm,
13785 Initialise(&masm, p0.VnB(), 0xa5a55a5a);
13842 Initialise(&masm, p0.VnB(), 0x55a55aaa);
13899 Initialise(&masm,
13961 Initialise(&masm,
14023 Initialise(&masm,
14076 Initialise(&masm,
14130 Initialise(&masm, pg.VnB(), pg_inputs);
14131 Initialise(&masm, pn.VnB(), pn_inputs);
14132 Initialise(&masm, pd_m.VnB(), pd_inputs);
14138 (masm.*macro)(pd_z.VnB(), pg.Zeroing(), pn.VnB());
14139 (masm.*macro_set_flags)(pd_z_s.VnB(), pg.Zeroing(), pn.VnB());
14142 (masm.*macro)(pd_m.VnB(), pg.Merging(), pn.VnB());
14328 Initialise(&masm, pg.VnB(), pg_inputs);
14329 Initialise(&masm, pn.VnB(), pn_inputs);
14330 Initialise(&masm, pm.VnB(), pm_inputs);
14331 Initialise(&masm, pdm.VnB(), pm_inputs);
14332 Initialise(&masm, pd.VnB(), pd_inputs);
14333 Initialise(&masm, pd_s.VnB(), pd_inputs);
14406 InsrHelper(&masm, z0.VnD(), in0);
14407 InsrHelper(&masm, z1.VnD(), in1);
14796 static void FPSegmentPatternHelper(MacroAssembler* masm,
14801 UseScratchRegisterScope temps(masm);
14803 masm->Index(ztmp, 0, 1);
14804 masm->Asr(ztmp, ztmp, kQRegSizeInBytesLog2 - dst.GetLaneSizeInBytesLog2());
14805 masm->Scvtf(ztmp, ptrue, ztmp);
14806 masm->Fadd(dst, src, ztmp);
14819 InsrHelper(&masm, z25.VnD(), in0);
14821 InsrHelper(&masm, z1.VnD(), in1);
14823 FPSegmentPatternHelper(&masm, z0.VnH(), p0.Merging(), z25.VnH());
14839 FPSegmentPatternHelper(&masm, z12.VnH(), p0.Merging(), z12.VnH());
14842 FPSegmentPatternHelper(&masm, z13.VnH(), p0.Merging(), z13.VnH());
14845 FPSegmentPatternHelper(&masm, z14.VnH(), p0.Merging(), z14.VnH());
14848 FPSegmentPatternHelper(&masm, z15.VnH(), p0.Merging(), z15.VnH());
14852 FPSegmentPatternHelper(&masm, z16.VnH(), p0.Merging(), z16.VnH());
14855 FPSegmentPatternHelper(&masm, z17.VnH(), p0.Merging(), z17.VnH());
14858 FPSegmentPatternHelper(&masm, z18.VnH(), p0.Merging(), z18.VnH());
14861 FPSegmentPatternHelper(&masm, z19.VnH(), p0.Merging(), z19.VnH());
14865 FPSegmentPatternHelper(&masm, z20.VnH(), p0.Merging(), z20.VnH());
14868 FPSegmentPatternHelper(&masm, z21.VnH(), p0.Merging(), z21.VnH());
14910 InsrHelper(&masm, z0.VnD(), in_h0);
14911 InsrHelper(&masm, z1.VnD(), in_h1);
14912 InsrHelper(&masm, z2.VnD(), in_s0);
14913 InsrHelper(&masm, z3.VnD(), in_s1);
14914 InsrHelper(&masm, z4.VnD(), in_d0);
14915 InsrHelper(&masm, z5.VnD(), in_d1);
14981 static void BasicFPArithHelper(MacroAssembler* masm,
14989 InsrHelper(masm, z0.VnD(), inputs);
14992 masm->Rev(rvrs, z0.WithLaneSize(ls));
14995 Initialise(masm, p2.VnB(), pred);
14998 masm->Mov(z2, z0);
14999 masm->Fadd(z2.WithLaneSize(ls),
15004 masm->Mov(z3, z0);
15005 masm->Fsub(z3.WithLaneSize(ls), p2m, z3.WithLaneSize(ls), rvrs);
15006 masm->Mov(z4, z0);
15007 masm->Fsub(z4.WithLaneSize(ls), p2m, rvrs, z4.WithLaneSize(ls));
15008 masm->Mov(z5, z0);
15009 masm->Fabd(z5.WithLaneSize(ls),
15014 masm->Mov(z6, z0);
15015 masm->Fmul(z6.WithLaneSize(ls),
15022 InsrHelper(masm, z7.VnD(), inputs_fmulx);
15024 masm->Rev(z8.WithLaneSize(ls), z7.WithLaneSize(ls));
15025 masm->Fmulx(z7.WithLaneSize(ls),
15031 InsrHelper(masm, z8.VnD(), inputs_nans);
15032 masm->Mov(z9, z8);
15033 masm->Fminnm(z9.WithLaneSize(ls),
15038 masm->Mov(z10, z8);
15039 masm->Fmaxnm(z10.WithLaneSize(ls),
15054 BasicFPArithHelper(&masm, kHRegSize, inputs, inputs_fmulx, inputs_nans);
15087 BasicFPArithHelper(&masm, kSRegSize, inputs, inputs_fmulx, inputs_nans);
15120 BasicFPArithHelper(&masm, kDRegSize, inputs, inputs_fmulx, inputs_nans);
15150 Initialise(&masm, p0.VnB(), pred);
15286 InsrHelper(&masm, z0.VnD(), inputs_h);
15288 InsrHelper(&masm, z1.VnD(), inputs_s);
15290 InsrHelper(&masm, z2.VnD(), inputs_d);
15293 InsrHelper(&masm, z3.VnD(), scales);
15297 Initialise(&masm, p1.VnB(), pred);
15379 InsrHelper(&masm, zn.WithLaneSize(lane_size_in_bits), zn_rawbits);
15385 (masm.*macro_m)(zd_all_active.WithLaneSize(dst_type_size_in_bits),
15390 Initialise(&masm, pg_merging, pg_inputs);
15396 (masm.*macro_m)(zd_merging.WithLaneSize(dst_type_size_in_bits),
15402 Initialise(&masm, pg_zeroing, pg_inputs);
15406 (masm.*macro_z)(zd_zeroing.WithLaneSize(dst_type_size_in_bits),
15933 InsrHelper(&masm, zn.WithLaneSize(lane_size_in_bits), zn_inputs);
15950 Initialise(&masm, pg_merged, pg_inputs);
16669 InsrHelper(&masm, za, za_rawbits);
16670 InsrHelper(&masm, zn, zn_rawbits);
16671 InsrHelper(&masm, zm, zm_rawbits);
16678 Initialise(&masm, p0.WithLaneSize(lane_size_in_bits), pg_inputs);
16691 (masm.*macro)(da_result, p0.Merging(), da_result, zn, zm, nan_option);
16694 (masm.*macro)(dn_result, p0.Merging(), za, dn_result, zm, nan_option);
16697 (masm.*macro)(dm_result, p0.Merging(), za, zn, dm_result, nan_option);
16700 (masm.*macro)(d_result, p0.Merging(), za, zn, zm, nan_option);
16953 InsrHelper(&masm, z30.VnD(), zm_inputs);
16956 FPSegmentPatternHelper(&masm, z0.VnH(), p0.Merging(), z30.VnH());
16958 InsrHelper(&masm, z1.VnD(), zn_inputs);
16959 InsrHelper(&masm, z2.VnD(), za_inputs);
16962 (masm.*macro_idx)(z3.VnH(), z2.VnH(), z1.VnH(), z3.VnH(), 0); // zd == zm
16964 (masm.*macro_idx)(z4.VnH(), z2.VnH(), z4.VnH(), z0.VnH(), 1); // zd == zn
16966 (masm.*macro_idx)(z5.VnH(), z5.VnH(), z1.VnH(), z0.VnH(), 4); // zd == za
16967 (masm.*macro_idx)(z6.VnH(), z2.VnH(), z1.VnH(), z0.VnH(), 7);
16969 FPSegmentPatternHelper(&masm, z0.VnS(), p0.Merging(), z30.VnS());
16972 (masm.*macro_idx)(z7.VnS(), z2.VnS(), z1.VnS(), z7.VnS(), 0); // zd == zm
16974 (masm.*macro_idx)(z8.VnS(), z2.VnS(), z8.VnS(), z0.VnS(), 1); // zd == zn
16976 (masm.*macro_idx)(z9.VnS(), z9.VnS(), z1.VnS(), z0.VnS(), 2); // zd == za
16977 (masm.*macro_idx)(z10.VnS(), z2.VnS(), z1.VnS(), z0.VnS(), 3);
16979 FPSegmentPatternHelper(&masm, z0.VnD(), p0.Merging(), z30.VnD());
16982 (masm.*macro_idx)(z11.VnD(), z2.VnD(), z1.VnD(), z11.VnD(), 0); // zd == zm
16984 (masm.*macro_idx)(z12.VnD(), z2.VnD(), z12.VnD(), z0.VnD(), 1); // zd == zn
16986 (masm.*macro_idx)(z13.VnD(), z13.VnD(), z1.VnD(), z0.VnD(), 0); // zd == za
16989 (masm.*macro_idx)(z14.VnD(), z2.VnD(), z14.VnD(), z14.VnD(), 1);
16997 FPSegmentPatternHelper(&masm, z0.VnH(), p0.Merging(), z0.VnH());
16998 (masm.*macro)(z15.VnH(), p0.Merging(), z2.VnH(), z1.VnH(), z0.VnH(), option);
17000 FPSegmentPatternHelper(&masm, z0.VnH(), p0.Merging(), z0.VnH());
17001 (masm.*macro)(z16.VnH(), p0.Merging(), z2.VnH(), z1.VnH(), z0.VnH(), option);
17003 FPSegmentPatternHelper(&masm, z0.VnH(), p0.Merging(), z0.VnH());
17004 (masm.*macro)(z17.VnH(), p0.Merging(), z2.VnH(), z1.VnH(), z0.VnH(), option);
17006 FPSegmentPatternHelper(&masm, z0.VnH(), p0.Merging(), z0.VnH());
17007 (masm.*macro)(z18.VnH(), p0.Merging(), z2.VnH(), z1.VnH(), z0.VnH(), option);
17010 FPSegmentPatternHelper(&masm, z0.VnS(), p0.Merging(), z0.VnS());
17011 (masm.*macro)(z19.VnS(), p0.Merging(), z2.VnS(), z1.VnS(), z0.VnS(), option);
17013 FPSegmentPatternHelper(&masm, z0.VnS(), p0.Merging(), z0.VnS());
17014 (masm.*macro)(z20.VnS(), p0.Merging(), z2.VnS(), z1.VnS(), z0.VnS(), option);
17016 FPSegmentPatternHelper(&masm, z0.VnS(), p0.Merging(), z0.VnS());
17017 (masm.*macro)(z21.VnS(), p0.Merging(), z2.VnS(), z1.VnS(), z0.VnS(), option);
17019 FPSegmentPatternHelper(&masm, z0.VnS(), p0.Merging(), z0.VnS());
17020 (masm.*macro)(z22.VnS(), p0.Merging(), z2.VnS(), z1.VnS(), z0.VnS(), option);
17023 FPSegmentPatternHelper(&masm, z0.VnD(), p0.Merging(), z0.VnD());
17024 (masm.*macro)(z23.VnD(), p0.Merging(), z2.VnD(), z1.VnD(), z0.VnD(), option);
17026 FPSegmentPatternHelper(&masm, z0.VnD(), p0.Merging(), z0.VnD());
17027 (masm.*macro)(z24.VnD(), p0.Merging(), z2.VnD(), z1.VnD(), z0.VnD(), option);
17028 FPSegmentPatternHelper(&masm, z0.VnD(), p0.Merging(), z30.VnD());
17030 FPSegmentPatternHelper(&masm, z29.VnD(), p0.Merging(), z29.VnD());
17031 (masm.*macro)(z25.VnD(), p0.Merging(), z2.VnD(), z0.VnD(), z29.VnD(), option);
17807 Initialise(&masm, p0.WithLaneSize(lane_size_in_bits), pg_inputs);
17831 (masm.*cmp)(pd_result_int_1, p0.Zeroing(), zt_int_1, zt_int_2);
17832 (masm.*fcmp)(pd_result_fp_1, p0.Zeroing(), zt_fp_1, zt_fp_2);
17834 (masm.*cmp)(pd_result_int_2, p0.Zeroing(), zt_int_1, zt_int_3);
17835 (masm.*fcmp)(pd_result_fp_2, p0.Zeroing(), zt_fp_1, zt_fp_3);
17845 InsrHelper(&masm, zn_fp, zn_inputs_rawbits);
17846 InsrHelper(&masm, zm_fp, zm_inputs_rawbits);
17849 (masm.*fcmp)(pd_result_fp_3, p1.Zeroing(), zn_fp, zm_fp);
17920 InsrHelper(&masm, zn, zn_rawbits);
17923 (masm.*GetFpCompareZeroFn(cond))(pd, p0.Zeroing(), zn, 0.0);
18030 InsrHelper(&masm, z29.WithLaneSize(ls), zn_inputs_mod);
18032 InsrHelper(&masm, z29.WithLaneSize(ls), zn_inputs);
18040 (masm.*macro_m)(z27.WithLaneSize(ds), p0.Merging(), z28.WithLaneSize(ss));
18042 Initialise(&masm,
18058 (masm.*macro_m)(z11.WithLaneSize(ds), p1.Merging(), z28.WithLaneSize(ss));
18061 InsrHelper(&masm, z25.WithLaneSize(ls), zd_expected);
18066 (masm.*macro_z)(z12.WithLaneSize(ds), p1.Zeroing(), z28.WithLaneSize(ss));
18069 InsrHelper(&masm, z30.WithLaneSize(ls), zd_expected);
18076 (masm.*macro_m)(z9.WithLaneSize(ds), p1.Merging(), z9.WithLaneSize(ss));
18531 Initialise(&masm,
18541 (masm.*sve_ld1)(zn, pg, SVEMemOperand(x2, enablable_offset));
18550 ScalarLoadHelper(&masm,
18568 (masm.*sve_ld1)(zn_temp, pg, SVEMemOperand(x2, offsets[i]));
18573 ScalarLoadHelper(&masm, x1, x3, msize_in_bits, is_signed);
18811 InsrHelper(&masm, z31.VnS(), zn_inputs_s);
18812 InsrHelper(&masm, z30.VnS(), zm_inputs_s);
18876 InsrHelper(&masm, z0.VnS(), zn_inputs_s);
18877 InsrHelper(&masm, z1.VnS(), zm_inputs_s);
19204 InsrHelper(&masm, z0.VnS(), zn_inputs);
19205 InsrHelper(&masm, z1.VnS(), zm_inputs);
19206 InsrHelper(&masm, z31.VnS(), za_inputs);
19377 InsrHelper(&masm, z0.VnH(), zn_inputs_h);
19378 InsrHelper(&masm, z1.VnH(), zm_inputs_h);
19379 InsrHelper(&masm, z2.VnH(), za_inputs_h);
19383 InsrHelper(&masm, z3.VnS(), zn_inputs_s);
19384 InsrHelper(&masm, z4.VnS(), zm_inputs_s);
19385 InsrHelper(&masm, z5.VnS(), za_inputs_s);
19389 InsrHelper(&masm, z6.VnD(), zn_inputs_d);
19390 InsrHelper(&masm, z7.VnD(), zm_inputs_d);
19391 InsrHelper(&masm, z8.VnD(), za_inputs_d);
19417 InsrHelper(&masm, z31.VnS(), zn_inputs_s);
19418 InsrHelper(&masm, z30.VnS(), zm_inputs_s);
19420 InsrHelper(&masm, z0.VnS(), zda_inputs_s);
19424 InsrHelper(&masm, z1.VnS(), zda_inputs_s);
19428 InsrHelper(&masm, z2.VnS(), zda_inputs_s);
19432 InsrHelper(&masm, z3.VnS(), zda_inputs_s);
19483 InsrHelper(&masm, z31.VnS(), zn_bottom_inputs);
19484 InsrHelper(&masm, z30.VnS(), zm_top_inputs);
19554 InsrHelper(&masm, z31.VnH(), zn_inputs);
19555 InsrHelper(&masm, z30.VnH(), zm_inputs);
19556 InsrHelper(&masm, z29.VnS(), za_inputs);
19748 ExactAssemblyScope scope(&masm, 193 * kInstructionSize);