1// Copyright 2019, VIXL authors 2// All rights reserved. 3// 4// Redistribution and use in source and binary forms, with or without 5// modification, are permitted provided that the following conditions are met: 6// 7// * Redistributions of source code must retain the above copyright notice, 8// this list of conditions and the following disclaimer. 9// * Redistributions in binary form must reproduce the above copyright notice, 10// this list of conditions and the following disclaimer in the documentation 11// and/or other materials provided with the distribution. 12// * Neither the name of ARM Limited nor the names of its contributors may be 13// used to endorse or promote products derived from this software without 14// specific prior written permission. 15// 16// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS CONTRIBUTORS "AS IS" AND 17// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 20// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 23// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 27#include "assembler-aarch64.h" 28 29namespace vixl { 30namespace aarch64 { 31 32void Assembler::ResolveSVEImm8Shift(int* imm8, int* shift) { 33 if (*shift < 0) { 34 VIXL_ASSERT(*shift == -1); 35 // Derive the shift amount from the immediate. 36 if (IsInt8(*imm8)) { 37 *shift = 0; 38 } else if ((*imm8 % 256) == 0) { 39 *imm8 /= 256; 40 *shift = 8; 41 } 42 } 43 44 VIXL_ASSERT(IsInt8(*imm8)); 45 VIXL_ASSERT((*shift == 0) || (*shift == 8)); 46} 47 48// SVEAddressGeneration. 49 50void Assembler::adr(const ZRegister& zd, const SVEMemOperand& addr) { 51 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 52 VIXL_ASSERT(addr.IsVectorPlusVector()); 53 VIXL_ASSERT( 54 AreSameLaneSize(zd, addr.GetVectorBase(), addr.GetVectorOffset())); 55 56 int lane_size = zd.GetLaneSizeInBits(); 57 VIXL_ASSERT((lane_size == kSRegSize) || (lane_size == kDRegSize)); 58 59 int shift_amount = addr.GetShiftAmount(); 60 VIXL_ASSERT((shift_amount >= 0) && (shift_amount <= 3)); 61 62 Instr op = 0xffffffff; 63 Instr msz = shift_amount << 10; 64 SVEOffsetModifier mod = addr.GetOffsetModifier(); 65 switch (mod) { 66 case SVE_UXTW: 67 VIXL_ASSERT(lane_size == kDRegSize); 68 op = ADR_z_az_d_u32_scaled; 69 break; 70 case SVE_SXTW: 71 VIXL_ASSERT(lane_size == kDRegSize); 72 op = ADR_z_az_d_s32_scaled; 73 break; 74 case SVE_LSL: 75 case NO_SVE_OFFSET_MODIFIER: 76 op = (lane_size == kSRegSize) ? ADR_z_az_s_same_scaled 77 : ADR_z_az_d_same_scaled; 78 break; 79 default: 80 VIXL_UNIMPLEMENTED(); 81 } 82 Emit(op | msz | Rd(zd) | Rn(addr.GetVectorBase()) | 83 Rm(addr.GetVectorOffset())); 84} 85 86void Assembler::SVELogicalImmediate(const ZRegister& zdn, 87 uint64_t imm, 88 Instr op) { 89 unsigned bit_n, imm_s, imm_r; 90 unsigned lane_size = zdn.GetLaneSizeInBits(); 91 // Check that the immediate can be encoded in the instruction. 92 if (IsImmLogical(imm, lane_size, &bit_n, &imm_s, &imm_r)) { 93 Emit(op | Rd(zdn) | SVEBitN(bit_n) | SVEImmRotate(imm_r, lane_size) | 94 SVEImmSetBits(imm_s, lane_size)); 95 } else { 96 VIXL_UNREACHABLE(); 97 } 98} 99 100void Assembler::and_(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { 101 USE(zn); 102 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 103 VIXL_ASSERT(zd.Is(zn)); 104 SVELogicalImmediate(zd, imm, AND_z_zi); 105} 106 107void Assembler::dupm(const ZRegister& zd, uint64_t imm) { 108 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 109 // DUPM_z_i is an SVEBroadcastBitmaskImmOp, but its encoding and constraints 110 // are similar enough to SVEBitwiseLogicalWithImm_UnpredicatedOp, that we can 111 // use the logical immediate encoder to get the correct behaviour. 112 SVELogicalImmediate(zd, imm, DUPM_z_i); 113} 114 115void Assembler::eor(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { 116 USE(zn); 117 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 118 VIXL_ASSERT(zd.Is(zn)); 119 SVELogicalImmediate(zd, imm, EOR_z_zi); 120} 121 122void Assembler::orr(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { 123 USE(zn); 124 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 125 VIXL_ASSERT(zd.Is(zn)); 126 SVELogicalImmediate(zd, imm, ORR_z_zi); 127} 128 129// SVEBitwiseLogicalUnpredicated. 130void Assembler::and_(const ZRegister& zd, 131 const ZRegister& zn, 132 const ZRegister& zm) { 133 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 134 VIXL_ASSERT(zd.IsLaneSizeD()); 135 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 136 Emit(AND_z_zz | Rd(zd) | Rn(zn) | Rm(zm)); 137} 138 139void Assembler::bic(const ZRegister& zd, 140 const ZRegister& zn, 141 const ZRegister& zm) { 142 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 143 VIXL_ASSERT(zd.IsLaneSizeD()); 144 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 145 Emit(BIC_z_zz | Rd(zd) | Rn(zn) | Rm(zm)); 146} 147 148void Assembler::eor(const ZRegister& zd, 149 const ZRegister& zn, 150 const ZRegister& zm) { 151 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 152 VIXL_ASSERT(zd.IsLaneSizeD()); 153 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 154 Emit(EOR_z_zz | Rd(zd) | Rn(zn) | Rm(zm)); 155} 156 157void Assembler::orr(const ZRegister& zd, 158 const ZRegister& zn, 159 const ZRegister& zm) { 160 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 161 VIXL_ASSERT(zd.IsLaneSizeD()); 162 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 163 Emit(ORR_z_zz | Rd(zd) | Rn(zn) | Rm(zm)); 164} 165 166// SVEBitwiseShiftPredicated. 167 168void Assembler::SVEBitwiseShiftImmediatePred(const ZRegister& zdn, 169 const PRegisterM& pg, 170 Instr encoded_imm_and_tsz, 171 Instr op) { 172 Instr tszl_and_imm = ExtractUnsignedBitfield32(4, 0, encoded_imm_and_tsz) 173 << 5; 174 Instr tszh = ExtractUnsignedBitfield32(6, 5, encoded_imm_and_tsz) << 22; 175 Emit(op | tszh | tszl_and_imm | PgLow8(pg) | Rd(zdn)); 176} 177 178void Assembler::asr(const ZRegister& zd, 179 const PRegisterM& pg, 180 const ZRegister& zn, 181 int shift) { 182 // ASR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> 183 // 0000 0100 ..00 0000 100. .... .... .... 184 // tszh<23:22> | opc<19:18> = 00 | L<17> = 0 | U<16> = 0 | Pg<12:10> | 185 // tszl<9:8> | imm3<7:5> | Zdn<4:0> 186 187 USE(zn); 188 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 189 VIXL_ASSERT(zd.Is(zn)); 190 Instr encoded_imm = 191 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); 192 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, ASR_z_p_zi); 193} 194 195void Assembler::asr(const ZRegister& zd, 196 const PRegisterM& pg, 197 const ZRegister& zn, 198 const ZRegister& zm) { 199 // ASR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D 200 // 0000 0100 ..01 1000 100. .... .... .... 201 // size<23:22> | R<18> = 0 | L<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | 202 // Zdn<4:0> 203 204 USE(zn); 205 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 206 VIXL_ASSERT(zd.Is(zn)); 207 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) || 208 ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) && 209 (zd.GetLaneSizeInBytes() != kDRegSizeInBytes))); 210 Instr op = ASR_z_p_zw; 211 if (AreSameLaneSize(zd, zn, zm)) { 212 op = ASR_z_p_zz; 213 } 214 Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 215} 216 217void Assembler::asrd(const ZRegister& zd, 218 const PRegisterM& pg, 219 const ZRegister& zn, 220 int shift) { 221 // ASRD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> 222 // 0000 0100 ..00 0100 100. .... .... .... 223 // tszh<23:22> | opc<19:18> = 01 | L<17> = 0 | U<16> = 0 | Pg<12:10> | 224 // tszl<9:8> | imm3<7:5> | Zdn<4:0> 225 226 USE(zn); 227 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 228 VIXL_ASSERT(zd.Is(zn)); 229 230 Instr encoded_imm = 231 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); 232 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, ASRD_z_p_zi); 233} 234 235void Assembler::asrr(const ZRegister& zd, 236 const PRegisterM& pg, 237 const ZRegister& zn, 238 const ZRegister& zm) { 239 // ASRR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 240 // 0000 0100 ..01 0100 100. .... .... .... 241 // size<23:22> | R<18> = 1 | L<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | 242 // Zdn<4:0> 243 244 USE(zn); 245 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 246 VIXL_ASSERT(zd.Is(zn)); 247 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 248 249 Emit(ASRR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 250} 251 252void Assembler::lsl(const ZRegister& zd, 253 const PRegisterM& pg, 254 const ZRegister& zn, 255 int shift) { 256 // LSL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> 257 // 0000 0100 ..00 0011 100. .... .... .... 258 // tszh<23:22> | opc<19:18> = 00 | L<17> = 1 | U<16> = 1 | Pg<12:10> | 259 // tszl<9:8> | imm3<7:5> | Zdn<4:0> 260 261 USE(zn); 262 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 263 VIXL_ASSERT(zd.Is(zn)); 264 265 Instr encoded_imm = 266 EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); 267 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, LSL_z_p_zi); 268} 269 270void Assembler::lsl(const ZRegister& zd, 271 const PRegisterM& pg, 272 const ZRegister& zn, 273 const ZRegister& zm) { 274 // LSL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D 275 // 0000 0100 ..01 1011 100. .... .... .... 276 // size<23:22> | R<18> = 0 | L<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | 277 // Zdn<4:0> 278 279 USE(zn); 280 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 281 VIXL_ASSERT(zd.Is(zn)); 282 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) || 283 ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) && 284 (zd.GetLaneSizeInBytes() != kDRegSizeInBytes))); 285 Instr op = LSL_z_p_zw; 286 if (AreSameLaneSize(zd, zn, zm)) { 287 op = LSL_z_p_zz; 288 } 289 Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 290} 291 292void Assembler::lslr(const ZRegister& zd, 293 const PRegisterM& pg, 294 const ZRegister& zn, 295 const ZRegister& zm) { 296 // LSLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 297 // 0000 0100 ..01 0111 100. .... .... .... 298 // size<23:22> | R<18> = 1 | L<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | 299 // Zdn<4:0> 300 301 USE(zn); 302 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 303 VIXL_ASSERT(zd.Is(zn)); 304 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 305 306 Emit(LSLR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 307} 308 309void Assembler::lsr(const ZRegister& zd, 310 const PRegisterM& pg, 311 const ZRegister& zn, 312 int shift) { 313 // LSR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> 314 // 0000 0100 ..00 0001 100. .... .... .... 315 // tszh<23:22> | opc<19:18> = 00 | L<17> = 0 | U<16> = 1 | Pg<12:10> | 316 // tszl<9:8> | imm3<7:5> | Zdn<4:0> 317 318 USE(zn); 319 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 320 VIXL_ASSERT(zd.Is(zn)); 321 322 Instr encoded_imm = 323 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); 324 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, LSR_z_p_zi); 325} 326 327void Assembler::lsr(const ZRegister& zd, 328 const PRegisterM& pg, 329 const ZRegister& zn, 330 const ZRegister& zm) { 331 // LSR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.D 332 // 0000 0100 ..01 1001 100. .... .... .... 333 // size<23:22> | R<18> = 0 | L<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> | 334 // Zdn<4:0> 335 336 USE(zn); 337 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 338 VIXL_ASSERT(zd.Is(zn)); 339 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm) || 340 ((zm.GetLaneSizeInBytes() == kDRegSizeInBytes) && 341 (zd.GetLaneSizeInBytes() != kDRegSizeInBytes))); 342 Instr op = LSR_z_p_zw; 343 if (AreSameLaneSize(zd, zn, zm)) { 344 op = LSR_z_p_zz; 345 } 346 Emit(op | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 347} 348 349void Assembler::lsrr(const ZRegister& zd, 350 const PRegisterM& pg, 351 const ZRegister& zn, 352 const ZRegister& zm) { 353 // LSRR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 354 // 0000 0100 ..01 0101 100. .... .... .... 355 // size<23:22> | R<18> = 1 | L<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> | 356 // Zdn<4:0> 357 358 USE(zn); 359 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 360 VIXL_ASSERT(zd.Is(zn)); 361 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 362 363 Emit(LSRR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 364} 365 366// SVEBitwiseShiftUnpredicated. 367 368Instr Assembler::EncodeSVEShiftLeftImmediate(int shift, int lane_size_in_bits) { 369 VIXL_ASSERT((shift >= 0) && (shift < lane_size_in_bits)); 370 return lane_size_in_bits + shift; 371} 372 373Instr Assembler::EncodeSVEShiftRightImmediate(int shift, 374 int lane_size_in_bits) { 375 VIXL_ASSERT((shift > 0) && (shift <= lane_size_in_bits)); 376 return (2 * lane_size_in_bits) - shift; 377} 378 379void Assembler::SVEBitwiseShiftImmediate(const ZRegister& zd, 380 const ZRegister& zn, 381 Instr encoded_imm_and_tsz, 382 Instr op) { 383 Instr tszl_and_imm = ExtractUnsignedBitfield32(4, 0, encoded_imm_and_tsz) 384 << 16; 385 Instr tszh = ExtractUnsignedBitfield32(6, 5, encoded_imm_and_tsz) << 22; 386 Emit(op | tszh | tszl_and_imm | Rd(zd) | Rn(zn)); 387} 388 389void Assembler::asr(const ZRegister& zd, const ZRegister& zn, int shift) { 390 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 391 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 392 Instr encoded_imm = 393 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); 394 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, ASR_z_zi); 395} 396 397void Assembler::asr(const ZRegister& zd, 398 const ZRegister& zn, 399 const ZRegister& zm) { 400 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 401 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 402 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes); 403 404 Emit(ASR_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 405} 406 407void Assembler::lsl(const ZRegister& zd, const ZRegister& zn, int shift) { 408 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 409 Instr encoded_imm = 410 EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); 411 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, LSL_z_zi); 412} 413 414void Assembler::lsl(const ZRegister& zd, 415 const ZRegister& zn, 416 const ZRegister& zm) { 417 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 418 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 419 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes); 420 421 Emit(LSL_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 422} 423 424void Assembler::lsr(const ZRegister& zd, const ZRegister& zn, int shift) { 425 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 426 Instr encoded_imm = 427 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); 428 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, LSR_z_zi); 429} 430 431void Assembler::lsr(const ZRegister& zd, 432 const ZRegister& zn, 433 const ZRegister& zm) { 434 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 435 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 436 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kDRegSizeInBytes); 437 438 Emit(LSR_z_zw | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 439} 440 441// SVEElementCount. 442 443#define VIXL_SVE_INC_DEC_LIST(V) \ 444 V(cntb, CNTB_r_s) \ 445 V(cnth, CNTH_r_s) \ 446 V(cntw, CNTW_r_s) \ 447 V(cntd, CNTD_r_s) \ 448 V(decb, DECB_r_rs) \ 449 V(dech, DECH_r_rs) \ 450 V(decw, DECW_r_rs) \ 451 V(decd, DECD_r_rs) \ 452 V(incb, INCB_r_rs) \ 453 V(inch, INCH_r_rs) \ 454 V(incw, INCW_r_rs) \ 455 V(incd, INCD_r_rs) \ 456 V(sqdecb, SQDECB_r_rs_x) \ 457 V(sqdech, SQDECH_r_rs_x) \ 458 V(sqdecw, SQDECW_r_rs_x) \ 459 V(sqdecd, SQDECD_r_rs_x) \ 460 V(sqincb, SQINCB_r_rs_x) \ 461 V(sqinch, SQINCH_r_rs_x) \ 462 V(sqincw, SQINCW_r_rs_x) \ 463 V(sqincd, SQINCD_r_rs_x) 464 465#define VIXL_DEFINE_ASM_FUNC(FN, OP) \ 466 void Assembler::FN(const Register& rdn, int pattern, int multiplier) { \ 467 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 468 VIXL_ASSERT(rdn.IsX()); \ 469 Emit(OP | Rd(rdn) | ImmSVEPredicateConstraint(pattern) | \ 470 ImmUnsignedField<19, 16>(multiplier - 1)); \ 471 } 472VIXL_SVE_INC_DEC_LIST(VIXL_DEFINE_ASM_FUNC) 473#undef VIXL_DEFINE_ASM_FUNC 474 475#define VIXL_SVE_UQINC_UQDEC_LIST(V) \ 476 V(uqdecb, (rdn.IsX() ? UQDECB_r_rs_x : UQDECB_r_rs_uw)) \ 477 V(uqdech, (rdn.IsX() ? UQDECH_r_rs_x : UQDECH_r_rs_uw)) \ 478 V(uqdecw, (rdn.IsX() ? UQDECW_r_rs_x : UQDECW_r_rs_uw)) \ 479 V(uqdecd, (rdn.IsX() ? UQDECD_r_rs_x : UQDECD_r_rs_uw)) \ 480 V(uqincb, (rdn.IsX() ? UQINCB_r_rs_x : UQINCB_r_rs_uw)) \ 481 V(uqinch, (rdn.IsX() ? UQINCH_r_rs_x : UQINCH_r_rs_uw)) \ 482 V(uqincw, (rdn.IsX() ? UQINCW_r_rs_x : UQINCW_r_rs_uw)) \ 483 V(uqincd, (rdn.IsX() ? UQINCD_r_rs_x : UQINCD_r_rs_uw)) 484 485#define VIXL_DEFINE_ASM_FUNC(FN, OP) \ 486 void Assembler::FN(const Register& rdn, int pattern, int multiplier) { \ 487 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 488 Emit(OP | Rd(rdn) | ImmSVEPredicateConstraint(pattern) | \ 489 ImmUnsignedField<19, 16>(multiplier - 1)); \ 490 } 491VIXL_SVE_UQINC_UQDEC_LIST(VIXL_DEFINE_ASM_FUNC) 492#undef VIXL_DEFINE_ASM_FUNC 493 494#define VIXL_SVE_SQX_INC_DEC_LIST(V) \ 495 V(sqdecb, SQDECB) \ 496 V(sqdech, SQDECH) \ 497 V(sqdecw, SQDECW) \ 498 V(sqdecd, SQDECD) \ 499 V(sqincb, SQINCB) \ 500 V(sqinch, SQINCH) \ 501 V(sqincw, SQINCW) \ 502 V(sqincd, SQINCD) 503 504#define VIXL_DEFINE_ASM_FUNC(FN, OP) \ 505 void Assembler::FN(const Register& xd, \ 506 const Register& wn, \ 507 int pattern, \ 508 int multiplier) { \ 509 USE(wn); \ 510 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 511 VIXL_ASSERT(wn.IsW() && xd.Is(wn.X())); \ 512 Emit(OP##_r_rs_sx | Rd(xd) | ImmSVEPredicateConstraint(pattern) | \ 513 ImmUnsignedField<19, 16>(multiplier - 1)); \ 514 } 515VIXL_SVE_SQX_INC_DEC_LIST(VIXL_DEFINE_ASM_FUNC) 516#undef VIXL_DEFINE_ASM_FUNC 517 518#define VIXL_SVE_INC_DEC_VEC_LIST(V) \ 519 V(dech, DEC, H) \ 520 V(decw, DEC, W) \ 521 V(decd, DEC, D) \ 522 V(inch, INC, H) \ 523 V(incw, INC, W) \ 524 V(incd, INC, D) \ 525 V(sqdech, SQDEC, H) \ 526 V(sqdecw, SQDEC, W) \ 527 V(sqdecd, SQDEC, D) \ 528 V(sqinch, SQINC, H) \ 529 V(sqincw, SQINC, W) \ 530 V(sqincd, SQINC, D) \ 531 V(uqdech, UQDEC, H) \ 532 V(uqdecw, UQDEC, W) \ 533 V(uqdecd, UQDEC, D) \ 534 V(uqinch, UQINC, H) \ 535 V(uqincw, UQINC, W) \ 536 V(uqincd, UQINC, D) 537 538#define VIXL_DEFINE_ASM_FUNC(FN, OP, T) \ 539 void Assembler::FN(const ZRegister& zdn, int pattern, int multiplier) { \ 540 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 541 VIXL_ASSERT(zdn.GetLaneSizeInBytes() == k##T##RegSizeInBytes); \ 542 Emit(OP##T##_z_zs | Rd(zdn) | ImmSVEPredicateConstraint(pattern) | \ 543 ImmUnsignedField<19, 16>(multiplier - 1)); \ 544 } 545VIXL_SVE_INC_DEC_VEC_LIST(VIXL_DEFINE_ASM_FUNC) 546#undef VIXL_DEFINE_ASM_FUNC 547 548// SVEFPAccumulatingReduction. 549 550void Assembler::fadda(const VRegister& vd, 551 const PRegister& pg, 552 const VRegister& vn, 553 const ZRegister& zm) { 554 // FADDA <V><dn>, <Pg>, <V><dn>, <Zm>.<T> 555 // 0110 0101 ..01 1000 001. .... .... .... 556 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Vdn<4:0> 557 558 USE(vn); 559 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 560 VIXL_ASSERT(vd.Is(vn)); 561 VIXL_ASSERT(vd.IsScalar()); 562 VIXL_ASSERT(zm.GetLaneSizeInBytes() != kBRegSizeInBytes); 563 VIXL_ASSERT(AreSameLaneSize(zm, vd)); 564 565 Emit(FADDA_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm)); 566} 567 568// SVEFPArithmetic_Predicated. 569 570void Assembler::fabd(const ZRegister& zd, 571 const PRegisterM& pg, 572 const ZRegister& zn, 573 const ZRegister& zm) { 574 // FABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 575 // 0110 0101 ..00 1000 100. .... .... .... 576 // size<23:22> | opc<19:16> = 1000 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 577 578 USE(zn); 579 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 580 VIXL_ASSERT(zd.Is(zn)); 581 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 582 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 583 584 Emit(FABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 585} 586 587void Assembler::fadd(const ZRegister& zd, 588 const PRegisterM& pg, 589 const ZRegister& zn, 590 double imm) { 591 // FADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> 592 // 0110 0101 ..01 1000 100. ..00 00.. .... 593 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | i1<5> | Zdn<4:0> 594 595 USE(zn); 596 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 597 VIXL_ASSERT(zd.Is(zn)); 598 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 599 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 600 VIXL_ASSERT((imm == 0.5) || (imm == 1.0)); 601 602 Instr i1 = (imm == 1.0) ? (1 << 5) : 0; 603 Emit(FADD_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); 604} 605 606void Assembler::fadd(const ZRegister& zd, 607 const PRegisterM& pg, 608 const ZRegister& zn, 609 const ZRegister& zm) { 610 // FADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 611 // 0110 0101 ..00 0000 100. .... .... .... 612 // size<23:22> | opc<19:16> = 0000 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 613 614 USE(zn); 615 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 616 VIXL_ASSERT(zd.Is(zn)); 617 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 618 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 619 620 Emit(FADD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 621} 622 623void Assembler::fdiv(const ZRegister& zd, 624 const PRegisterM& pg, 625 const ZRegister& zn, 626 const ZRegister& zm) { 627 // FDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 628 // 0110 0101 ..00 1101 100. .... .... .... 629 // size<23:22> | opc<19:16> = 1101 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 630 631 USE(zn); 632 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 633 VIXL_ASSERT(zd.Is(zn)); 634 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 635 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 636 637 Emit(FDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 638} 639 640void Assembler::fdivr(const ZRegister& zd, 641 const PRegisterM& pg, 642 const ZRegister& zn, 643 const ZRegister& zm) { 644 // FDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 645 // 0110 0101 ..00 1100 100. .... .... .... 646 // size<23:22> | opc<19:16> = 1100 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 647 648 USE(zn); 649 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 650 VIXL_ASSERT(zd.Is(zn)); 651 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 652 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 653 654 Emit(FDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 655} 656 657void Assembler::fmax(const ZRegister& zd, 658 const PRegisterM& pg, 659 const ZRegister& zn, 660 double imm) { 661 // FMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> 662 // 0110 0101 ..01 1110 100. ..00 00.. .... 663 // size<23:22> | opc<18:16> = 110 | Pg<12:10> | i1<5> | Zdn<4:0> 664 665 USE(zn); 666 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 667 VIXL_ASSERT(zd.Is(zn)); 668 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 669 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 670 VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0)); 671 672 Instr i1 = (imm == 1.0) ? (1 << 5) : 0; 673 Emit(FMAX_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); 674} 675 676void Assembler::fmax(const ZRegister& zd, 677 const PRegisterM& pg, 678 const ZRegister& zn, 679 const ZRegister& zm) { 680 // FMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 681 // 0110 0101 ..00 0110 100. .... .... .... 682 // size<23:22> | opc<19:16> = 0110 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 683 684 USE(zn); 685 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 686 VIXL_ASSERT(zd.Is(zn)); 687 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 688 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 689 690 Emit(FMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 691} 692 693void Assembler::fmaxnm(const ZRegister& zd, 694 const PRegisterM& pg, 695 const ZRegister& zn, 696 double imm) { 697 // FMAXNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> 698 // 0110 0101 ..01 1100 100. ..00 00.. .... 699 // size<23:22> | opc<18:16> = 100 | Pg<12:10> | i1<5> | Zdn<4:0> 700 701 USE(zn); 702 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 703 VIXL_ASSERT(zd.Is(zn)); 704 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 705 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 706 VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0)); 707 708 Instr i1 = (imm == 1.0) ? (1 << 5) : 0; 709 Emit(FMAXNM_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); 710} 711 712void Assembler::fmaxnm(const ZRegister& zd, 713 const PRegisterM& pg, 714 const ZRegister& zn, 715 const ZRegister& zm) { 716 // FMAXNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 717 // 0110 0101 ..00 0100 100. .... .... .... 718 // size<23:22> | opc<19:16> = 0100 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 719 720 USE(zn); 721 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 722 VIXL_ASSERT(zd.Is(zn)); 723 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 724 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 725 726 Emit(FMAXNM_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 727} 728 729void Assembler::fmin(const ZRegister& zd, 730 const PRegisterM& pg, 731 const ZRegister& zn, 732 double imm) { 733 // FMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> 734 // 0110 0101 ..01 1111 100. ..00 00.. .... 735 // size<23:22> | opc<18:16> = 111 | Pg<12:10> | i1<5> | Zdn<4:0> 736 737 USE(zn); 738 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 739 VIXL_ASSERT(zd.Is(zn)); 740 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 741 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 742 VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0)); 743 744 Instr i1 = (imm == 1.0) ? (1 << 5) : 0; 745 Emit(FMIN_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); 746} 747 748void Assembler::fmin(const ZRegister& zd, 749 const PRegisterM& pg, 750 const ZRegister& zn, 751 const ZRegister& zm) { 752 // FMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 753 // 0110 0101 ..00 0111 100. .... .... .... 754 // size<23:22> | opc<19:16> = 0111 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 755 756 USE(zn); 757 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 758 VIXL_ASSERT(zd.Is(zn)); 759 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 760 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 761 762 Emit(FMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 763} 764 765void Assembler::fminnm(const ZRegister& zd, 766 const PRegisterM& pg, 767 const ZRegister& zn, 768 double imm) { 769 // FMINNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> 770 // 0110 0101 ..01 1101 100. ..00 00.. .... 771 // size<23:22> | opc<18:16> = 101 | Pg<12:10> | i1<5> | Zdn<4:0> 772 773 USE(zn); 774 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 775 VIXL_ASSERT(zd.Is(zn)); 776 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 777 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 778 VIXL_ASSERT(((imm == 0.0) && (copysign(1.0, imm) == 1.0)) || (imm == 1.0)); 779 780 Instr i1 = (imm == 1.0) ? (1 << 5) : 0; 781 Emit(FMINNM_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); 782} 783 784void Assembler::fminnm(const ZRegister& zd, 785 const PRegisterM& pg, 786 const ZRegister& zn, 787 const ZRegister& zm) { 788 // FMINNM <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 789 // 0110 0101 ..00 0101 100. .... .... .... 790 // size<23:22> | opc<19:16> = 0101 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 791 792 USE(zn); 793 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 794 VIXL_ASSERT(zd.Is(zn)); 795 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 796 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 797 798 Emit(FMINNM_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 799} 800 801void Assembler::fmul(const ZRegister& zd, 802 const PRegisterM& pg, 803 const ZRegister& zn, 804 double imm) { 805 // FMUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> 806 // 0110 0101 ..01 1010 100. ..00 00.. .... 807 // size<23:22> | opc<18:16> = 010 | Pg<12:10> | i1<5> | Zdn<4:0> 808 809 USE(zn); 810 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 811 VIXL_ASSERT(zd.Is(zn)); 812 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 813 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 814 VIXL_ASSERT((imm == 0.5) || (imm == 2.0)); 815 816 Instr i1 = (imm == 2.0) ? (1 << 5) : 0; 817 Emit(FMUL_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); 818} 819 820void Assembler::fmul(const ZRegister& zd, 821 const PRegisterM& pg, 822 const ZRegister& zn, 823 const ZRegister& zm) { 824 // FMUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 825 // 0110 0101 ..00 0010 100. .... .... .... 826 // size<23:22> | opc<19:16> = 0010 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 827 828 USE(zn); 829 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 830 VIXL_ASSERT(zd.Is(zn)); 831 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 832 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 833 834 Emit(FMUL_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 835} 836 837void Assembler::fmulx(const ZRegister& zd, 838 const PRegisterM& pg, 839 const ZRegister& zn, 840 const ZRegister& zm) { 841 // FMULX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 842 // 0110 0101 ..00 1010 100. .... .... .... 843 // size<23:22> | opc<19:16> = 1010 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 844 845 USE(zn); 846 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 847 VIXL_ASSERT(zd.Is(zn)); 848 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 849 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 850 851 Emit(FMULX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 852} 853 854void Assembler::fscale(const ZRegister& zd, 855 const PRegisterM& pg, 856 const ZRegister& zn, 857 const ZRegister& zm) { 858 // FSCALE <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 859 // 0110 0101 ..00 1001 100. .... .... .... 860 // size<23:22> | opc<19:16> = 1001 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 861 862 USE(zn); 863 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 864 VIXL_ASSERT(zd.Is(zn)); 865 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 866 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 867 868 Emit(FSCALE_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 869} 870 871void Assembler::fsub(const ZRegister& zd, 872 const PRegisterM& pg, 873 const ZRegister& zn, 874 double imm) { 875 // FSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> 876 // 0110 0101 ..01 1001 100. ..00 00.. .... 877 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | i1<5> | Zdn<4:0> 878 879 USE(zn); 880 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 881 VIXL_ASSERT(zd.Is(zn)); 882 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 883 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 884 VIXL_ASSERT((imm == 0.5) || (imm == 1.0)); 885 886 Instr i1 = (imm == 1.0) ? (1 << 5) : 0; 887 Emit(FSUB_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); 888} 889 890void Assembler::fsub(const ZRegister& zd, 891 const PRegisterM& pg, 892 const ZRegister& zn, 893 const ZRegister& zm) { 894 // FSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 895 // 0110 0101 ..00 0001 100. .... .... .... 896 // size<23:22> | opc<19:16> = 0001 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 897 898 USE(zn); 899 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 900 VIXL_ASSERT(zd.Is(zn)); 901 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 902 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 903 904 Emit(FSUB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 905} 906 907void Assembler::fsubr(const ZRegister& zd, 908 const PRegisterM& pg, 909 const ZRegister& zn, 910 double imm) { 911 // FSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <const> 912 // 0110 0101 ..01 1011 100. ..00 00.. .... 913 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | i1<5> | Zdn<4:0> 914 915 USE(zn); 916 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 917 VIXL_ASSERT(zd.Is(zn)); 918 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 919 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 920 VIXL_ASSERT((imm == 0.5) || (imm == 1.0)); 921 922 Instr i1 = (imm == 1.0) ? (1 << 5) : 0; 923 Emit(FSUBR_z_p_zs | SVESize(zd) | Rd(zd) | PgLow8(pg) | i1); 924} 925 926void Assembler::fsubr(const ZRegister& zd, 927 const PRegisterM& pg, 928 const ZRegister& zn, 929 const ZRegister& zm) { 930 // FSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 931 // 0110 0101 ..00 0011 100. .... .... .... 932 // size<23:22> | opc<19:16> = 0011 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 933 934 USE(zn); 935 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 936 VIXL_ASSERT(zd.Is(zn)); 937 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 938 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 939 940 Emit(FSUBR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 941} 942 943void Assembler::ftmad(const ZRegister& zd, 944 const ZRegister& zn, 945 const ZRegister& zm, 946 int imm3) { 947 // FTMAD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, #<imm> 948 // 0110 0101 ..01 0... 1000 00.. .... .... 949 // size<23:22> | imm3<18:16> | Zm<9:5> | Zdn<4:0> 950 951 USE(zn); 952 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 953 VIXL_ASSERT(zd.Is(zn)); 954 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 955 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 956 957 Emit(FTMAD_z_zzi | SVESize(zd) | Rd(zd) | Rn(zm) | 958 ImmUnsignedField<18, 16>(imm3)); 959} 960 961// SVEFPArithmeticUnpredicated. 962 963void Assembler::fadd(const ZRegister& zd, 964 const ZRegister& zn, 965 const ZRegister& zm) { 966 // FADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 967 // 0110 0101 ..0. .... 0000 00.. .... .... 968 // size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0> 969 970 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 971 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 972 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 973 974 Emit(FADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 975} 976 977void Assembler::fmul(const ZRegister& zd, 978 const ZRegister& zn, 979 const ZRegister& zm) { 980 // FMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 981 // 0110 0101 ..0. .... 0000 10.. .... .... 982 // size<23:22> | Zm<20:16> | opc<12:10> = 010 | Zn<9:5> | Zd<4:0> 983 984 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 985 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 986 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 987 988 Emit(FMUL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 989} 990 991void Assembler::frecps(const ZRegister& zd, 992 const ZRegister& zn, 993 const ZRegister& zm) { 994 // FRECPS <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 995 // 0110 0101 ..0. .... 0001 10.. .... .... 996 // size<23:22> | Zm<20:16> | opc<12:10> = 110 | Zn<9:5> | Zd<4:0> 997 998 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 999 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 1000 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1001 1002 Emit(FRECPS_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 1003} 1004 1005void Assembler::frsqrts(const ZRegister& zd, 1006 const ZRegister& zn, 1007 const ZRegister& zm) { 1008 // FRSQRTS <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 1009 // 0110 0101 ..0. .... 0001 11.. .... .... 1010 // size<23:22> | Zm<20:16> | opc<12:10> = 111 | Zn<9:5> | Zd<4:0> 1011 1012 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1013 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 1014 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1015 1016 Emit(FRSQRTS_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 1017} 1018 1019void Assembler::fsub(const ZRegister& zd, 1020 const ZRegister& zn, 1021 const ZRegister& zm) { 1022 // FSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 1023 // 0110 0101 ..0. .... 0000 01.. .... .... 1024 // size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0> 1025 1026 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1027 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 1028 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1029 1030 Emit(FSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 1031} 1032 1033void Assembler::ftsmul(const ZRegister& zd, 1034 const ZRegister& zn, 1035 const ZRegister& zm) { 1036 // FTSMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 1037 // 0110 0101 ..0. .... 0000 11.. .... .... 1038 // size<23:22> | Zm<20:16> | opc<12:10> = 011 | Zn<9:5> | Zd<4:0> 1039 1040 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1041 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 1042 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1043 1044 Emit(FTSMUL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 1045} 1046 1047// SVEFPCompareVectors. 1048 1049void Assembler::facge(const PRegisterWithLaneSize& pd, 1050 const PRegisterZ& pg, 1051 const ZRegister& zn, 1052 const ZRegister& zm) { 1053 // FACGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 1054 // 0110 0101 ..0. .... 110. .... ...1 .... 1055 // size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> | 1056 // o3<4> = 1 | Pd<3:0> 1057 1058 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1059 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 1060 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1061 1062 Emit(FACGE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1063} 1064 1065void Assembler::facgt(const PRegisterWithLaneSize& pd, 1066 const PRegisterZ& pg, 1067 const ZRegister& zn, 1068 const ZRegister& zm) { 1069 // FACGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 1070 // 0110 0101 ..0. .... 111. .... ...1 .... 1071 // size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 1 | Pg<12:10> | Zn<9:5> | 1072 // o3<4> = 1 | Pd<3:0> 1073 1074 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1075 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 1076 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1077 1078 Emit(FACGT_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1079} 1080 1081void Assembler::fcmeq(const PRegisterWithLaneSize& pd, 1082 const PRegisterZ& pg, 1083 const ZRegister& zn, 1084 const ZRegister& zm) { 1085 // FCMEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 1086 // 0110 0101 ..0. .... 011. .... ...0 .... 1087 // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> | 1088 // o3<4> = 0 | Pd<3:0> 1089 1090 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1091 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 1092 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1093 1094 Emit(FCMEQ_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1095} 1096 1097void Assembler::fcmge(const PRegisterWithLaneSize& pd, 1098 const PRegisterZ& pg, 1099 const ZRegister& zn, 1100 const ZRegister& zm) { 1101 // FCMGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 1102 // 0110 0101 ..0. .... 010. .... ...0 .... 1103 // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> | 1104 // o3<4> = 0 | Pd<3:0> 1105 1106 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1107 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 1108 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1109 1110 Emit(FCMGE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1111} 1112 1113void Assembler::fcmgt(const PRegisterWithLaneSize& pd, 1114 const PRegisterZ& pg, 1115 const ZRegister& zn, 1116 const ZRegister& zm) { 1117 // FCMGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 1118 // 0110 0101 ..0. .... 010. .... ...1 .... 1119 // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> | 1120 // o3<4> = 1 | Pd<3:0> 1121 1122 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1123 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 1124 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1125 1126 Emit(FCMGT_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1127} 1128 1129void Assembler::fcmne(const PRegisterWithLaneSize& pd, 1130 const PRegisterZ& pg, 1131 const ZRegister& zn, 1132 const ZRegister& zm) { 1133 // FCMNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 1134 // 0110 0101 ..0. .... 011. .... ...1 .... 1135 // size<23:22> | Zm<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> | 1136 // o3<4> = 1 | Pd<3:0> 1137 1138 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1139 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 1140 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1141 1142 Emit(FCMNE_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1143} 1144 1145void Assembler::fcmuo(const PRegisterWithLaneSize& pd, 1146 const PRegisterZ& pg, 1147 const ZRegister& zn, 1148 const ZRegister& zm) { 1149 // FCMUO <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 1150 // 0110 0101 ..0. .... 110. .... ...0 .... 1151 // size<23:22> | Zm<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> | 1152 // o3<4> = 0 | Pd<3:0> 1153 1154 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1155 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 1156 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1157 1158 Emit(FCMUO_p_p_zz | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1159} 1160 1161// SVEFPCompareWithZero. 1162 1163void Assembler::fcmeq(const PRegisterWithLaneSize& pd, 1164 const PRegisterZ& pg, 1165 const ZRegister& zn, 1166 double zero) { 1167 // FCMEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 1168 // 0110 0101 ..01 0010 001. .... ...0 .... 1169 // size<23:22> | eq<17> = 1 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | 1170 // Pd<3:0> 1171 1172 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1173 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1174 VIXL_ASSERT(zero == 0.0); 1175 USE(zero); 1176 1177 Emit(FCMEQ_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); 1178} 1179 1180void Assembler::fcmge(const PRegisterWithLaneSize& pd, 1181 const PRegisterZ& pg, 1182 const ZRegister& zn, 1183 double zero) { 1184 // FCMGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 1185 // 0110 0101 ..01 0000 001. .... ...0 .... 1186 // size<23:22> | eq<17> = 0 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | 1187 // Pd<3:0> 1188 1189 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1190 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1191 VIXL_ASSERT(zero == 0.0); 1192 USE(zero); 1193 1194 Emit(FCMGE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); 1195} 1196 1197void Assembler::fcmgt(const PRegisterWithLaneSize& pd, 1198 const PRegisterZ& pg, 1199 const ZRegister& zn, 1200 double zero) { 1201 // FCMGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 1202 // 0110 0101 ..01 0000 001. .... ...1 .... 1203 // size<23:22> | eq<17> = 0 | lt<16> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 1 | 1204 // Pd<3:0> 1205 1206 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1207 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1208 VIXL_ASSERT(zero == 0.0); 1209 USE(zero); 1210 1211 Emit(FCMGT_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); 1212} 1213 1214void Assembler::fcmle(const PRegisterWithLaneSize& pd, 1215 const PRegisterZ& pg, 1216 const ZRegister& zn, 1217 double zero) { 1218 // FCMLE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 1219 // 0110 0101 ..01 0001 001. .... ...1 .... 1220 // size<23:22> | eq<17> = 0 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 1 | 1221 // Pd<3:0> 1222 1223 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1224 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1225 VIXL_ASSERT(zero == 0.0); 1226 USE(zero); 1227 1228 Emit(FCMLE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); 1229} 1230 1231void Assembler::fcmlt(const PRegisterWithLaneSize& pd, 1232 const PRegisterZ& pg, 1233 const ZRegister& zn, 1234 double zero) { 1235 // FCMLT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 1236 // 0110 0101 ..01 0001 001. .... ...0 .... 1237 // size<23:22> | eq<17> = 0 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | 1238 // Pd<3:0> 1239 1240 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1241 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1242 VIXL_ASSERT(zero == 0.0); 1243 USE(zero); 1244 1245 Emit(FCMLT_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); 1246} 1247 1248void Assembler::fcmne(const PRegisterWithLaneSize& pd, 1249 const PRegisterZ& pg, 1250 const ZRegister& zn, 1251 double zero) { 1252 // FCMNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #0.0 1253 // 0110 0101 ..01 0011 001. .... ...0 .... 1254 // size<23:22> | eq<17> = 1 | lt<16> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | 1255 // Pd<3:0> 1256 1257 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1258 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1259 VIXL_ASSERT(zero == 0.0); 1260 USE(zero); 1261 1262 Emit(FCMNE_p_p_z0 | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn)); 1263} 1264 1265// SVEFPComplexAddition. 1266 1267void Assembler::fcadd(const ZRegister& zd, 1268 const PRegisterM& pg, 1269 const ZRegister& zn, 1270 const ZRegister& zm, 1271 int rot) { 1272 // FCADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T>, <const> 1273 // 0110 0100 ..00 000. 100. .... .... .... 1274 // size<23:22> | rot<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 1275 1276 USE(zn); 1277 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1278 VIXL_ASSERT(zd.Is(zn)); 1279 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 1280 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1281 VIXL_ASSERT((rot == 90) || (rot == 270)); 1282 1283 Instr rotate_bit = (rot == 90) ? 0 : (1 << 16); 1284 Emit(FCADD_z_p_zz | rotate_bit | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 1285} 1286 1287// SVEFPComplexMulAdd. 1288 1289void Assembler::fcmla(const ZRegister& zda, 1290 const PRegisterM& pg, 1291 const ZRegister& zn, 1292 const ZRegister& zm, 1293 int rot) { 1294 // FCMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T>, <const> 1295 // 0110 0100 ..0. .... 0... .... .... .... 1296 // size<23:22> | Zm<20:16> | rot<14:13> | Pg<12:10> | Zn<9:5> | Zda<4:0> 1297 1298 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1299 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 1300 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 1301 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); 1302 1303 Instr rotate_bit = (rot / 90) << 13; 1304 Emit(FCMLA_z_p_zzz | rotate_bit | SVESize(zda) | Rd(zda) | PgLow8(pg) | 1305 Rn(zn) | Rm(zm)); 1306} 1307 1308// SVEFPComplexMulAddIndex. 1309 1310void Assembler::fcmla(const ZRegister& zda, 1311 const ZRegister& zn, 1312 const ZRegister& zm, 1313 int index, 1314 int rot) { 1315 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1316 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 1317 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); 1318 1319 Instr rotate_bit = (rot / 90) << 10; 1320 Emit(FCMLA_z_zzzi_h | SVEMulComplexIndexHelper(zm, index) | rotate_bit | 1321 Rd(zda) | Rn(zn)); 1322} 1323 1324// SVEFPFastReduction. 1325 1326void Assembler::faddv(const VRegister& vd, 1327 const PRegister& pg, 1328 const ZRegister& zn) { 1329 // FADDV <V><d>, <Pg>, <Zn>.<T> 1330 // 0110 0101 ..00 0000 001. .... .... .... 1331 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Vd<4:0> 1332 1333 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1334 VIXL_ASSERT(vd.IsScalar()); 1335 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1336 VIXL_ASSERT(AreSameLaneSize(zn, vd)); 1337 1338 Emit(FADDV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 1339} 1340 1341void Assembler::fmaxnmv(const VRegister& vd, 1342 const PRegister& pg, 1343 const ZRegister& zn) { 1344 // FMAXNMV <V><d>, <Pg>, <Zn>.<T> 1345 // 0110 0101 ..00 0100 001. .... .... .... 1346 // size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Vd<4:0> 1347 1348 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1349 VIXL_ASSERT(vd.IsScalar()); 1350 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1351 VIXL_ASSERT(AreSameLaneSize(zn, vd)); 1352 1353 Emit(FMAXNMV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 1354} 1355 1356void Assembler::fmaxv(const VRegister& vd, 1357 const PRegister& pg, 1358 const ZRegister& zn) { 1359 // FMAXV <V><d>, <Pg>, <Zn>.<T> 1360 // 0110 0101 ..00 0110 001. .... .... .... 1361 // size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Vd<4:0> 1362 1363 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1364 VIXL_ASSERT(vd.IsScalar()); 1365 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1366 VIXL_ASSERT(AreSameLaneSize(zn, vd)); 1367 1368 Emit(FMAXV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 1369} 1370 1371void Assembler::fminnmv(const VRegister& vd, 1372 const PRegister& pg, 1373 const ZRegister& zn) { 1374 // FMINNMV <V><d>, <Pg>, <Zn>.<T> 1375 // 0110 0101 ..00 0101 001. .... .... .... 1376 // size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Vd<4:0> 1377 1378 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1379 VIXL_ASSERT(vd.IsScalar()); 1380 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1381 VIXL_ASSERT(AreSameLaneSize(zn, vd)); 1382 1383 Emit(FMINNMV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 1384} 1385 1386void Assembler::fminv(const VRegister& vd, 1387 const PRegister& pg, 1388 const ZRegister& zn) { 1389 // FMINV <V><d>, <Pg>, <Zn>.<T> 1390 // 0110 0101 ..00 0111 001. .... .... .... 1391 // size<23:22> | opc<18:16> = 111 | Pg<12:10> | Zn<9:5> | Vd<4:0> 1392 1393 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1394 VIXL_ASSERT(vd.IsScalar()); 1395 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1396 VIXL_ASSERT(AreSameLaneSize(zn, vd)); 1397 1398 Emit(FMINV_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 1399} 1400 1401// SVEFPMulAdd. 1402 1403void Assembler::fmad(const ZRegister& zdn, 1404 const PRegisterM& pg, 1405 const ZRegister& zm, 1406 const ZRegister& za) { 1407 // FMAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> 1408 // 0110 0101 ..1. .... 100. .... .... .... 1409 // size<23:22> | Za<20:16> | opc<14:13> = 00 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 1410 1411 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1412 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); 1413 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1414 1415 Emit(FMAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za)); 1416} 1417 1418void Assembler::fmla(const ZRegister& zda, 1419 const PRegisterM& pg, 1420 const ZRegister& zn, 1421 const ZRegister& zm) { 1422 // FMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> 1423 // 0110 0101 ..1. .... 000. .... .... .... 1424 // size<23:22> | Zm<20:16> | opc<14:13> = 00 | Pg<12:10> | Zn<9:5> | Zda<4:0> 1425 1426 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1427 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 1428 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 1429 1430 Emit(FMLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1431} 1432 1433void Assembler::fmls(const ZRegister& zda, 1434 const PRegisterM& pg, 1435 const ZRegister& zn, 1436 const ZRegister& zm) { 1437 // FMLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> 1438 // 0110 0101 ..1. .... 001. .... .... .... 1439 // size<23:22> | Zm<20:16> | opc<14:13> = 01 | Pg<12:10> | Zn<9:5> | Zda<4:0> 1440 1441 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1442 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 1443 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 1444 1445 Emit(FMLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1446} 1447 1448void Assembler::fmsb(const ZRegister& zdn, 1449 const PRegisterM& pg, 1450 const ZRegister& zm, 1451 const ZRegister& za) { 1452 // FMSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> 1453 // 0110 0101 ..1. .... 101. .... .... .... 1454 // size<23:22> | Za<20:16> | opc<14:13> = 01 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 1455 1456 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1457 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); 1458 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1459 1460 Emit(FMSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za)); 1461} 1462 1463void Assembler::fnmad(const ZRegister& zdn, 1464 const PRegisterM& pg, 1465 const ZRegister& zm, 1466 const ZRegister& za) { 1467 // FNMAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> 1468 // 0110 0101 ..1. .... 110. .... .... .... 1469 // size<23:22> | Za<20:16> | opc<14:13> = 10 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 1470 1471 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1472 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); 1473 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1474 1475 Emit(FNMAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za)); 1476} 1477 1478void Assembler::fnmla(const ZRegister& zda, 1479 const PRegisterM& pg, 1480 const ZRegister& zn, 1481 const ZRegister& zm) { 1482 // FNMLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> 1483 // 0110 0101 ..1. .... 010. .... .... .... 1484 // size<23:22> | Zm<20:16> | opc<14:13> = 10 | Pg<12:10> | Zn<9:5> | Zda<4:0> 1485 1486 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1487 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 1488 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 1489 1490 Emit(FNMLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1491} 1492 1493void Assembler::fnmls(const ZRegister& zda, 1494 const PRegisterM& pg, 1495 const ZRegister& zn, 1496 const ZRegister& zm) { 1497 // FNMLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> 1498 // 0110 0101 ..1. .... 011. .... .... .... 1499 // size<23:22> | Zm<20:16> | opc<14:13> = 11 | Pg<12:10> | Zn<9:5> | Zda<4:0> 1500 1501 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1502 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 1503 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 1504 1505 Emit(FNMLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); 1506} 1507 1508void Assembler::fnmsb(const ZRegister& zdn, 1509 const PRegisterM& pg, 1510 const ZRegister& zm, 1511 const ZRegister& za) { 1512 // FNMSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> 1513 // 0110 0101 ..1. .... 111. .... .... .... 1514 // size<23:22> | Za<20:16> | opc<14:13> = 11 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 1515 1516 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1517 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); 1518 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 1519 1520 Emit(FNMSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rn(zm) | Rm(za)); 1521} 1522 1523Instr Assembler::SVEMulIndexHelper(unsigned lane_size_in_bytes_log2, 1524 const ZRegister& zm, 1525 int index, 1526 Instr op_h, 1527 Instr op_s, 1528 Instr op_d) { 1529 Instr size = lane_size_in_bytes_log2 << SVESize_offset; 1530 Instr zm_with_index = Rm(zm); 1531 Instr op = 0xffffffff; 1532 // Allowable register number and lane index depends on the lane size. 1533 switch (lane_size_in_bytes_log2) { 1534 case kHRegSizeInBytesLog2: 1535 VIXL_ASSERT(zm.GetCode() <= 7); 1536 VIXL_ASSERT(IsUint3(index)); 1537 // For H-sized lanes, size is encoded as 0b0x, where x is used as the top 1538 // bit of the index. So, if index is less than four, the top bit of index 1539 // is zero, and therefore size is 0b00. Otherwise, it's 0b01, the usual 1540 // encoding for H-sized lanes. 1541 if (index < 4) size = 0; 1542 // Top two bits of "zm" encode the index. 1543 zm_with_index |= (index & 3) << (Rm_offset + 3); 1544 op = op_h; 1545 break; 1546 case kSRegSizeInBytesLog2: 1547 VIXL_CHECK(zm.GetCode() <= 7); 1548 VIXL_CHECK(IsUint2(index)); 1549 // Top two bits of "zm" encode the index. 1550 zm_with_index |= (index & 3) << (Rm_offset + 3); 1551 op = op_s; 1552 break; 1553 case kDRegSizeInBytesLog2: 1554 VIXL_CHECK(zm.GetCode() <= 15); 1555 VIXL_CHECK(IsUint1(index)); 1556 // Top bit of "zm" encodes the index. 1557 zm_with_index |= (index & 1) << (Rm_offset + 4); 1558 op = op_d; 1559 break; 1560 default: 1561 VIXL_UNIMPLEMENTED(); 1562 } 1563 return op | zm_with_index | size; 1564} 1565 1566Instr Assembler::SVEMulLongIndexHelper(const ZRegister& zm, int index) { 1567 Instr imm_field; 1568 Instr zm_id; 1569 if (zm.IsLaneSizeH()) { 1570 VIXL_CHECK(zm.GetCode() <= 7); 1571 VIXL_CHECK(IsUint3(index)); 1572 imm_field = ExtractUnsignedBitfield32(2, 1, index) << 19; 1573 zm_id = Rx<18, 16>(zm); 1574 } else { 1575 VIXL_ASSERT(zm.IsLaneSizeS()); 1576 VIXL_CHECK(zm.GetCode() <= 15); 1577 VIXL_CHECK(IsUint2(index)); 1578 imm_field = ExtractBit(index, 1) << 20; 1579 zm_id = Rx<19, 16>(zm); 1580 } 1581 1582 // Synthesize the low part of immediate encoding. 1583 imm_field |= ExtractBit(index, 0) << 11; 1584 1585 return zm_id | imm_field; 1586} 1587 1588Instr Assembler::SVEMulComplexIndexHelper(const ZRegister& zm, int index) { 1589 Instr zm_idx_size; 1590 if (zm.IsLaneSizeH()) { 1591 // Zm<18:16> | i2<20:19> 1592 VIXL_CHECK(zm.GetCode() <= 7); 1593 VIXL_CHECK(IsUint2(index)); 1594 zm_idx_size = (index << 19) | Rx<18, 16>(zm) | 0; 1595 } else { 1596 VIXL_ASSERT(zm.IsLaneSizeS()); 1597 // Zm<19:16> | i1<20> 1598 VIXL_CHECK(zm.GetCode() <= 15); 1599 VIXL_CHECK(IsUint1(index)); 1600 zm_idx_size = (index << 20) | Rx<19, 16>(zm) | (1 << 22); 1601 } 1602 return zm_idx_size; 1603} 1604 1605// SVEFPMulAddIndex. 1606 1607void Assembler::fmla(const ZRegister& zda, 1608 const ZRegister& zn, 1609 const ZRegister& zm, 1610 int index) { 1611 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1612 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 1613 1614 // The encoding of opcode, index, Zm, and size are synthesized in this 1615 // variable. 1616 Instr synthesized_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(), 1617 zm, 1618 index, 1619 FMLA_z_zzzi_h, 1620 FMLA_z_zzzi_s, 1621 FMLA_z_zzzi_d); 1622 1623 Emit(synthesized_op | Rd(zda) | Rn(zn)); 1624} 1625 1626void Assembler::fmls(const ZRegister& zda, 1627 const ZRegister& zn, 1628 const ZRegister& zm, 1629 int index) { 1630 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1631 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 1632 1633 // The encoding of opcode, index, Zm, and size are synthesized in this 1634 // variable. 1635 Instr synthesized_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(), 1636 zm, 1637 index, 1638 FMLS_z_zzzi_h, 1639 FMLS_z_zzzi_s, 1640 FMLS_z_zzzi_d); 1641 1642 Emit(synthesized_op | Rd(zda) | Rn(zn)); 1643} 1644 1645// SVEFPMulIndex. 1646 1647// This prototype maps to 3 instruction encodings: 1648void Assembler::fmul(const ZRegister& zd, 1649 const ZRegister& zn, 1650 const ZRegister& zm, 1651 unsigned index) { 1652 // FMUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T>[<imm>] 1653 // 0110 0100 ..1. .... 0010 00.. .... .... 1654 // size<23:22> | opc<20:16> | Zn<9:5> | Zd<4:0> 1655 1656 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1657 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 1658 1659 // The encoding of opcode, index, Zm, and size are synthesized in this 1660 // variable. 1661 Instr synthesized_op = SVEMulIndexHelper(zd.GetLaneSizeInBytesLog2(), 1662 zm, 1663 index, 1664 FMUL_z_zzi_h, 1665 FMUL_z_zzi_s, 1666 FMUL_z_zzi_d); 1667 1668 Emit(synthesized_op | Rd(zd) | Rn(zn)); 1669} 1670 1671// SVEFPUnaryOpPredicated. 1672 1673void Assembler::fcvt(const ZRegister& zd, 1674 const PRegisterM& pg, 1675 const ZRegister& zn) { 1676 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1677 1678 Instr op = 0xffffffff; 1679 switch (zn.GetLaneSizeInBytes()) { 1680 case kHRegSizeInBytes: 1681 switch (zd.GetLaneSizeInBytes()) { 1682 case kSRegSizeInBytes: 1683 op = FCVT_z_p_z_h2s; 1684 break; 1685 case kDRegSizeInBytes: 1686 op = FCVT_z_p_z_h2d; 1687 break; 1688 } 1689 break; 1690 case kSRegSizeInBytes: 1691 switch (zd.GetLaneSizeInBytes()) { 1692 case kHRegSizeInBytes: 1693 op = FCVT_z_p_z_s2h; 1694 break; 1695 case kDRegSizeInBytes: 1696 op = FCVT_z_p_z_s2d; 1697 break; 1698 } 1699 break; 1700 case kDRegSizeInBytes: 1701 switch (zd.GetLaneSizeInBytes()) { 1702 case kHRegSizeInBytes: 1703 op = FCVT_z_p_z_d2h; 1704 break; 1705 case kSRegSizeInBytes: 1706 op = FCVT_z_p_z_d2s; 1707 break; 1708 } 1709 break; 1710 } 1711 VIXL_ASSERT(op != 0xffffffff); 1712 1713 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); 1714} 1715 1716void Assembler::fcvtzs(const ZRegister& zd, 1717 const PRegisterM& pg, 1718 const ZRegister& zn) { 1719 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1720 Instr op = 0xffffffff; 1721 switch (zn.GetLaneSizeInBytes()) { 1722 case kHRegSizeInBytes: 1723 switch (zd.GetLaneSizeInBytes()) { 1724 case kHRegSizeInBytes: 1725 op = FCVTZS_z_p_z_fp162h; 1726 break; 1727 case kSRegSizeInBytes: 1728 op = FCVTZS_z_p_z_fp162w; 1729 break; 1730 case kDRegSizeInBytes: 1731 op = FCVTZS_z_p_z_fp162x; 1732 break; 1733 } 1734 break; 1735 case kSRegSizeInBytes: 1736 switch (zd.GetLaneSizeInBytes()) { 1737 case kSRegSizeInBytes: 1738 op = FCVTZS_z_p_z_s2w; 1739 break; 1740 case kDRegSizeInBytes: 1741 op = FCVTZS_z_p_z_s2x; 1742 break; 1743 } 1744 break; 1745 case kDRegSizeInBytes: 1746 switch (zd.GetLaneSizeInBytes()) { 1747 case kSRegSizeInBytes: 1748 op = FCVTZS_z_p_z_d2w; 1749 break; 1750 case kDRegSizeInBytes: 1751 op = FCVTZS_z_p_z_d2x; 1752 break; 1753 } 1754 break; 1755 } 1756 VIXL_ASSERT(op != 0xffffffff); 1757 1758 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); 1759} 1760 1761void Assembler::fcvtzu(const ZRegister& zd, 1762 const PRegisterM& pg, 1763 const ZRegister& zn) { 1764 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1765 Instr op = 0xffffffff; 1766 switch (zn.GetLaneSizeInBytes()) { 1767 case kHRegSizeInBytes: 1768 switch (zd.GetLaneSizeInBytes()) { 1769 case kHRegSizeInBytes: 1770 op = FCVTZU_z_p_z_fp162h; 1771 break; 1772 case kSRegSizeInBytes: 1773 op = FCVTZU_z_p_z_fp162w; 1774 break; 1775 case kDRegSizeInBytes: 1776 op = FCVTZU_z_p_z_fp162x; 1777 break; 1778 } 1779 break; 1780 case kSRegSizeInBytes: 1781 switch (zd.GetLaneSizeInBytes()) { 1782 case kSRegSizeInBytes: 1783 op = FCVTZU_z_p_z_s2w; 1784 break; 1785 case kDRegSizeInBytes: 1786 op = FCVTZU_z_p_z_s2x; 1787 break; 1788 } 1789 break; 1790 case kDRegSizeInBytes: 1791 switch (zd.GetLaneSizeInBytes()) { 1792 case kSRegSizeInBytes: 1793 op = FCVTZU_z_p_z_d2w; 1794 break; 1795 case kDRegSizeInBytes: 1796 op = FCVTZU_z_p_z_d2x; 1797 break; 1798 } 1799 break; 1800 } 1801 VIXL_ASSERT(op != 0xffffffff); 1802 1803 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); 1804} 1805 1806void Assembler::frecpx(const ZRegister& zd, 1807 const PRegisterM& pg, 1808 const ZRegister& zn) { 1809 // FRECPX <Zd>.<T>, <Pg>/M, <Zn>.<T> 1810 // 0110 0101 ..00 1100 101. .... .... .... 1811 // size<23:22> | opc<17:16> = 00 | Pg<12:10> | Zn<9:5> | Zd<4:0> 1812 1813 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1814 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 1815 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1816 1817 Emit(FRECPX_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 1818} 1819 1820void Assembler::frinta(const ZRegister& zd, 1821 const PRegisterM& pg, 1822 const ZRegister& zn) { 1823 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1824 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 1825 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1826 1827 Emit(FRINTA_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 1828} 1829 1830void Assembler::frinti(const ZRegister& zd, 1831 const PRegisterM& pg, 1832 const ZRegister& zn) { 1833 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1834 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 1835 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1836 1837 Emit(FRINTI_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 1838} 1839 1840void Assembler::frintm(const ZRegister& zd, 1841 const PRegisterM& pg, 1842 const ZRegister& zn) { 1843 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1844 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 1845 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1846 1847 Emit(FRINTM_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 1848} 1849 1850void Assembler::frintn(const ZRegister& zd, 1851 const PRegisterM& pg, 1852 const ZRegister& zn) { 1853 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1854 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 1855 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1856 1857 Emit(FRINTN_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 1858} 1859 1860void Assembler::frintp(const ZRegister& zd, 1861 const PRegisterM& pg, 1862 const ZRegister& zn) { 1863 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1864 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 1865 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1866 1867 Emit(FRINTP_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 1868} 1869 1870void Assembler::frintx(const ZRegister& zd, 1871 const PRegisterM& pg, 1872 const ZRegister& zn) { 1873 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1874 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 1875 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1876 1877 Emit(FRINTX_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 1878} 1879 1880void Assembler::frintz(const ZRegister& zd, 1881 const PRegisterM& pg, 1882 const ZRegister& zn) { 1883 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1884 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 1885 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1886 1887 Emit(FRINTZ_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 1888} 1889 1890void Assembler::fsqrt(const ZRegister& zd, 1891 const PRegisterM& pg, 1892 const ZRegister& zn) { 1893 // FSQRT <Zd>.<T>, <Pg>/M, <Zn>.<T> 1894 // 0110 0101 ..00 1101 101. .... .... .... 1895 // size<23:22> | opc<17:16> = 01 | Pg<12:10> | Zn<9:5> | Zd<4:0> 1896 1897 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1898 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 1899 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 1900 1901 Emit(FSQRT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 1902} 1903 1904void Assembler::scvtf(const ZRegister& zd, 1905 const PRegisterM& pg, 1906 const ZRegister& zn) { 1907 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1908 Instr op = 0xffffffff; 1909 switch (zn.GetLaneSizeInBytes()) { 1910 case kHRegSizeInBytes: 1911 switch (zd.GetLaneSizeInBytes()) { 1912 case kHRegSizeInBytes: 1913 op = SCVTF_z_p_z_h2fp16; 1914 break; 1915 } 1916 break; 1917 case kSRegSizeInBytes: 1918 switch (zd.GetLaneSizeInBytes()) { 1919 case kHRegSizeInBytes: 1920 op = SCVTF_z_p_z_w2fp16; 1921 break; 1922 case kSRegSizeInBytes: 1923 op = SCVTF_z_p_z_w2s; 1924 break; 1925 case kDRegSizeInBytes: 1926 op = SCVTF_z_p_z_w2d; 1927 break; 1928 } 1929 break; 1930 case kDRegSizeInBytes: 1931 switch (zd.GetLaneSizeInBytes()) { 1932 case kHRegSizeInBytes: 1933 op = SCVTF_z_p_z_x2fp16; 1934 break; 1935 case kSRegSizeInBytes: 1936 op = SCVTF_z_p_z_x2s; 1937 break; 1938 case kDRegSizeInBytes: 1939 op = SCVTF_z_p_z_x2d; 1940 break; 1941 } 1942 break; 1943 } 1944 VIXL_ASSERT(op != 0xffffffff); 1945 1946 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); 1947} 1948 1949void Assembler::ucvtf(const ZRegister& zd, 1950 const PRegisterM& pg, 1951 const ZRegister& zn) { 1952 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 1953 Instr op = 0xffffffff; 1954 switch (zn.GetLaneSizeInBytes()) { 1955 case kHRegSizeInBytes: 1956 switch (zd.GetLaneSizeInBytes()) { 1957 case kHRegSizeInBytes: 1958 op = UCVTF_z_p_z_h2fp16; 1959 break; 1960 } 1961 break; 1962 case kSRegSizeInBytes: 1963 switch (zd.GetLaneSizeInBytes()) { 1964 case kHRegSizeInBytes: 1965 op = UCVTF_z_p_z_w2fp16; 1966 break; 1967 case kSRegSizeInBytes: 1968 op = UCVTF_z_p_z_w2s; 1969 break; 1970 case kDRegSizeInBytes: 1971 op = UCVTF_z_p_z_w2d; 1972 break; 1973 } 1974 break; 1975 case kDRegSizeInBytes: 1976 switch (zd.GetLaneSizeInBytes()) { 1977 case kHRegSizeInBytes: 1978 op = UCVTF_z_p_z_x2fp16; 1979 break; 1980 case kSRegSizeInBytes: 1981 op = UCVTF_z_p_z_x2s; 1982 break; 1983 case kDRegSizeInBytes: 1984 op = UCVTF_z_p_z_x2d; 1985 break; 1986 } 1987 break; 1988 } 1989 VIXL_ASSERT(op != 0xffffffff); 1990 1991 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); 1992} 1993 1994// SVEFPUnaryOpUnpredicated. 1995 1996void Assembler::frecpe(const ZRegister& zd, const ZRegister& zn) { 1997 // FRECPE <Zd>.<T>, <Zn>.<T> 1998 // 0110 0101 ..00 1110 0011 00.. .... .... 1999 // size<23:22> | opc<18:16> = 110 | Zn<9:5> | Zd<4:0> 2000 2001 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2002 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 2003 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 2004 2005 Emit(FRECPE_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); 2006} 2007 2008void Assembler::frsqrte(const ZRegister& zd, const ZRegister& zn) { 2009 // FRSQRTE <Zd>.<T>, <Zn>.<T> 2010 // 0110 0101 ..00 1111 0011 00.. .... .... 2011 // size<23:22> | opc<18:16> = 111 | Zn<9:5> | Zd<4:0> 2012 2013 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2014 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 2015 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 2016 2017 Emit(FRSQRTE_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); 2018} 2019 2020// SVEIncDecByPredicateCount. 2021 2022void Assembler::decp(const Register& rdn, const PRegisterWithLaneSize& pg) { 2023 // DECP <Xdn>, <Pg>.<T> 2024 // 0010 0101 ..10 1101 1000 100. .... .... 2025 // size<23:22> | op<17> = 0 | D<16> = 1 | opc2<10:9> = 00 | Pg<8:5> | 2026 // Rdn<4:0> 2027 2028 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2029 VIXL_ASSERT(rdn.IsX()); 2030 2031 Emit(DECP_r_p_r | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg)); 2032} 2033 2034void Assembler::decp(const ZRegister& zdn, const PRegister& pg) { 2035 // DECP <Zdn>.<T>, <Pg> 2036 // 0010 0101 ..10 1101 1000 000. .... .... 2037 // size<23:22> | op<17> = 0 | D<16> = 1 | opc2<10:9> = 00 | Pg<8:5> | 2038 // Zdn<4:0> 2039 2040 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2041 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 2042 VIXL_ASSERT(pg.IsUnqualified()); 2043 2044 Emit(DECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); 2045} 2046 2047void Assembler::incp(const Register& rdn, const PRegisterWithLaneSize& pg) { 2048 // INCP <Xdn>, <Pg>.<T> 2049 // 0010 0101 ..10 1100 1000 100. .... .... 2050 // size<23:22> | op<17> = 0 | D<16> = 0 | opc2<10:9> = 00 | Pg<8:5> | 2051 // Rdn<4:0> 2052 2053 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2054 VIXL_ASSERT(rdn.IsX()); 2055 2056 Emit(INCP_r_p_r | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg)); 2057} 2058 2059void Assembler::incp(const ZRegister& zdn, const PRegister& pg) { 2060 // INCP <Zdn>.<T>, <Pg> 2061 // 0010 0101 ..10 1100 1000 000. .... .... 2062 // size<23:22> | op<17> = 0 | D<16> = 0 | opc2<10:9> = 00 | Pg<8:5> | 2063 // Zdn<4:0> 2064 2065 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2066 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 2067 VIXL_ASSERT(pg.IsUnqualified()); 2068 2069 Emit(INCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); 2070} 2071 2072void Assembler::sqdecp(const Register& xd, 2073 const PRegisterWithLaneSize& pg, 2074 const Register& wn) { 2075 // SQDECP <Xdn>, <Pg>.<T>, <Wdn> 2076 // 0010 0101 ..10 1010 1000 100. .... .... 2077 // size<23:22> | D<17> = 1 | U<16> = 0 | sf<10> = 0 | op<9> = 0 | Pg<8:5> | 2078 // Rdn<4:0> 2079 2080 USE(wn); 2081 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2082 VIXL_ASSERT(xd.IsX() && wn.IsW() && xd.Aliases(wn)); 2083 2084 Emit(SQDECP_r_p_r_sx | SVESize(pg) | Rd(xd) | Rx<8, 5>(pg)); 2085} 2086 2087void Assembler::sqdecp(const Register& xdn, const PRegisterWithLaneSize& pg) { 2088 // SQDECP <Xdn>, <Pg>.<T> 2089 // 0010 0101 ..10 1010 1000 110. .... .... 2090 // size<23:22> | D<17> = 1 | U<16> = 0 | sf<10> = 1 | op<9> = 0 | Pg<8:5> | 2091 // Rdn<4:0> 2092 2093 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2094 VIXL_ASSERT(xdn.IsX()); 2095 2096 Emit(SQDECP_r_p_r_x | SVESize(pg) | Rd(xdn) | Rx<8, 5>(pg)); 2097} 2098 2099void Assembler::sqdecp(const ZRegister& zdn, const PRegister& pg) { 2100 // SQDECP <Zdn>.<T>, <Pg> 2101 // 0010 0101 ..10 1010 1000 000. .... .... 2102 // size<23:22> | D<17> = 1 | U<16> = 0 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0> 2103 2104 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2105 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 2106 VIXL_ASSERT(pg.IsUnqualified()); 2107 2108 Emit(SQDECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); 2109} 2110 2111void Assembler::sqincp(const Register& xd, 2112 const PRegisterWithLaneSize& pg, 2113 const Register& wn) { 2114 // SQINCP <Xdn>, <Pg>.<T>, <Wdn> 2115 // 0010 0101 ..10 1000 1000 100. .... .... 2116 // size<23:22> | D<17> = 0 | U<16> = 0 | sf<10> = 0 | op<9> = 0 | Pg<8:5> | 2117 // Rdn<4:0> 2118 2119 USE(wn); 2120 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2121 VIXL_ASSERT(xd.IsX() && wn.IsW() && xd.Aliases(wn)); 2122 2123 Emit(SQINCP_r_p_r_sx | SVESize(pg) | Rd(xd) | Rx<8, 5>(pg)); 2124} 2125 2126void Assembler::sqincp(const Register& xdn, const PRegisterWithLaneSize& pg) { 2127 // SQINCP <Xdn>, <Pg>.<T> 2128 // 0010 0101 ..10 1000 1000 110. .... .... 2129 // size<23:22> | D<17> = 0 | U<16> = 0 | sf<10> = 1 | op<9> = 0 | Pg<8:5> | 2130 // Rdn<4:0> 2131 2132 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2133 VIXL_ASSERT(xdn.IsX()); 2134 2135 Emit(SQINCP_r_p_r_x | SVESize(pg) | Rd(xdn) | Rx<8, 5>(pg)); 2136} 2137 2138void Assembler::sqincp(const ZRegister& zdn, const PRegister& pg) { 2139 // SQINCP <Zdn>.<T>, <Pg> 2140 // 0010 0101 ..10 1000 1000 000. .... .... 2141 // size<23:22> | D<17> = 0 | U<16> = 0 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0> 2142 2143 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2144 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 2145 VIXL_ASSERT(pg.IsUnqualified()); 2146 2147 Emit(SQINCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); 2148} 2149 2150void Assembler::uqdecp(const Register& rdn, const PRegisterWithLaneSize& pg) { 2151 // UQDECP <Wdn>, <Pg>.<T> 2152 // UQDECP <Xdn>, <Pg>.<T> 2153 // 0010 0101 ..10 1011 1000 10.. .... .... 2154 // size<23:22> | D<17> = 1 | U<16> = 1 | sf<10> | op<9> = 0 | Pg<8:5> | 2155 // Rdn<4:0> 2156 2157 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2158 2159 Instr op = rdn.IsX() ? UQDECP_r_p_r_x : UQDECP_r_p_r_uw; 2160 Emit(op | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg)); 2161} 2162 2163void Assembler::uqdecp(const ZRegister& zdn, const PRegister& pg) { 2164 // UQDECP <Zdn>.<T>, <Pg> 2165 // 0010 0101 ..10 1011 1000 000. .... .... 2166 // size<23:22> | D<17> = 1 | U<16> = 1 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0> 2167 2168 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2169 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 2170 VIXL_ASSERT(pg.IsUnqualified()); 2171 2172 Emit(UQDECP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); 2173} 2174 2175void Assembler::uqincp(const Register& rdn, const PRegisterWithLaneSize& pg) { 2176 // UQINCP <Wdn>, <Pg>.<T> 2177 // 0010 0101 ..10 1001 1000 100. .... .... 2178 // size<23:22> | D<17> = 0 | U<16> = 1 | sf<10> = 0 | op<9> = 0 | Pg<8:5> | 2179 // Rdn<4:0> 2180 2181 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2182 2183 Instr op = rdn.IsX() ? UQINCP_r_p_r_x : UQINCP_r_p_r_uw; 2184 Emit(op | SVESize(pg) | Rd(rdn) | Rx<8, 5>(pg)); 2185} 2186 2187void Assembler::uqincp(const ZRegister& zdn, const PRegister& pg) { 2188 // UQINCP <Zdn>.<T>, <Pg> 2189 // 0010 0101 ..10 1001 1000 000. .... .... 2190 // size<23:22> | D<17> = 0 | U<16> = 1 | opc<10:9> = 00 | Pg<8:5> | Zdn<4:0> 2191 2192 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2193 VIXL_ASSERT(zdn.GetLaneSizeInBytes() != kBRegSizeInBytes); 2194 VIXL_ASSERT(pg.IsUnqualified()); 2195 2196 Emit(UQINCP_z_p_z | SVESize(zdn) | Rd(zdn) | Pg<8, 5>(pg)); 2197} 2198 2199// SVEIndexGeneration. 2200 2201void Assembler::index(const ZRegister& zd, int start, int step) { 2202 // INDEX <Zd>.<T>, #<imm1>, #<imm2> 2203 // 0000 0100 ..1. .... 0100 00.. .... .... 2204 // size<23:22> | step<20:16> | start<9:5> | Zd<4:0> 2205 2206 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2207 2208 Emit(INDEX_z_ii | SVESize(zd) | ImmField<20, 16>(step) | 2209 ImmField<9, 5>(start) | Rd(zd)); 2210} 2211 2212void Assembler::index(const ZRegister& zd, 2213 const Register& rn, 2214 const Register& rm) { 2215 // INDEX <Zd>.<T>, <R><n>, <R><m> 2216 // 0000 0100 ..1. .... 0100 11.. .... .... 2217 // size<23:22> | Rm<20:16> | Rn<9:5> | Zd<4:0> 2218 2219 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2220 VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >= 2221 zd.GetLaneSizeInBits()); 2222 VIXL_ASSERT(static_cast<unsigned>(rm.GetSizeInBits()) >= 2223 zd.GetLaneSizeInBits()); 2224 2225 Emit(INDEX_z_rr | SVESize(zd) | Rd(zd) | Rn(rn) | Rm(rm)); 2226} 2227 2228void Assembler::index(const ZRegister& zd, const Register& rn, int imm5) { 2229 // INDEX <Zd>.<T>, <R><n>, #<imm> 2230 // 0000 0100 ..1. .... 0100 01.. .... .... 2231 // size<23:22> | imm5<20:16> | Rn<9:5> | Zd<4:0> 2232 2233 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2234 VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >= 2235 zd.GetLaneSizeInBits()); 2236 2237 Emit(INDEX_z_ri | SVESize(zd) | Rd(zd) | Rn(rn) | ImmField<20, 16>(imm5)); 2238} 2239 2240void Assembler::index(const ZRegister& zd, int imm5, const Register& rm) { 2241 // INDEX <Zd>.<T>, #<imm>, <R><m> 2242 // 0000 0100 ..1. .... 0100 10.. .... .... 2243 // size<23:22> | Rm<20:16> | imm5<9:5> | Zd<4:0> 2244 2245 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2246 VIXL_ASSERT(static_cast<unsigned>(rm.GetSizeInBits()) >= 2247 zd.GetLaneSizeInBits()); 2248 2249 Emit(INDEX_z_ir | SVESize(zd) | Rd(zd) | ImmField<9, 5>(imm5) | Rm(rm)); 2250} 2251 2252// SVEIntArithmeticUnpredicated. 2253 2254void Assembler::add(const ZRegister& zd, 2255 const ZRegister& zn, 2256 const ZRegister& zm) { 2257 // ADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 2258 // 0000 0100 ..1. .... 0000 00.. .... .... 2259 // size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0> 2260 2261 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2262 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 2263 2264 Emit(ADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 2265} 2266 2267void Assembler::sqadd(const ZRegister& zd, 2268 const ZRegister& zn, 2269 const ZRegister& zm) { 2270 // SQADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 2271 // 0000 0100 ..1. .... 0001 00.. .... .... 2272 // size<23:22> | Zm<20:16> | opc<12:10> = 100 | Zn<9:5> | Zd<4:0> 2273 2274 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2275 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 2276 2277 Emit(SQADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 2278} 2279 2280void Assembler::sqsub(const ZRegister& zd, 2281 const ZRegister& zn, 2282 const ZRegister& zm) { 2283 // SQSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 2284 // 0000 0100 ..1. .... 0001 10.. .... .... 2285 // size<23:22> | Zm<20:16> | opc<12:10> = 110 | Zn<9:5> | Zd<4:0> 2286 2287 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2288 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 2289 2290 Emit(SQSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 2291} 2292 2293void Assembler::sub(const ZRegister& zd, 2294 const ZRegister& zn, 2295 const ZRegister& zm) { 2296 // SUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 2297 // 0000 0100 ..1. .... 0000 01.. .... .... 2298 // size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0> 2299 2300 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2301 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 2302 2303 Emit(SUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 2304} 2305 2306void Assembler::uqadd(const ZRegister& zd, 2307 const ZRegister& zn, 2308 const ZRegister& zm) { 2309 // UQADD <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 2310 // 0000 0100 ..1. .... 0001 01.. .... .... 2311 // size<23:22> | Zm<20:16> | opc<12:10> = 101 | Zn<9:5> | Zd<4:0> 2312 2313 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2314 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 2315 2316 Emit(UQADD_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 2317} 2318 2319void Assembler::uqsub(const ZRegister& zd, 2320 const ZRegister& zn, 2321 const ZRegister& zm) { 2322 // UQSUB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 2323 // 0000 0100 ..1. .... 0001 11.. .... .... 2324 // size<23:22> | Zm<20:16> | opc<12:10> = 111 | Zn<9:5> | Zd<4:0> 2325 2326 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2327 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 2328 2329 Emit(UQSUB_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 2330} 2331 2332// SVEIntBinaryArithmeticPredicated. 2333 2334void Assembler::add(const ZRegister& zd, 2335 const PRegisterM& pg, 2336 const ZRegister& zn, 2337 const ZRegister& zm) { 2338 // ADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2339 // 0000 0100 ..00 0000 000. .... .... .... 2340 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2341 2342 USE(zn); 2343 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2344 VIXL_ASSERT(zd.Is(zn)); 2345 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2346 2347 Emit(ADD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2348} 2349 2350void Assembler::and_(const ZRegister& zd, 2351 const PRegisterM& pg, 2352 const ZRegister& zn, 2353 const ZRegister& zm) { 2354 // AND <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2355 // 0000 0100 ..01 1010 000. .... .... .... 2356 // size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2357 2358 USE(zn); 2359 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2360 VIXL_ASSERT(zd.Is(zn)); 2361 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2362 2363 Emit(AND_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2364} 2365 2366void Assembler::bic(const ZRegister& zd, 2367 const PRegisterM& pg, 2368 const ZRegister& zn, 2369 const ZRegister& zm) { 2370 // BIC <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2371 // 0000 0100 ..01 1011 000. .... .... .... 2372 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2373 2374 USE(zn); 2375 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2376 VIXL_ASSERT(zd.Is(zn)); 2377 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2378 2379 Emit(BIC_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2380} 2381 2382void Assembler::eor(const ZRegister& zd, 2383 const PRegisterM& pg, 2384 const ZRegister& zn, 2385 const ZRegister& zm) { 2386 // EOR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2387 // 0000 0100 ..01 1001 000. .... .... .... 2388 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2389 2390 USE(zn); 2391 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2392 VIXL_ASSERT(zd.Is(zn)); 2393 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2394 2395 Emit(EOR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2396} 2397 2398void Assembler::mul(const ZRegister& zd, 2399 const PRegisterM& pg, 2400 const ZRegister& zn, 2401 const ZRegister& zm) { 2402 // MUL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2403 // 0000 0100 ..01 0000 000. .... .... .... 2404 // size<23:22> | H<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2405 2406 USE(zn); 2407 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2408 VIXL_ASSERT(zd.Is(zn)); 2409 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2410 2411 Emit(MUL_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2412} 2413 2414void Assembler::orr(const ZRegister& zd, 2415 const PRegisterM& pg, 2416 const ZRegister& zn, 2417 const ZRegister& zm) { 2418 // ORR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2419 // 0000 0100 ..01 1000 000. .... .... .... 2420 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2421 2422 USE(zn); 2423 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2424 VIXL_ASSERT(zd.Is(zn)); 2425 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2426 2427 Emit(ORR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2428} 2429 2430void Assembler::sabd(const ZRegister& zd, 2431 const PRegisterM& pg, 2432 const ZRegister& zn, 2433 const ZRegister& zm) { 2434 // SABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2435 // 0000 0100 ..00 1100 000. .... .... .... 2436 // size<23:22> | opc<18:17> = 10 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2437 2438 USE(zn); 2439 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2440 VIXL_ASSERT(zd.Is(zn)); 2441 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2442 2443 Emit(SABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2444} 2445 2446void Assembler::sdiv(const ZRegister& zd, 2447 const PRegisterM& pg, 2448 const ZRegister& zn, 2449 const ZRegister& zm) { 2450 // SDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2451 // 0000 0100 ..01 0100 000. .... .... .... 2452 // size<23:22> | R<17> = 0 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2453 2454 USE(zn); 2455 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2456 VIXL_ASSERT(zd.Is(zn)); 2457 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2458 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); 2459 2460 Emit(SDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2461} 2462 2463void Assembler::sdivr(const ZRegister& zd, 2464 const PRegisterM& pg, 2465 const ZRegister& zn, 2466 const ZRegister& zm) { 2467 // SDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2468 // 0000 0100 ..01 0110 000. .... .... .... 2469 // size<23:22> | R<17> = 1 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2470 2471 USE(zn); 2472 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2473 VIXL_ASSERT(zd.Is(zn)); 2474 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2475 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); 2476 2477 Emit(SDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2478} 2479 2480void Assembler::smax(const ZRegister& zd, 2481 const PRegisterM& pg, 2482 const ZRegister& zn, 2483 const ZRegister& zm) { 2484 // SMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2485 // 0000 0100 ..00 1000 000. .... .... .... 2486 // size<23:22> | opc<18:17> = 00 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2487 2488 USE(zn); 2489 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2490 VIXL_ASSERT(zd.Is(zn)); 2491 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2492 2493 Emit(SMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2494} 2495 2496void Assembler::smin(const ZRegister& zd, 2497 const PRegisterM& pg, 2498 const ZRegister& zn, 2499 const ZRegister& zm) { 2500 // SMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2501 // 0000 0100 ..00 1010 000. .... .... .... 2502 // size<23:22> | opc<18:17> = 01 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2503 2504 USE(zn); 2505 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2506 VIXL_ASSERT(zd.Is(zn)); 2507 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2508 2509 Emit(SMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2510} 2511 2512void Assembler::smulh(const ZRegister& zd, 2513 const PRegisterM& pg, 2514 const ZRegister& zn, 2515 const ZRegister& zm) { 2516 // SMULH <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2517 // 0000 0100 ..01 0010 000. .... .... .... 2518 // size<23:22> | H<17> = 1 | U<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2519 2520 USE(zn); 2521 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2522 VIXL_ASSERT(zd.Is(zn)); 2523 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2524 2525 Emit(SMULH_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2526} 2527 2528void Assembler::sub(const ZRegister& zd, 2529 const PRegisterM& pg, 2530 const ZRegister& zn, 2531 const ZRegister& zm) { 2532 // SUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2533 // 0000 0100 ..00 0001 000. .... .... .... 2534 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2535 2536 USE(zn); 2537 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2538 VIXL_ASSERT(zd.Is(zn)); 2539 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2540 2541 Emit(SUB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2542} 2543 2544void Assembler::subr(const ZRegister& zd, 2545 const PRegisterM& pg, 2546 const ZRegister& zn, 2547 const ZRegister& zm) { 2548 // SUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2549 // 0000 0100 ..00 0011 000. .... .... .... 2550 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2551 2552 USE(zn); 2553 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2554 VIXL_ASSERT(zd.Is(zn)); 2555 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2556 2557 Emit(SUBR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2558} 2559 2560void Assembler::uabd(const ZRegister& zd, 2561 const PRegisterM& pg, 2562 const ZRegister& zn, 2563 const ZRegister& zm) { 2564 // UABD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2565 // 0000 0100 ..00 1101 000. .... .... .... 2566 // size<23:22> | opc<18:17> = 10 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2567 2568 USE(zn); 2569 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2570 VIXL_ASSERT(zd.Is(zn)); 2571 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2572 2573 Emit(UABD_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2574} 2575 2576void Assembler::udiv(const ZRegister& zd, 2577 const PRegisterM& pg, 2578 const ZRegister& zn, 2579 const ZRegister& zm) { 2580 // UDIV <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2581 // 0000 0100 ..01 0101 000. .... .... .... 2582 // size<23:22> | R<17> = 0 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2583 2584 USE(zn); 2585 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2586 VIXL_ASSERT(zd.Is(zn)); 2587 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2588 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); 2589 2590 Emit(UDIV_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2591} 2592 2593void Assembler::udivr(const ZRegister& zd, 2594 const PRegisterM& pg, 2595 const ZRegister& zn, 2596 const ZRegister& zm) { 2597 // UDIVR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2598 // 0000 0100 ..01 0111 000. .... .... .... 2599 // size<23:22> | R<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2600 2601 USE(zn); 2602 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2603 VIXL_ASSERT(zd.Is(zn)); 2604 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2605 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); 2606 2607 Emit(UDIVR_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2608} 2609 2610void Assembler::umax(const ZRegister& zd, 2611 const PRegisterM& pg, 2612 const ZRegister& zn, 2613 const ZRegister& zm) { 2614 // UMAX <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2615 // 0000 0100 ..00 1001 000. .... .... .... 2616 // size<23:22> | opc<18:17> = 00 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2617 2618 USE(zn); 2619 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2620 VIXL_ASSERT(zd.Is(zn)); 2621 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2622 2623 Emit(UMAX_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2624} 2625 2626void Assembler::umin(const ZRegister& zd, 2627 const PRegisterM& pg, 2628 const ZRegister& zn, 2629 const ZRegister& zm) { 2630 // UMIN <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2631 // 0000 0100 ..00 1011 000. .... .... .... 2632 // size<23:22> | opc<18:17> = 01 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2633 2634 USE(zn); 2635 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2636 VIXL_ASSERT(zd.Is(zn)); 2637 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2638 2639 Emit(UMIN_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2640} 2641 2642void Assembler::umulh(const ZRegister& zd, 2643 const PRegisterM& pg, 2644 const ZRegister& zn, 2645 const ZRegister& zm) { 2646 // UMULH <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 2647 // 0000 0100 ..01 0011 000. .... .... .... 2648 // size<23:22> | H<17> = 1 | U<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 2649 2650 USE(zn); 2651 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2652 VIXL_ASSERT(zd.Is(zn)); 2653 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 2654 2655 Emit(UMULH_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 2656} 2657 2658// SVEIntCompareScalars. 2659 2660void Assembler::ctermeq(const Register& rn, const Register& rm) { 2661 // CTERMEQ <R><n>, <R><m> 2662 // 0010 0101 1.1. .... 0010 00.. ...0 0000 2663 // op<23> = 1 | sz<22> | Rm<20:16> | Rn<9:5> | ne<4> = 0 2664 2665 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2666 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 2667 const Instr sz = rn.Is64Bits() ? 0x00400000 : 0x00000000; 2668 2669 Emit(CTERMEQ_rr | sz | Rn(rn) | Rm(rm)); 2670} 2671 2672void Assembler::ctermne(const Register& rn, const Register& rm) { 2673 // CTERMNE <R><n>, <R><m> 2674 // 0010 0101 1.1. .... 0010 00.. ...1 0000 2675 // op<23> = 1 | sz<22> | Rm<20:16> | Rn<9:5> | ne<4> = 1 2676 2677 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2678 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 2679 const Instr sz = rn.Is64Bits() ? 0x00400000 : 0x00000000; 2680 2681 Emit(CTERMNE_rr | sz | Rn(rn) | Rm(rm)); 2682} 2683 2684void Assembler::whilele(const PRegisterWithLaneSize& pd, 2685 const Register& rn, 2686 const Register& rm) { 2687 // WHILELE <Pd>.<T>, <R><n>, <R><m> 2688 // 0010 0101 ..1. .... 000. 01.. ...1 .... 2689 // size<23:22> | Rm<20:16> | sf<12> | U<11> = 0 | lt<10> = 1 | Rn<9:5> | 2690 // eq<4> = 1 | Pd<3:0> 2691 2692 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2693 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 2694 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; 2695 2696 Emit(WHILELE_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); 2697} 2698 2699void Assembler::whilelo(const PRegisterWithLaneSize& pd, 2700 const Register& rn, 2701 const Register& rm) { 2702 // WHILELO <Pd>.<T>, <R><n>, <R><m> 2703 // 0010 0101 ..1. .... 000. 11.. ...0 .... 2704 // size<23:22> | Rm<20:16> | sf<12> | U<11> = 1 | lt<10> = 1 | Rn<9:5> | 2705 // eq<4> = 0 | Pd<3:0> 2706 2707 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2708 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 2709 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; 2710 2711 Emit(WHILELO_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); 2712} 2713 2714void Assembler::whilels(const PRegisterWithLaneSize& pd, 2715 const Register& rn, 2716 const Register& rm) { 2717 // WHILELS <Pd>.<T>, <R><n>, <R><m> 2718 // 0010 0101 ..1. .... 000. 11.. ...1 .... 2719 // size<23:22> | Rm<20:16> | sf<12> | U<11> = 1 | lt<10> = 1 | Rn<9:5> | 2720 // eq<4> = 1 | Pd<3:0> 2721 2722 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2723 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 2724 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; 2725 2726 Emit(WHILELS_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); 2727} 2728 2729void Assembler::whilelt(const PRegisterWithLaneSize& pd, 2730 const Register& rn, 2731 const Register& rm) { 2732 // WHILELT <Pd>.<T>, <R><n>, <R><m> 2733 // 0010 0101 ..1. .... 000. 01.. ...0 .... 2734 // size<23:22> | Rm<20:16> | sf<12> | U<11> = 0 | lt<10> = 1 | Rn<9:5> | 2735 // eq<4> = 0 | Pd<3:0> 2736 2737 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2738 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 2739 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; 2740 2741 Emit(WHILELT_p_p_rr | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); 2742} 2743 2744void Assembler::CompareVectors(const PRegisterWithLaneSize& pd, 2745 const PRegisterZ& pg, 2746 const ZRegister& zn, 2747 const ZRegister& zm, 2748 SVEIntCompareVectorsOp op) { 2749 Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 2750} 2751 2752void Assembler::CompareVectors(const PRegisterWithLaneSize& pd, 2753 const PRegisterZ& pg, 2754 const ZRegister& zn, 2755 int imm, 2756 SVEIntCompareSignedImmOp op) { 2757 Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm)); 2758} 2759 2760void Assembler::CompareVectors(const PRegisterWithLaneSize& pd, 2761 const PRegisterZ& pg, 2762 const ZRegister& zn, 2763 unsigned imm, 2764 SVEIntCompareUnsignedImmOp op) { 2765 Emit(op | SVESize(zn) | Pd(pd) | PgLow8(pg) | Rn(zn) | 2766 ImmUnsignedField<20, 14>(imm)); 2767} 2768 2769void Assembler::cmp(Condition cond, 2770 const PRegisterWithLaneSize& pd, 2771 const PRegisterZ& pg, 2772 const ZRegister& zn, 2773 const ZRegister& zm) { 2774 switch (cond) { 2775 case eq: 2776 cmpeq(pd, pg, zn, zm); 2777 break; 2778 case ge: 2779 cmpge(pd, pg, zn, zm); 2780 break; 2781 case gt: 2782 cmpgt(pd, pg, zn, zm); 2783 break; 2784 case le: 2785 cmple(pd, pg, zn, zm); 2786 break; 2787 case lt: 2788 cmplt(pd, pg, zn, zm); 2789 break; 2790 case ne: 2791 cmpne(pd, pg, zn, zm); 2792 break; 2793 case hi: 2794 cmphi(pd, pg, zn, zm); 2795 break; 2796 case hs: 2797 cmphs(pd, pg, zn, zm); 2798 break; 2799 case lo: 2800 cmplo(pd, pg, zn, zm); 2801 break; 2802 case ls: 2803 cmpls(pd, pg, zn, zm); 2804 break; 2805 default: 2806 VIXL_UNREACHABLE(); 2807 } 2808} 2809 2810// SVEIntCompareSignedImm. 2811 2812void Assembler::cmpeq(const PRegisterWithLaneSize& pd, 2813 const PRegisterZ& pg, 2814 const ZRegister& zn, 2815 int imm5) { 2816 // CMPEQ <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2817 // 0010 0101 ..0. .... 100. .... ...0 .... 2818 // size<23:22> | imm5<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> 2819 // | ne<4> = 0 | Pd<3:0> 2820 2821 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2822 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2823 2824 CompareVectors(pd, pg, zn, imm5, CMPEQ_p_p_zi); 2825} 2826 2827void Assembler::cmpge(const PRegisterWithLaneSize& pd, 2828 const PRegisterZ& pg, 2829 const ZRegister& zn, 2830 int imm5) { 2831 // CMPGE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2832 // 0010 0101 ..0. .... 000. .... ...0 .... 2833 // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> 2834 // | ne<4> = 0 | Pd<3:0> 2835 2836 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2837 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2838 2839 CompareVectors(pd, pg, zn, imm5, CMPGE_p_p_zi); 2840} 2841 2842void Assembler::cmpgt(const PRegisterWithLaneSize& pd, 2843 const PRegisterZ& pg, 2844 const ZRegister& zn, 2845 int imm5) { 2846 // CMPGT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2847 // 0010 0101 ..0. .... 000. .... ...1 .... 2848 // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 0 | Pg<12:10> | Zn<9:5> 2849 // | ne<4> = 1 | Pd<3:0> 2850 2851 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2852 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2853 2854 CompareVectors(pd, pg, zn, imm5, CMPGT_p_p_zi); 2855} 2856 2857void Assembler::cmple(const PRegisterWithLaneSize& pd, 2858 const PRegisterZ& pg, 2859 const ZRegister& zn, 2860 int imm5) { 2861 // CMPLE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2862 // 0010 0101 ..0. .... 001. .... ...1 .... 2863 // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> 2864 // | ne<4> = 1 | Pd<3:0> 2865 2866 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2867 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2868 2869 CompareVectors(pd, pg, zn, imm5, CMPLE_p_p_zi); 2870} 2871 2872void Assembler::cmplt(const PRegisterWithLaneSize& pd, 2873 const PRegisterZ& pg, 2874 const ZRegister& zn, 2875 int imm5) { 2876 // CMPLT <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2877 // 0010 0101 ..0. .... 001. .... ...0 .... 2878 // size<23:22> | imm5<20:16> | op<15> = 0 | o2<13> = 1 | Pg<12:10> | Zn<9:5> 2879 // | ne<4> = 0 | Pd<3:0> 2880 2881 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2882 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2883 2884 CompareVectors(pd, pg, zn, imm5, CMPLT_p_p_zi); 2885} 2886 2887void Assembler::cmpne(const PRegisterWithLaneSize& pd, 2888 const PRegisterZ& pg, 2889 const ZRegister& zn, 2890 int imm5) { 2891 // CMPNE <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2892 // 0010 0101 ..0. .... 100. .... ...1 .... 2893 // size<23:22> | imm5<20:16> | op<15> = 1 | o2<13> = 0 | Pg<12:10> | Zn<9:5> 2894 // | ne<4> = 1 | Pd<3:0> 2895 2896 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2897 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2898 2899 CompareVectors(pd, pg, zn, imm5, CMPNE_p_p_zi); 2900} 2901 2902// SVEIntCompareUnsignedImm. 2903 2904void Assembler::cmphi(const PRegisterWithLaneSize& pd, 2905 const PRegisterZ& pg, 2906 const ZRegister& zn, 2907 unsigned imm7) { 2908 // CMPHI <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2909 // 0010 0100 ..1. .... ..0. .... ...1 .... 2910 // size<23:22> | imm7<20:14> | lt<13> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 1 | 2911 // Pd<3:0> 2912 2913 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2914 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2915 2916 CompareVectors(pd, pg, zn, imm7, CMPHI_p_p_zi); 2917} 2918 2919void Assembler::cmphs(const PRegisterWithLaneSize& pd, 2920 const PRegisterZ& pg, 2921 const ZRegister& zn, 2922 unsigned imm7) { 2923 // CMPHS <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2924 // 0010 0100 ..1. .... ..0. .... ...0 .... 2925 // size<23:22> | imm7<20:14> | lt<13> = 0 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | 2926 // Pd<3:0> 2927 2928 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2929 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2930 2931 CompareVectors(pd, pg, zn, imm7, CMPHS_p_p_zi); 2932} 2933 2934void Assembler::cmplo(const PRegisterWithLaneSize& pd, 2935 const PRegisterZ& pg, 2936 const ZRegister& zn, 2937 unsigned imm7) { 2938 // CMPLO <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2939 // 0010 0100 ..1. .... ..1. .... ...0 .... 2940 // size<23:22> | imm7<20:14> | lt<13> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 0 | 2941 // Pd<3:0> 2942 2943 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2944 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2945 2946 CompareVectors(pd, pg, zn, imm7, CMPLO_p_p_zi); 2947} 2948 2949void Assembler::cmpls(const PRegisterWithLaneSize& pd, 2950 const PRegisterZ& pg, 2951 const ZRegister& zn, 2952 unsigned imm7) { 2953 // CMPLS <Pd>.<T>, <Pg>/Z, <Zn>.<T>, #<imm> 2954 // 0010 0100 ..1. .... ..1. .... ...1 .... 2955 // size<23:22> | imm7<20:14> | lt<13> = 1 | Pg<12:10> | Zn<9:5> | ne<4> = 1 | 2956 // Pd<3:0> 2957 2958 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2959 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2960 2961 CompareVectors(pd, pg, zn, imm7, CMPLS_p_p_zi); 2962} 2963 2964// SVEIntCompareVectors. 2965 2966// This prototype maps to 2 instruction encodings: 2967// CMPEQ_p_p_zw 2968// CMPEQ_p_p_zz 2969void Assembler::cmpeq(const PRegisterWithLaneSize& pd, 2970 const PRegisterZ& pg, 2971 const ZRegister& zn, 2972 const ZRegister& zm) { 2973 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2974 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2975 SVEIntCompareVectorsOp op = CMPEQ_p_p_zz; 2976 if (!AreSameLaneSize(zn, zm)) { 2977 VIXL_ASSERT(zm.IsLaneSizeD()); 2978 op = CMPEQ_p_p_zw; 2979 } 2980 CompareVectors(pd, pg, zn, zm, op); 2981} 2982 2983// This prototype maps to 2 instruction encodings: 2984// CMPGE_p_p_zw 2985// CMPGE_p_p_zz 2986void Assembler::cmpge(const PRegisterWithLaneSize& pd, 2987 const PRegisterZ& pg, 2988 const ZRegister& zn, 2989 const ZRegister& zm) { 2990 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 2991 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 2992 SVEIntCompareVectorsOp op = CMPGE_p_p_zz; 2993 if (!AreSameLaneSize(zn, zm)) { 2994 VIXL_ASSERT(zm.IsLaneSizeD()); 2995 op = CMPGE_p_p_zw; 2996 } 2997 CompareVectors(pd, pg, zn, zm, op); 2998} 2999 3000// This prototype maps to 2 instruction encodings: 3001// CMPGT_p_p_zw 3002// CMPGT_p_p_zz 3003void Assembler::cmpgt(const PRegisterWithLaneSize& pd, 3004 const PRegisterZ& pg, 3005 const ZRegister& zn, 3006 const ZRegister& zm) { 3007 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3008 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 3009 SVEIntCompareVectorsOp op = CMPGT_p_p_zz; 3010 if (!AreSameLaneSize(zn, zm)) { 3011 VIXL_ASSERT(zm.IsLaneSizeD()); 3012 op = CMPGT_p_p_zw; 3013 } 3014 CompareVectors(pd, pg, zn, zm, op); 3015} 3016 3017// This prototype maps to 2 instruction encodings: 3018// CMPHI_p_p_zw 3019// CMPHI_p_p_zz 3020void Assembler::cmphi(const PRegisterWithLaneSize& pd, 3021 const PRegisterZ& pg, 3022 const ZRegister& zn, 3023 const ZRegister& zm) { 3024 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3025 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 3026 SVEIntCompareVectorsOp op = CMPHI_p_p_zz; 3027 if (!AreSameLaneSize(zn, zm)) { 3028 VIXL_ASSERT(zm.IsLaneSizeD()); 3029 op = CMPHI_p_p_zw; 3030 } 3031 CompareVectors(pd, pg, zn, zm, op); 3032} 3033 3034// This prototype maps to 2 instruction encodings: 3035// CMPHS_p_p_zw 3036// CMPHS_p_p_zz 3037void Assembler::cmphs(const PRegisterWithLaneSize& pd, 3038 const PRegisterZ& pg, 3039 const ZRegister& zn, 3040 const ZRegister& zm) { 3041 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3042 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 3043 SVEIntCompareVectorsOp op = CMPHS_p_p_zz; 3044 if (!AreSameLaneSize(zn, zm)) { 3045 VIXL_ASSERT(zm.IsLaneSizeD()); 3046 op = CMPHS_p_p_zw; 3047 } 3048 CompareVectors(pd, pg, zn, zm, op); 3049} 3050 3051void Assembler::cmple(const PRegisterWithLaneSize& pd, 3052 const PRegisterZ& pg, 3053 const ZRegister& zn, 3054 const ZRegister& zm) { 3055 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3056 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 3057 if (AreSameLaneSize(zn, zm)) { 3058 cmpge(pd, pg, zm, zn); 3059 return; 3060 } 3061 VIXL_ASSERT(zm.IsLaneSizeD()); 3062 VIXL_ASSERT(!zn.IsLaneSizeD()); 3063 3064 CompareVectors(pd, pg, zn, zm, CMPLE_p_p_zw); 3065} 3066 3067void Assembler::cmplo(const PRegisterWithLaneSize& pd, 3068 const PRegisterZ& pg, 3069 const ZRegister& zn, 3070 const ZRegister& zm) { 3071 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3072 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 3073 if (AreSameLaneSize(zn, zm)) { 3074 cmphi(pd, pg, zm, zn); 3075 return; 3076 } 3077 VIXL_ASSERT(zm.IsLaneSizeD()); 3078 VIXL_ASSERT(!zn.IsLaneSizeD()); 3079 3080 CompareVectors(pd, pg, zn, zm, CMPLO_p_p_zw); 3081} 3082 3083void Assembler::cmpls(const PRegisterWithLaneSize& pd, 3084 const PRegisterZ& pg, 3085 const ZRegister& zn, 3086 const ZRegister& zm) { 3087 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3088 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 3089 if (AreSameLaneSize(zn, zm)) { 3090 cmphs(pd, pg, zm, zn); 3091 return; 3092 } 3093 VIXL_ASSERT(zm.IsLaneSizeD()); 3094 VIXL_ASSERT(!zn.IsLaneSizeD()); 3095 3096 CompareVectors(pd, pg, zn, zm, CMPLS_p_p_zw); 3097} 3098 3099void Assembler::cmplt(const PRegisterWithLaneSize& pd, 3100 const PRegisterZ& pg, 3101 const ZRegister& zn, 3102 const ZRegister& zm) { 3103 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3104 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 3105 if (AreSameLaneSize(zn, zm)) { 3106 cmpgt(pd, pg, zm, zn); 3107 return; 3108 } 3109 VIXL_ASSERT(zm.IsLaneSizeD()); 3110 VIXL_ASSERT(!zn.IsLaneSizeD()); 3111 3112 CompareVectors(pd, pg, zn, zm, CMPLT_p_p_zw); 3113} 3114 3115// This prototype maps to 2 instruction encodings: 3116// CMPNE_p_p_zw 3117// CMPNE_p_p_zz 3118void Assembler::cmpne(const PRegisterWithLaneSize& pd, 3119 const PRegisterZ& pg, 3120 const ZRegister& zn, 3121 const ZRegister& zm) { 3122 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3123 VIXL_ASSERT(AreSameLaneSize(pd, zn)); 3124 SVEIntCompareVectorsOp op = CMPNE_p_p_zz; 3125 if (!AreSameLaneSize(zn, zm)) { 3126 VIXL_ASSERT(zm.IsLaneSizeD()); 3127 op = CMPNE_p_p_zw; 3128 } 3129 CompareVectors(pd, pg, zn, zm, op); 3130} 3131 3132// SVEIntMiscUnpredicated. 3133 3134void Assembler::fexpa(const ZRegister& zd, const ZRegister& zn) { 3135 // FEXPA <Zd>.<T>, <Zn>.<T> 3136 // 0000 0100 ..10 0000 1011 10.. .... .... 3137 // size<23:22> | opc<20:16> = 00000 | Zn<9:5> | Zd<4:0> 3138 3139 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3140 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3141 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 3142 3143 Emit(FEXPA_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); 3144} 3145 3146void Assembler::ftssel(const ZRegister& zd, 3147 const ZRegister& zn, 3148 const ZRegister& zm) { 3149 // FTSSEL <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 3150 // 0000 0100 ..1. .... 1011 00.. .... .... 3151 // size<23:22> | Zm<20:16> | op<10> = 0 | Zn<9:5> | Zd<4:0> 3152 3153 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3154 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 3155 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 3156 3157 Emit(FTSSEL_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 3158} 3159 3160void Assembler::movprfx(const ZRegister& zd, const ZRegister& zn) { 3161 // MOVPRFX <Zd>, <Zn> 3162 // 0000 0100 0010 0000 1011 11.. .... .... 3163 // opc<23:22> = 00 | opc2<20:16> = 00000 | Zn<9:5> | Zd<4:0> 3164 3165 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3166 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3167 3168 Emit(MOVPRFX_z_z | Rd(zd) | Rn(zn)); 3169} 3170 3171// SVEIntMulAddPredicated. 3172 3173void Assembler::mad(const ZRegister& zdn, 3174 const PRegisterM& pg, 3175 const ZRegister& zm, 3176 const ZRegister& za) { 3177 // MAD <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> 3178 // 0000 0100 ..0. .... 110. .... .... .... 3179 // size<23:22> | Zm<20:16> | op<13> = 0 | Pg<12:10> | Za<9:5> | Zdn<4:0> 3180 3181 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3182 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); 3183 3184 Emit(MAD_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rm(zm) | Rn(za)); 3185} 3186 3187void Assembler::mla(const ZRegister& zda, 3188 const PRegisterM& pg, 3189 const ZRegister& zn, 3190 const ZRegister& zm) { 3191 // MLA <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> 3192 // 0000 0100 ..0. .... 010. .... .... .... 3193 // size<23:22> | Zm<20:16> | op<13> = 0 | Pg<12:10> | Zn<9:5> | Zda<4:0> 3194 3195 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3196 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 3197 3198 Emit(MLA_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); 3199} 3200 3201void Assembler::mls(const ZRegister& zda, 3202 const PRegisterM& pg, 3203 const ZRegister& zn, 3204 const ZRegister& zm) { 3205 // MLS <Zda>.<T>, <Pg>/M, <Zn>.<T>, <Zm>.<T> 3206 // 0000 0100 ..0. .... 011. .... .... .... 3207 // size<23:22> | Zm<20:16> | op<13> = 1 | Pg<12:10> | Zn<9:5> | Zda<4:0> 3208 3209 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3210 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 3211 3212 Emit(MLS_z_p_zzz | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn) | Rm(zm)); 3213} 3214 3215void Assembler::msb(const ZRegister& zdn, 3216 const PRegisterM& pg, 3217 const ZRegister& zm, 3218 const ZRegister& za) { 3219 // MSB <Zdn>.<T>, <Pg>/M, <Zm>.<T>, <Za>.<T> 3220 // 0000 0100 ..0. .... 111. .... .... .... 3221 // size<23:22> | Zm<20:16> | op<13> = 1 | Pg<12:10> | Za<9:5> | Zdn<4:0> 3222 3223 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3224 VIXL_ASSERT(AreSameLaneSize(zdn, zm, za)); 3225 3226 Emit(MSB_z_p_zzz | SVESize(zdn) | Rd(zdn) | PgLow8(pg) | Rm(zm) | Rn(za)); 3227} 3228 3229// SVEIntMulAddUnpredicated. 3230 3231void Assembler::sdot(const ZRegister& zda, 3232 const ZRegister& zn, 3233 const ZRegister& zm) { 3234 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3235 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 3236 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); 3237 VIXL_ASSERT(AreSameLaneSize(zm, zn)); 3238 3239 Emit(SDOT_z_zzz | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 3240} 3241 3242void Assembler::udot(const ZRegister& zda, 3243 const ZRegister& zn, 3244 const ZRegister& zm) { 3245 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3246 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 3247 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); 3248 VIXL_ASSERT(AreSameLaneSize(zm, zn)); 3249 3250 Emit(UDOT_z_zzz | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 3251} 3252 3253// SVEIntReduction. 3254 3255void Assembler::andv(const VRegister& vd, 3256 const PRegister& pg, 3257 const ZRegister& zn) { 3258 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3259 VIXL_ASSERT(vd.IsScalar()); 3260 3261 Emit(ANDV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 3262} 3263 3264void Assembler::eorv(const VRegister& vd, 3265 const PRegister& pg, 3266 const ZRegister& zn) { 3267 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3268 VIXL_ASSERT(vd.IsScalar()); 3269 3270 Emit(EORV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 3271} 3272 3273void Assembler::movprfx(const ZRegister& zd, 3274 const PRegister& pg, 3275 const ZRegister& zn) { 3276 // MOVPRFX <Zd>.<T>, <Pg>/<ZM>, <Zn>.<T> 3277 // 0000 0100 ..01 000. 001. .... .... .... 3278 // size<23:22> | opc<18:17> = 00 | M<16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 3279 3280 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3281 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3282 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); 3283 VIXL_ASSERT(!pg.HasLaneSize()); 3284 3285 Instr m = pg.IsMerging() ? 0x00010000 : 0x00000000; 3286 Emit(MOVPRFX_z_p_z | SVESize(zd) | m | Rd(zd) | PgLow8(pg) | Rn(zn)); 3287} 3288 3289void Assembler::orv(const VRegister& vd, 3290 const PRegister& pg, 3291 const ZRegister& zn) { 3292 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3293 VIXL_ASSERT(vd.IsScalar()); 3294 3295 Emit(ORV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 3296} 3297 3298void Assembler::saddv(const VRegister& dd, 3299 const PRegister& pg, 3300 const ZRegister& zn) { 3301 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3302 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kDRegSizeInBytes); 3303 3304 Emit(SADDV_r_p_z | SVESize(zn) | Rd(dd) | PgLow8(pg) | Rn(zn)); 3305} 3306 3307void Assembler::smaxv(const VRegister& vd, 3308 const PRegister& pg, 3309 const ZRegister& zn) { 3310 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3311 VIXL_ASSERT(vd.IsScalar()); 3312 3313 Emit(SMAXV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 3314} 3315 3316void Assembler::sminv(const VRegister& vd, 3317 const PRegister& pg, 3318 const ZRegister& zn) { 3319 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3320 VIXL_ASSERT(vd.IsScalar()); 3321 3322 Emit(SMINV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 3323} 3324 3325void Assembler::uaddv(const VRegister& dd, 3326 const PRegister& pg, 3327 const ZRegister& zn) { 3328 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3329 3330 Emit(UADDV_r_p_z | SVESize(zn) | Rd(dd) | PgLow8(pg) | Rn(zn)); 3331} 3332 3333void Assembler::umaxv(const VRegister& vd, 3334 const PRegister& pg, 3335 const ZRegister& zn) { 3336 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3337 VIXL_ASSERT(vd.IsScalar()); 3338 3339 Emit(UMAXV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 3340} 3341 3342void Assembler::uminv(const VRegister& vd, 3343 const PRegister& pg, 3344 const ZRegister& zn) { 3345 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3346 VIXL_ASSERT(vd.IsScalar()); 3347 3348 Emit(UMINV_r_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 3349} 3350 3351// SVEIntUnaryArithmeticPredicated. 3352 3353void Assembler::abs(const ZRegister& zd, 3354 const PRegisterM& pg, 3355 const ZRegister& zn) { 3356 // ABS <Zd>.<T>, <Pg>/M, <Zn>.<T> 3357 // 0000 0100 ..01 0110 101. .... .... .... 3358 // size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3359 3360 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3361 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3362 3363 Emit(ABS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3364} 3365 3366void Assembler::cls(const ZRegister& zd, 3367 const PRegisterM& pg, 3368 const ZRegister& zn) { 3369 // CLS <Zd>.<T>, <Pg>/M, <Zn>.<T> 3370 // 0000 0100 ..01 1000 101. .... .... .... 3371 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3372 3373 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3374 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3375 3376 Emit(CLS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3377} 3378 3379void Assembler::clz(const ZRegister& zd, 3380 const PRegisterM& pg, 3381 const ZRegister& zn) { 3382 // CLZ <Zd>.<T>, <Pg>/M, <Zn>.<T> 3383 // 0000 0100 ..01 1001 101. .... .... .... 3384 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3385 3386 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3387 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3388 3389 Emit(CLZ_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3390} 3391 3392void Assembler::cnot(const ZRegister& zd, 3393 const PRegisterM& pg, 3394 const ZRegister& zn) { 3395 // CNOT <Zd>.<T>, <Pg>/M, <Zn>.<T> 3396 // 0000 0100 ..01 1011 101. .... .... .... 3397 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3398 3399 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3400 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3401 3402 Emit(CNOT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3403} 3404 3405void Assembler::cnt(const ZRegister& zd, 3406 const PRegisterM& pg, 3407 const ZRegister& zn) { 3408 // CNT <Zd>.<T>, <Pg>/M, <Zn>.<T> 3409 // 0000 0100 ..01 1010 101. .... .... .... 3410 // size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3411 3412 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3413 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3414 3415 Emit(CNT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3416} 3417 3418void Assembler::fabs(const ZRegister& zd, 3419 const PRegisterM& pg, 3420 const ZRegister& zn) { 3421 // FABS <Zd>.<T>, <Pg>/M, <Zn>.<T> 3422 // 0000 0100 ..01 1100 101. .... .... .... 3423 // size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3424 3425 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3426 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3427 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 3428 3429 Emit(FABS_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3430} 3431 3432void Assembler::fneg(const ZRegister& zd, 3433 const PRegisterM& pg, 3434 const ZRegister& zn) { 3435 // FNEG <Zd>.<T>, <Pg>/M, <Zn>.<T> 3436 // 0000 0100 ..01 1101 101. .... .... .... 3437 // size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3438 3439 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3440 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3441 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 3442 3443 Emit(FNEG_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3444} 3445 3446void Assembler::neg(const ZRegister& zd, 3447 const PRegisterM& pg, 3448 const ZRegister& zn) { 3449 // NEG <Zd>.<T>, <Pg>/M, <Zn>.<T> 3450 // 0000 0100 ..01 0111 101. .... .... .... 3451 // size<23:22> | opc<18:16> = 111 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3452 3453 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3454 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3455 3456 Emit(NEG_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3457} 3458 3459void Assembler::not_(const ZRegister& zd, 3460 const PRegisterM& pg, 3461 const ZRegister& zn) { 3462 // NOT <Zd>.<T>, <Pg>/M, <Zn>.<T> 3463 // 0000 0100 ..01 1110 101. .... .... .... 3464 // size<23:22> | opc<18:16> = 110 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3465 3466 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3467 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3468 3469 Emit(NOT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3470} 3471 3472void Assembler::sxtb(const ZRegister& zd, 3473 const PRegisterM& pg, 3474 const ZRegister& zn) { 3475 // SXTB <Zd>.<T>, <Pg>/M, <Zn>.<T> 3476 // 0000 0100 ..01 0000 101. .... .... .... 3477 // size<23:22> | opc<18:16> = 000 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3478 3479 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3480 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3481 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kBRegSizeInBytes); 3482 3483 Emit(SXTB_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3484} 3485 3486void Assembler::sxth(const ZRegister& zd, 3487 const PRegisterM& pg, 3488 const ZRegister& zn) { 3489 // SXTH <Zd>.<T>, <Pg>/M, <Zn>.<T> 3490 // 0000 0100 ..01 0010 101. .... .... .... 3491 // size<23:22> | opc<18:16> = 010 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3492 3493 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3494 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3495 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kHRegSizeInBytes); 3496 3497 Emit(SXTH_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3498} 3499 3500void Assembler::sxtw(const ZRegister& zd, 3501 const PRegisterM& pg, 3502 const ZRegister& zn) { 3503 // SXTW <Zd>.D, <Pg>/M, <Zn>.D 3504 // 0000 0100 ..01 0100 101. .... .... .... 3505 // size<23:22> | opc<18:16> = 100 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3506 3507 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3508 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3509 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kSRegSizeInBytes); 3510 3511 Emit(SXTW_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3512} 3513 3514void Assembler::uxtb(const ZRegister& zd, 3515 const PRegisterM& pg, 3516 const ZRegister& zn) { 3517 // UXTB <Zd>.<T>, <Pg>/M, <Zn>.<T> 3518 // 0000 0100 ..01 0001 101. .... .... .... 3519 // size<23:22> | opc<18:16> = 001 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3520 3521 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3522 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3523 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kBRegSizeInBytes); 3524 3525 Emit(UXTB_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3526} 3527 3528void Assembler::uxth(const ZRegister& zd, 3529 const PRegisterM& pg, 3530 const ZRegister& zn) { 3531 // UXTH <Zd>.<T>, <Pg>/M, <Zn>.<T> 3532 // 0000 0100 ..01 0011 101. .... .... .... 3533 // size<23:22> | opc<18:16> = 011 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3534 3535 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3536 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3537 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kHRegSizeInBytes); 3538 3539 Emit(UXTH_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3540} 3541 3542void Assembler::uxtw(const ZRegister& zd, 3543 const PRegisterM& pg, 3544 const ZRegister& zn) { 3545 // UXTW <Zd>.D, <Pg>/M, <Zn>.D 3546 // 0000 0100 ..01 0101 101. .... .... .... 3547 // size<23:22> | opc<18:16> = 101 | Pg<12:10> | Zn<9:5> | Zd<4:0> 3548 3549 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3550 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3551 VIXL_ASSERT(zd.GetLaneSizeInBytes() > kSRegSizeInBytes); 3552 3553 Emit(UXTW_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 3554} 3555 3556// SVEIntWideImmPredicated. 3557 3558void Assembler::cpy(const ZRegister& zd, 3559 const PRegister& pg, 3560 int imm8, 3561 int shift) { 3562 // CPY <Zd>.<T>, <Pg>/<ZM>, #<imm>{, <shift>} 3563 // 0000 0101 ..01 .... 0... .... .... .... 3564 // size<23:22> | Pg<19:16> | M<14> | sh<13> | imm8<12:5> | Zd<4:0> 3565 3566 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3567 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); 3568 3569 ResolveSVEImm8Shift(&imm8, &shift); 3570 3571 Instr sh = (shift > 0) ? (1 << 13) : 0; 3572 Instr m = pg.IsMerging() ? (1 << 14) : 0; 3573 Emit(CPY_z_p_i | m | sh | SVESize(zd) | Rd(zd) | Pg<19, 16>(pg) | 3574 ImmField<12, 5>(imm8)); 3575} 3576 3577void Assembler::fcpy(const ZRegister& zd, const PRegisterM& pg, double imm) { 3578 // FCPY <Zd>.<T>, <Pg>/M, #<const> 3579 // 0000 0101 ..01 .... 110. .... .... .... 3580 // size<23:22> | Pg<19:16> | imm8<12:5> | Zd<4:0> 3581 3582 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3583 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 3584 3585 Instr imm_field = ImmUnsignedField<12, 5>(FP64ToImm8(imm)); 3586 Emit(FCPY_z_p_i | SVESize(zd) | Rd(zd) | Pg<19, 16>(pg) | imm_field); 3587} 3588 3589// SVEIntAddSubtractImmUnpredicated. 3590 3591void Assembler::SVEIntAddSubtractImmUnpredicatedHelper( 3592 SVEIntAddSubtractImm_UnpredicatedOp op, 3593 const ZRegister& zd, 3594 int imm8, 3595 int shift) { 3596 if (shift < 0) { 3597 VIXL_ASSERT(shift == -1); 3598 // Derive the shift amount from the immediate. 3599 if (IsUint8(imm8)) { 3600 shift = 0; 3601 } else if (IsUint16(imm8) && ((imm8 % 256) == 0)) { 3602 imm8 /= 256; 3603 shift = 8; 3604 } 3605 } 3606 3607 VIXL_ASSERT(IsUint8(imm8)); 3608 VIXL_ASSERT((shift == 0) || (shift == 8)); 3609 3610 Instr shift_bit = (shift > 0) ? (1 << 13) : 0; 3611 Emit(op | SVESize(zd) | Rd(zd) | shift_bit | ImmUnsignedField<12, 5>(imm8)); 3612} 3613 3614void Assembler::add(const ZRegister& zd, 3615 const ZRegister& zn, 3616 int imm8, 3617 int shift) { 3618 // ADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} 3619 // 0010 0101 ..10 0000 11.. .... .... .... 3620 // size<23:22> | opc<18:16> = 000 | sh<13> | imm8<12:5> | Zdn<4:0> 3621 3622 USE(zn); 3623 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3624 VIXL_ASSERT(zd.Is(zn)); 3625 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3626 3627 SVEIntAddSubtractImmUnpredicatedHelper(ADD_z_zi, zd, imm8, shift); 3628} 3629 3630void Assembler::dup(const ZRegister& zd, int imm8, int shift) { 3631 // DUP <Zd>.<T>, #<imm>{, <shift>} 3632 // 0010 0101 ..11 1000 11.. .... .... .... 3633 // size<23:22> | opc<18:17> = 00 | sh<13> | imm8<12:5> | Zd<4:0> 3634 3635 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3636 3637 ResolveSVEImm8Shift(&imm8, &shift); 3638 VIXL_ASSERT((shift < 8) || !zd.IsLaneSizeB()); 3639 3640 Instr shift_bit = (shift > 0) ? (1 << 13) : 0; 3641 Emit(DUP_z_i | SVESize(zd) | Rd(zd) | shift_bit | ImmField<12, 5>(imm8)); 3642} 3643 3644void Assembler::fdup(const ZRegister& zd, double imm) { 3645 // FDUP <Zd>.<T>, #<const> 3646 // 0010 0101 ..11 1001 110. .... .... .... 3647 // size<23:22> | opc<18:17> = 00 | o2<13> = 0 | imm8<12:5> | Zd<4:0> 3648 3649 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3650 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 3651 3652 Instr encoded_imm = FP64ToImm8(imm) << 5; 3653 Emit(FDUP_z_i | SVESize(zd) | encoded_imm | Rd(zd)); 3654} 3655 3656void Assembler::mul(const ZRegister& zd, const ZRegister& zn, int imm8) { 3657 // MUL <Zdn>.<T>, <Zdn>.<T>, #<imm> 3658 // 0010 0101 ..11 0000 110. .... .... .... 3659 // size<23:22> | opc<18:16> = 000 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> 3660 3661 USE(zn); 3662 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3663 VIXL_ASSERT(zd.Is(zn)); 3664 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3665 3666 Emit(MUL_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8)); 3667} 3668 3669void Assembler::smax(const ZRegister& zd, const ZRegister& zn, int imm8) { 3670 // SMAX <Zdn>.<T>, <Zdn>.<T>, #<imm> 3671 // 0010 0101 ..10 1000 110. .... .... .... 3672 // size<23:22> | opc<18:16> = 000 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> 3673 3674 USE(zn); 3675 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3676 VIXL_ASSERT(zd.Is(zn)); 3677 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3678 3679 Emit(SMAX_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8)); 3680} 3681 3682void Assembler::smin(const ZRegister& zd, const ZRegister& zn, int imm8) { 3683 // SMIN <Zdn>.<T>, <Zdn>.<T>, #<imm> 3684 // 0010 0101 ..10 1010 110. .... .... .... 3685 // size<23:22> | opc<18:16> = 010 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> 3686 3687 USE(zn); 3688 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3689 VIXL_ASSERT(zd.Is(zn)); 3690 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3691 3692 Emit(SMIN_z_zi | SVESize(zd) | Rd(zd) | ImmField<12, 5>(imm8)); 3693} 3694 3695void Assembler::sqadd(const ZRegister& zd, 3696 const ZRegister& zn, 3697 int imm8, 3698 int shift) { 3699 // SQADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} 3700 // 0010 0101 ..10 0100 11.. .... .... .... 3701 // size<23:22> | opc<18:16> = 100 | sh<13> | imm8<12:5> | Zdn<4:0> 3702 3703 USE(zn); 3704 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3705 VIXL_ASSERT(zd.Is(zn)); 3706 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3707 3708 SVEIntAddSubtractImmUnpredicatedHelper(SQADD_z_zi, zd, imm8, shift); 3709} 3710 3711void Assembler::sqsub(const ZRegister& zd, 3712 const ZRegister& zn, 3713 int imm8, 3714 int shift) { 3715 // SQSUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} 3716 // 0010 0101 ..10 0110 11.. .... .... .... 3717 // size<23:22> | opc<18:16> = 110 | sh<13> | imm8<12:5> | Zdn<4:0> 3718 3719 USE(zn); 3720 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3721 VIXL_ASSERT(zd.Is(zn)); 3722 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3723 3724 SVEIntAddSubtractImmUnpredicatedHelper(SQSUB_z_zi, zd, imm8, shift); 3725} 3726 3727void Assembler::sub(const ZRegister& zd, 3728 const ZRegister& zn, 3729 int imm8, 3730 int shift) { 3731 // SUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} 3732 // 0010 0101 ..10 0001 11.. .... .... .... 3733 // size<23:22> | opc<18:16> = 001 | sh<13> | imm8<12:5> | Zdn<4:0> 3734 3735 USE(zn); 3736 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3737 VIXL_ASSERT(zd.Is(zn)); 3738 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3739 3740 SVEIntAddSubtractImmUnpredicatedHelper(SUB_z_zi, zd, imm8, shift); 3741} 3742 3743void Assembler::subr(const ZRegister& zd, 3744 const ZRegister& zn, 3745 int imm8, 3746 int shift) { 3747 // SUBR <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} 3748 // 0010 0101 ..10 0011 11.. .... .... .... 3749 // size<23:22> | opc<18:16> = 011 | sh<13> | imm8<12:5> | Zdn<4:0> 3750 3751 USE(zn); 3752 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3753 VIXL_ASSERT(zd.Is(zn)); 3754 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3755 3756 SVEIntAddSubtractImmUnpredicatedHelper(SUBR_z_zi, zd, imm8, shift); 3757} 3758 3759void Assembler::umax(const ZRegister& zd, const ZRegister& zn, int imm8) { 3760 // UMAX <Zdn>.<T>, <Zdn>.<T>, #<imm> 3761 // 0010 0101 ..10 1001 110. .... .... .... 3762 // size<23:22> | opc<18:16> = 001 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> 3763 3764 USE(zn); 3765 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3766 VIXL_ASSERT(zd.Is(zn)); 3767 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3768 3769 Emit(UMAX_z_zi | SVESize(zd) | Rd(zd) | ImmUnsignedField<12, 5>(imm8)); 3770} 3771 3772void Assembler::umin(const ZRegister& zd, const ZRegister& zn, int imm8) { 3773 // UMIN <Zdn>.<T>, <Zdn>.<T>, #<imm> 3774 // 0010 0101 ..10 1011 110. .... .... .... 3775 // size<23:22> | opc<18:16> = 011 | o2<13> = 0 | imm8<12:5> | Zdn<4:0> 3776 3777 USE(zn); 3778 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3779 VIXL_ASSERT(zd.Is(zn)); 3780 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3781 3782 Emit(UMIN_z_zi | SVESize(zd) | Rd(zd) | ImmUnsignedField<12, 5>(imm8)); 3783} 3784 3785void Assembler::uqadd(const ZRegister& zd, 3786 const ZRegister& zn, 3787 int imm8, 3788 int shift) { 3789 // UQADD <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} 3790 // 0010 0101 ..10 0101 11.. .... .... .... 3791 // size<23:22> | opc<18:16> = 101 | sh<13> | imm8<12:5> | Zdn<4:0> 3792 3793 USE(zn); 3794 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3795 VIXL_ASSERT(zd.Is(zn)); 3796 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3797 3798 SVEIntAddSubtractImmUnpredicatedHelper(UQADD_z_zi, zd, imm8, shift); 3799} 3800 3801void Assembler::uqsub(const ZRegister& zd, 3802 const ZRegister& zn, 3803 int imm8, 3804 int shift) { 3805 // UQSUB <Zdn>.<T>, <Zdn>.<T>, #<imm>{, <shift>} 3806 // 0010 0101 ..10 0111 11.. .... .... .... 3807 // size<23:22> | opc<18:16> = 111 | sh<13> | imm8<12:5> | Zdn<4:0> 3808 3809 USE(zn); 3810 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 3811 VIXL_ASSERT(zd.Is(zn)); 3812 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 3813 3814 SVEIntAddSubtractImmUnpredicatedHelper(UQSUB_z_zi, zd, imm8, shift); 3815} 3816 3817// SVEMemLoad. 3818 3819void Assembler::SVELdSt1Helper(unsigned msize_in_bytes_log2, 3820 const ZRegister& zt, 3821 const PRegister& pg, 3822 const SVEMemOperand& addr, 3823 bool is_signed, 3824 Instr op) { 3825 VIXL_ASSERT(addr.IsContiguous()); 3826 3827 Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, 1, addr); 3828 Instr dtype = 3829 SVEDtype(msize_in_bytes_log2, zt.GetLaneSizeInBytesLog2(), is_signed); 3830 Emit(op | mem_op | dtype | Rt(zt) | PgLow8(pg)); 3831} 3832 3833void Assembler::SVELdSt234Helper(int num_regs, 3834 const ZRegister& zt1, 3835 const PRegister& pg, 3836 const SVEMemOperand& addr, 3837 Instr op) { 3838 VIXL_ASSERT((num_regs >= 2) && (num_regs <= 4)); 3839 3840 unsigned msize_in_bytes_log2 = zt1.GetLaneSizeInBytesLog2(); 3841 Instr num = (num_regs - 1) << 21; 3842 Instr msz = msize_in_bytes_log2 << 23; 3843 Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, num_regs, addr); 3844 Emit(op | mem_op | msz | num | Rt(zt1) | PgLow8(pg)); 3845} 3846 3847void Assembler::SVELd1Helper(unsigned msize_in_bytes_log2, 3848 const ZRegister& zt, 3849 const PRegisterZ& pg, 3850 const SVEMemOperand& addr, 3851 bool is_signed) { 3852 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2); 3853 if (is_signed) { 3854 // Sign-extension is only possible when the vector elements are larger than 3855 // the elements in memory. 3856 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2); 3857 } 3858 3859 if (addr.IsScatterGather()) { 3860 bool is_load = true; 3861 bool is_ff = false; 3862 SVEScatterGatherHelper(msize_in_bytes_log2, 3863 zt, 3864 pg, 3865 addr, 3866 is_load, 3867 is_signed, 3868 is_ff); 3869 return; 3870 } 3871 3872 Instr op = 0xffffffff; 3873 if (addr.IsScalarPlusImmediate()) { 3874 op = SVEContiguousLoad_ScalarPlusImmFixed; 3875 } else if (addr.IsScalarPlusScalar()) { 3876 // Rm must not be xzr. 3877 VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); 3878 op = SVEContiguousLoad_ScalarPlusScalarFixed; 3879 } else { 3880 VIXL_UNIMPLEMENTED(); 3881 } 3882 SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, is_signed, op); 3883} 3884 3885void Assembler::SVELdff1Helper(unsigned msize_in_bytes_log2, 3886 const ZRegister& zt, 3887 const PRegisterZ& pg, 3888 const SVEMemOperand& addr, 3889 bool is_signed) { 3890 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2); 3891 if (is_signed) { 3892 // Sign-extension is only possible when the vector elements are larger than 3893 // the elements in memory. 3894 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2); 3895 } 3896 3897 if (addr.IsScatterGather()) { 3898 bool is_load = true; 3899 bool is_ff = true; 3900 SVEScatterGatherHelper(msize_in_bytes_log2, 3901 zt, 3902 pg, 3903 addr, 3904 is_load, 3905 is_signed, 3906 is_ff); 3907 return; 3908 } 3909 3910 if (addr.IsPlainScalar()) { 3911 // SVEMemOperand(x0) is treated as a scalar-plus-immediate form ([x0, #0]). 3912 // In these instructions, we want to treat it as [x0, xzr]. 3913 SVEMemOperand addr_scalar_plus_scalar(addr.GetScalarBase(), xzr); 3914 // Guard against infinite recursion. 3915 VIXL_ASSERT(!addr_scalar_plus_scalar.IsPlainScalar()); 3916 SVELdff1Helper(msize_in_bytes_log2, 3917 zt, 3918 pg, 3919 addr_scalar_plus_scalar, 3920 is_signed); 3921 return; 3922 } 3923 3924 Instr op = 0xffffffff; 3925 if (addr.IsScalarPlusScalar()) { 3926 op = SVEContiguousFirstFaultLoad_ScalarPlusScalarFixed; 3927 } else { 3928 VIXL_UNIMPLEMENTED(); 3929 } 3930 SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, is_signed, op); 3931} 3932 3933void Assembler::SVEScatterGatherHelper(unsigned msize_in_bytes_log2, 3934 const ZRegister& zt, 3935 const PRegister& pg, 3936 const SVEMemOperand& addr, 3937 bool is_load, 3938 bool is_signed, 3939 bool is_first_fault) { 3940 VIXL_ASSERT(addr.IsScatterGather()); 3941 VIXL_ASSERT(zt.IsLaneSizeS() || zt.IsLaneSizeD()); 3942 VIXL_ASSERT(is_load || !is_first_fault); 3943 VIXL_ASSERT(is_load || !is_signed); 3944 3945 Instr op = 0xffffffff; 3946 if (addr.IsVectorPlusImmediate()) { 3947 VIXL_ASSERT(AreSameLaneSize(zt, addr.GetVectorBase())); 3948 if (is_load) { 3949 if (zt.IsLaneSizeS()) { 3950 op = SVE32BitGatherLoad_VectorPlusImmFixed; 3951 } else { 3952 op = SVE64BitGatherLoad_VectorPlusImmFixed; 3953 } 3954 } else { 3955 if (zt.IsLaneSizeS()) { 3956 op = SVE32BitScatterStore_VectorPlusImmFixed; 3957 } else { 3958 op = SVE64BitScatterStore_VectorPlusImmFixed; 3959 } 3960 } 3961 } else { 3962 VIXL_ASSERT(addr.IsScalarPlusVector()); 3963 VIXL_ASSERT(AreSameLaneSize(zt, addr.GetVectorOffset())); 3964 SVEOffsetModifier mod = addr.GetOffsetModifier(); 3965 if (zt.IsLaneSizeS()) { 3966 VIXL_ASSERT((mod == SVE_UXTW) || (mod == SVE_SXTW)); 3967 unsigned shift_amount = addr.GetShiftAmount(); 3968 if (shift_amount == 0) { 3969 if (is_load) { 3970 op = SVE32BitGatherLoad_ScalarPlus32BitUnscaledOffsetsFixed; 3971 } else { 3972 op = SVE32BitScatterStore_ScalarPlus32BitUnscaledOffsetsFixed; 3973 } 3974 } else if (shift_amount == 1) { 3975 VIXL_ASSERT(msize_in_bytes_log2 == kHRegSizeInBytesLog2); 3976 if (is_load) { 3977 op = SVE32BitGatherLoadHalfwords_ScalarPlus32BitScaledOffsetsFixed; 3978 } else { 3979 op = SVE32BitScatterStore_ScalarPlus32BitScaledOffsetsFixed; 3980 } 3981 } else { 3982 VIXL_ASSERT(shift_amount == 2); 3983 VIXL_ASSERT(msize_in_bytes_log2 == kSRegSizeInBytesLog2); 3984 if (is_load) { 3985 op = SVE32BitGatherLoadWords_ScalarPlus32BitScaledOffsetsFixed; 3986 } else { 3987 op = SVE32BitScatterStore_ScalarPlus32BitScaledOffsetsFixed; 3988 } 3989 } 3990 } else if (zt.IsLaneSizeD()) { 3991 switch (mod) { 3992 case NO_SVE_OFFSET_MODIFIER: 3993 if (is_load) { 3994 op = SVE64BitGatherLoad_ScalarPlus64BitUnscaledOffsetsFixed; 3995 } else { 3996 op = SVE64BitScatterStore_ScalarPlus64BitUnscaledOffsetsFixed; 3997 } 3998 break; 3999 case SVE_LSL: 4000 if (is_load) { 4001 op = SVE64BitGatherLoad_ScalarPlus64BitScaledOffsetsFixed; 4002 } else { 4003 op = SVE64BitScatterStore_ScalarPlus64BitScaledOffsetsFixed; 4004 } 4005 break; 4006 case SVE_UXTW: 4007 case SVE_SXTW: { 4008 unsigned shift_amount = addr.GetShiftAmount(); 4009 if (shift_amount == 0) { 4010 if (is_load) { 4011 op = 4012 SVE64BitGatherLoad_ScalarPlusUnpacked32BitUnscaledOffsetsFixed; 4013 } else { 4014 op = 4015 SVE64BitScatterStore_ScalarPlusUnpacked32BitUnscaledOffsetsFixed; 4016 } 4017 } else { 4018 VIXL_ASSERT(shift_amount == msize_in_bytes_log2); 4019 if (is_load) { 4020 op = SVE64BitGatherLoad_ScalarPlus32BitUnpackedScaledOffsetsFixed; 4021 } else { 4022 op = 4023 SVE64BitScatterStore_ScalarPlusUnpacked32BitScaledOffsetsFixed; 4024 } 4025 } 4026 break; 4027 } 4028 default: 4029 VIXL_UNIMPLEMENTED(); 4030 } 4031 } 4032 } 4033 4034 Instr mem_op = SVEMemOperandHelper(msize_in_bytes_log2, 1, addr, is_load); 4035 Instr msz = ImmUnsignedField<24, 23>(msize_in_bytes_log2); 4036 Instr u = (!is_load || is_signed) ? 0 : (1 << 14); 4037 Instr ff = is_first_fault ? (1 << 13) : 0; 4038 Emit(op | mem_op | msz | u | ff | Rt(zt) | PgLow8(pg)); 4039} 4040 4041void Assembler::SVELd234Helper(int num_regs, 4042 const ZRegister& zt1, 4043 const PRegisterZ& pg, 4044 const SVEMemOperand& addr) { 4045 if (addr.IsScalarPlusScalar()) { 4046 // Rm must not be xzr. 4047 VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); 4048 } 4049 4050 Instr op; 4051 if (addr.IsScalarPlusImmediate()) { 4052 op = SVELoadMultipleStructures_ScalarPlusImmFixed; 4053 } else if (addr.IsScalarPlusScalar()) { 4054 op = SVELoadMultipleStructures_ScalarPlusScalarFixed; 4055 } else { 4056 // These instructions don't support any other addressing modes. 4057 VIXL_ABORT(); 4058 } 4059 SVELdSt234Helper(num_regs, zt1, pg, addr, op); 4060} 4061 4062// SVEMemContiguousLoad. 4063 4064#define VIXL_DEFINE_LD1(MSZ, LANE_SIZE) \ 4065 void Assembler::ld1##MSZ(const ZRegister& zt, \ 4066 const PRegisterZ& pg, \ 4067 const SVEMemOperand& addr) { \ 4068 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 4069 SVELd1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, false); \ 4070 } 4071#define VIXL_DEFINE_LD2(MSZ, LANE_SIZE) \ 4072 void Assembler::ld2##MSZ(const ZRegister& zt1, \ 4073 const ZRegister& zt2, \ 4074 const PRegisterZ& pg, \ 4075 const SVEMemOperand& addr) { \ 4076 USE(zt2); \ 4077 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 4078 VIXL_ASSERT(AreConsecutive(zt1, zt2)); \ 4079 VIXL_ASSERT(AreSameFormat(zt1, zt2)); \ 4080 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ 4081 SVELd234Helper(2, zt1, pg, addr); \ 4082 } 4083#define VIXL_DEFINE_LD3(MSZ, LANE_SIZE) \ 4084 void Assembler::ld3##MSZ(const ZRegister& zt1, \ 4085 const ZRegister& zt2, \ 4086 const ZRegister& zt3, \ 4087 const PRegisterZ& pg, \ 4088 const SVEMemOperand& addr) { \ 4089 USE(zt2, zt3); \ 4090 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 4091 VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3)); \ 4092 VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3)); \ 4093 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ 4094 SVELd234Helper(3, zt1, pg, addr); \ 4095 } 4096#define VIXL_DEFINE_LD4(MSZ, LANE_SIZE) \ 4097 void Assembler::ld4##MSZ(const ZRegister& zt1, \ 4098 const ZRegister& zt2, \ 4099 const ZRegister& zt3, \ 4100 const ZRegister& zt4, \ 4101 const PRegisterZ& pg, \ 4102 const SVEMemOperand& addr) { \ 4103 USE(zt2, zt3, zt4); \ 4104 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 4105 VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3, zt4)); \ 4106 VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3, zt4)); \ 4107 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ 4108 SVELd234Helper(4, zt1, pg, addr); \ 4109 } 4110 4111VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD1) 4112VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD2) 4113VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD3) 4114VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LD4) 4115 4116#define VIXL_DEFINE_LD1S(MSZ, LANE_SIZE) \ 4117 void Assembler::ld1s##MSZ(const ZRegister& zt, \ 4118 const PRegisterZ& pg, \ 4119 const SVEMemOperand& addr) { \ 4120 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 4121 SVELd1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, true); \ 4122 } 4123VIXL_SVE_LOAD_STORE_SIGNED_VARIANT_LIST(VIXL_DEFINE_LD1S) 4124 4125// SVEMem32BitGatherAndUnsizedContiguous. 4126 4127void Assembler::SVELd1BroadcastHelper(unsigned msize_in_bytes_log2, 4128 const ZRegister& zt, 4129 const PRegisterZ& pg, 4130 const SVEMemOperand& addr, 4131 bool is_signed) { 4132 VIXL_ASSERT(addr.IsScalarPlusImmediate()); 4133 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() >= msize_in_bytes_log2); 4134 if (is_signed) { 4135 // Sign-extension is only possible when the vector elements are larger than 4136 // the elements in memory. 4137 VIXL_ASSERT(zt.GetLaneSizeInBytesLog2() != msize_in_bytes_log2); 4138 } 4139 4140 int64_t imm = addr.GetImmediateOffset(); 4141 int divisor = 1 << msize_in_bytes_log2; 4142 VIXL_ASSERT(imm % divisor == 0); 4143 Instr dtype = SVEDtypeSplit(msize_in_bytes_log2, 4144 zt.GetLaneSizeInBytesLog2(), 4145 is_signed); 4146 4147 Emit(SVELoadAndBroadcastElementFixed | dtype | RnSP(addr.GetScalarBase()) | 4148 ImmUnsignedField<21, 16>(imm / divisor) | Rt(zt) | PgLow8(pg)); 4149} 4150 4151// This prototype maps to 4 instruction encodings: 4152// LD1RB_z_p_bi_u16 4153// LD1RB_z_p_bi_u32 4154// LD1RB_z_p_bi_u64 4155// LD1RB_z_p_bi_u8 4156void Assembler::ld1rb(const ZRegister& zt, 4157 const PRegisterZ& pg, 4158 const SVEMemOperand& addr) { 4159 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4160 4161 SVELd1BroadcastHelper(kBRegSizeInBytesLog2, zt, pg, addr, false); 4162} 4163 4164// This prototype maps to 3 instruction encodings: 4165// LD1RH_z_p_bi_u16 4166// LD1RH_z_p_bi_u32 4167// LD1RH_z_p_bi_u64 4168void Assembler::ld1rh(const ZRegister& zt, 4169 const PRegisterZ& pg, 4170 const SVEMemOperand& addr) { 4171 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4172 4173 SVELd1BroadcastHelper(kHRegSizeInBytesLog2, zt, pg, addr, false); 4174} 4175 4176// This prototype maps to 2 instruction encodings: 4177// LD1RW_z_p_bi_u32 4178// LD1RW_z_p_bi_u64 4179void Assembler::ld1rw(const ZRegister& zt, 4180 const PRegisterZ& pg, 4181 const SVEMemOperand& addr) { 4182 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4183 4184 SVELd1BroadcastHelper(kSRegSizeInBytesLog2, zt, pg, addr, false); 4185} 4186 4187void Assembler::ld1rd(const ZRegister& zt, 4188 const PRegisterZ& pg, 4189 const SVEMemOperand& addr) { 4190 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4191 4192 SVELd1BroadcastHelper(kDRegSizeInBytesLog2, zt, pg, addr, false); 4193} 4194 4195// This prototype maps to 3 instruction encodings: 4196// LD1RSB_z_p_bi_s16 4197// LD1RSB_z_p_bi_s32 4198// LD1RSB_z_p_bi_s64 4199void Assembler::ld1rsb(const ZRegister& zt, 4200 const PRegisterZ& pg, 4201 const SVEMemOperand& addr) { 4202 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4203 4204 SVELd1BroadcastHelper(kBRegSizeInBytesLog2, zt, pg, addr, true); 4205} 4206 4207// This prototype maps to 2 instruction encodings: 4208// LD1RSH_z_p_bi_s32 4209// LD1RSH_z_p_bi_s64 4210void Assembler::ld1rsh(const ZRegister& zt, 4211 const PRegisterZ& pg, 4212 const SVEMemOperand& addr) { 4213 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4214 4215 SVELd1BroadcastHelper(kHRegSizeInBytesLog2, zt, pg, addr, true); 4216} 4217 4218void Assembler::ld1rsw(const ZRegister& zt, 4219 const PRegisterZ& pg, 4220 const SVEMemOperand& addr) { 4221 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4222 4223 SVELd1BroadcastHelper(kWRegSizeInBytesLog2, zt, pg, addr, true); 4224} 4225 4226void Assembler::ldr(const CPURegister& rt, const SVEMemOperand& addr) { 4227 // LDR <Pt/Zt>, [<Xn|SP>{, #<imm>, MUL VL}] 4228 4229 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4230 VIXL_ASSERT(rt.IsPRegister() || rt.IsZRegister()); 4231 VIXL_ASSERT(addr.IsPlainScalar() || 4232 (addr.IsScalarPlusImmediate() && 4233 (addr.GetOffsetModifier() == SVE_MUL_VL))); 4234 int64_t imm9 = addr.GetImmediateOffset(); 4235 VIXL_ASSERT(IsInt9(imm9)); 4236 Instr imm9l = ExtractUnsignedBitfield32(2, 0, imm9) << 10; 4237 Instr imm9h = ExtractUnsignedBitfield32(8, 3, imm9) << 16; 4238 4239 Instr op = LDR_z_bi; 4240 if (rt.IsPRegister()) { 4241 op = LDR_p_bi; 4242 } 4243 Emit(op | Rt(rt) | RnSP(addr.GetScalarBase()) | imm9h | imm9l); 4244} 4245 4246// SVEMem64BitGather. 4247 4248// This prototype maps to 3 instruction encodings: 4249// LDFF1B_z_p_bz_d_64_unscaled 4250// LDFF1B_z_p_bz_d_x32_unscaled 4251void Assembler::ldff1b(const ZRegister& zt, 4252 const PRegisterZ& pg, 4253 const Register& xn, 4254 const ZRegister& zm) { 4255 // LDFF1B { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D] 4256 // 1100 0100 010. .... 111. .... .... .... 4257 // msz<24:23> = 00 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5> 4258 // | Zt<4:0> 4259 4260 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4261 4262 Emit(LDFF1B_z_p_bz_d_64_unscaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); 4263} 4264 4265// This prototype maps to 2 instruction encodings: 4266// LDFF1B_z_p_ai_d 4267// LDFF1B_z_p_ai_s 4268void Assembler::ldff1b(const ZRegister& zt, 4269 const PRegisterZ& pg, 4270 const ZRegister& zn, 4271 int imm5) { 4272 // LDFF1B { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] 4273 // 1100 0100 001. .... 111. .... .... .... 4274 // msz<24:23> = 00 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | 4275 // Zn<9:5> | Zt<4:0> 4276 4277 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4278 4279 Emit(LDFF1B_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5)); 4280} 4281 4282// This prototype maps to 4 instruction encodings: 4283// LDFF1D_z_p_bz_d_64_scaled 4284// LDFF1D_z_p_bz_d_64_unscaled 4285// LDFF1D_z_p_bz_d_x32_scaled 4286// LDFF1D_z_p_bz_d_x32_unscaled 4287void Assembler::ldff1d(const ZRegister& zt, 4288 const PRegisterZ& pg, 4289 const Register& xn, 4290 const ZRegister& zm) { 4291 // LDFF1D { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #3] 4292 // 1100 0101 111. .... 111. .... .... .... 4293 // msz<24:23> = 11 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5> 4294 // | Zt<4:0> 4295 4296 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4297 4298 Emit(LDFF1D_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); 4299} 4300 4301void Assembler::ldff1d(const ZRegister& zt, 4302 const PRegisterZ& pg, 4303 const ZRegister& zn, 4304 int imm5) { 4305 // LDFF1D { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] 4306 // 1100 0101 101. .... 111. .... .... .... 4307 // msz<24:23> = 11 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | 4308 // Zn<9:5> | Zt<4:0> 4309 4310 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4311 4312 Emit(LDFF1D_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5)); 4313} 4314 4315// This prototype maps to 6 instruction encodings: 4316// LDFF1H_z_p_bz_d_64_scaled 4317// LDFF1H_z_p_bz_d_64_unscaled 4318// LDFF1H_z_p_bz_d_x32_scaled 4319// LDFF1H_z_p_bz_d_x32_unscaled 4320void Assembler::ldff1h(const ZRegister& zt, 4321 const PRegisterZ& pg, 4322 const Register& xn, 4323 const ZRegister& zm) { 4324 // LDFF1H { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #1] 4325 // 1100 0100 111. .... 111. .... .... .... 4326 // msz<24:23> = 01 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5> 4327 // | Zt<4:0> 4328 4329 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4330 4331 Emit(LDFF1H_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); 4332} 4333 4334// This prototype maps to 2 instruction encodings: 4335// LDFF1H_z_p_ai_d 4336// LDFF1H_z_p_ai_s 4337void Assembler::ldff1h(const ZRegister& zt, 4338 const PRegisterZ& pg, 4339 const ZRegister& zn, 4340 int imm5) { 4341 // LDFF1H { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] 4342 // 1100 0100 101. .... 111. .... .... .... 4343 // msz<24:23> = 01 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | 4344 // Zn<9:5> | Zt<4:0> 4345 4346 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4347 4348 Emit(LDFF1H_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5)); 4349} 4350 4351// This prototype maps to 3 instruction encodings: 4352// LDFF1SB_z_p_bz_d_64_unscaled 4353// LDFF1SB_z_p_bz_d_x32_unscaled 4354void Assembler::ldff1sb(const ZRegister& zt, 4355 const PRegisterZ& pg, 4356 const Register& xn, 4357 const ZRegister& zm) { 4358 // LDFF1SB { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D] 4359 // 1100 0100 010. .... 101. .... .... .... 4360 // msz<24:23> = 00 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5> 4361 // | Zt<4:0> 4362 4363 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4364 4365 Emit(LDFF1SB_z_p_bz_d_64_unscaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); 4366} 4367 4368// This prototype maps to 2 instruction encodings: 4369// LDFF1SB_z_p_ai_d 4370// LDFF1SB_z_p_ai_s 4371void Assembler::ldff1sb(const ZRegister& zt, 4372 const PRegisterZ& pg, 4373 const ZRegister& zn, 4374 int imm5) { 4375 // LDFF1SB { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] 4376 // 1100 0100 001. .... 101. .... .... .... 4377 // msz<24:23> = 00 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | 4378 // Zn<9:5> | Zt<4:0> 4379 4380 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4381 4382 Emit(LDFF1SB_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | 4383 ImmField<20, 16>(imm5)); 4384} 4385 4386// This prototype maps to 6 instruction encodings: 4387// LDFF1SH_z_p_bz_d_64_scaled 4388// LDFF1SH_z_p_bz_d_64_unscaled 4389// LDFF1SH_z_p_bz_d_x32_scaled 4390// LDFF1SH_z_p_bz_d_x32_unscaled 4391void Assembler::ldff1sh(const ZRegister& zt, 4392 const PRegisterZ& pg, 4393 const Register& xn, 4394 const ZRegister& zm) { 4395 // LDFF1SH { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #1] 4396 // 1100 0100 111. .... 101. .... .... .... 4397 // msz<24:23> = 01 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5> 4398 // | Zt<4:0> 4399 4400 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4401 4402 Emit(LDFF1SH_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); 4403} 4404 4405// This prototype maps to 2 instruction encodings: 4406// LDFF1SH_z_p_ai_d 4407// LDFF1SH_z_p_ai_s 4408void Assembler::ldff1sh(const ZRegister& zt, 4409 const PRegisterZ& pg, 4410 const ZRegister& zn, 4411 int imm5) { 4412 // LDFF1SH { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] 4413 // 1100 0100 101. .... 101. .... .... .... 4414 // msz<24:23> = 01 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | 4415 // Zn<9:5> | Zt<4:0> 4416 4417 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4418 4419 Emit(LDFF1SH_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | 4420 ImmField<20, 16>(imm5)); 4421} 4422 4423// This prototype maps to 4 instruction encodings: 4424// LDFF1SW_z_p_bz_d_64_scaled 4425// LDFF1SW_z_p_bz_d_64_unscaled 4426// LDFF1SW_z_p_bz_d_x32_scaled 4427// LDFF1SW_z_p_bz_d_x32_unscaled 4428void Assembler::ldff1sw(const ZRegister& zt, 4429 const PRegisterZ& pg, 4430 const Register& xn, 4431 const ZRegister& zm) { 4432 // LDFF1SW { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #2] 4433 // 1100 0101 011. .... 101. .... .... .... 4434 // msz<24:23> = 10 | Zm<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | Rn<9:5> 4435 // | Zt<4:0> 4436 4437 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4438 4439 Emit(LDFF1SW_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); 4440} 4441 4442void Assembler::ldff1sw(const ZRegister& zt, 4443 const PRegisterZ& pg, 4444 const ZRegister& zn, 4445 int imm5) { 4446 // LDFF1SW { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] 4447 // 1100 0101 001. .... 101. .... .... .... 4448 // msz<24:23> = 10 | imm5<20:16> | U<14> = 0 | ff<13> = 1 | Pg<12:10> | 4449 // Zn<9:5> | Zt<4:0> 4450 4451 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4452 4453 Emit(LDFF1SW_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | 4454 ImmField<20, 16>(imm5)); 4455} 4456 4457// This prototype maps to 6 instruction encodings: 4458// LDFF1W_z_p_bz_d_64_scaled 4459// LDFF1W_z_p_bz_d_64_unscaled 4460// LDFF1W_z_p_bz_d_x32_scaled 4461// LDFF1W_z_p_bz_d_x32_unscaled 4462void Assembler::ldff1w(const ZRegister& zt, 4463 const PRegisterZ& pg, 4464 const Register& xn, 4465 const ZRegister& zm) { 4466 // LDFF1W { <Zt>.D }, <Pg>/Z, [<Xn|SP>, <Zm>.D, LSL #2] 4467 // 1100 0101 011. .... 111. .... .... .... 4468 // msz<24:23> = 10 | Zm<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | Rn<9:5> 4469 // | Zt<4:0> 4470 4471 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4472 4473 Emit(LDFF1W_z_p_bz_d_64_scaled | Rt(zt) | PgLow8(pg) | RnSP(xn) | Rm(zm)); 4474} 4475 4476// This prototype maps to 2 instruction encodings: 4477// LDFF1W_z_p_ai_d 4478// LDFF1W_z_p_ai_s 4479void Assembler::ldff1w(const ZRegister& zt, 4480 const PRegisterZ& pg, 4481 const ZRegister& zn, 4482 int imm5) { 4483 // LDFF1W { <Zt>.D }, <Pg>/Z, [<Zn>.D{, #<imm>}] 4484 // 1100 0101 001. .... 111. .... .... .... 4485 // msz<24:23> = 10 | imm5<20:16> | U<14> = 1 | ff<13> = 1 | Pg<12:10> | 4486 // Zn<9:5> | Zt<4:0> 4487 4488 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4489 4490 Emit(LDFF1W_z_p_ai_d | Rt(zt) | PgLow8(pg) | Rn(zn) | ImmField<20, 16>(imm5)); 4491} 4492 4493void Assembler::SVEGatherPrefetchVectorPlusImmediateHelper( 4494 PrefetchOperation prfop, 4495 const PRegister& pg, 4496 const SVEMemOperand& addr, 4497 int prefetch_size) { 4498 VIXL_ASSERT(addr.IsVectorPlusImmediate()); 4499 ZRegister zn = addr.GetVectorBase(); 4500 VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD()); 4501 4502 Instr op = 0xffffffff; 4503 switch (prefetch_size) { 4504 case kBRegSize: 4505 op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFB_i_p_ai_s) 4506 : static_cast<Instr>(PRFB_i_p_ai_d); 4507 break; 4508 case kHRegSize: 4509 op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFH_i_p_ai_s) 4510 : static_cast<Instr>(PRFH_i_p_ai_d); 4511 break; 4512 case kSRegSize: 4513 op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFW_i_p_ai_s) 4514 : static_cast<Instr>(PRFW_i_p_ai_d); 4515 break; 4516 case kDRegSize: 4517 op = zn.IsLaneSizeS() ? static_cast<Instr>(PRFD_i_p_ai_s) 4518 : static_cast<Instr>(PRFD_i_p_ai_d); 4519 break; 4520 default: 4521 VIXL_UNIMPLEMENTED(); 4522 break; 4523 } 4524 4525 int64_t imm5 = addr.GetImmediateOffset(); 4526 Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | Rn(zn) | 4527 ImmUnsignedField<20, 16>(imm5)); 4528} 4529 4530void Assembler::SVEGatherPrefetchScalarPlusImmediateHelper( 4531 PrefetchOperation prfop, 4532 const PRegister& pg, 4533 const SVEMemOperand& addr, 4534 int prefetch_size) { 4535 VIXL_ASSERT(addr.IsScalarPlusImmediate()); 4536 int64_t imm6 = addr.GetImmediateOffset(); 4537 4538 Instr op = 0xffffffff; 4539 switch (prefetch_size) { 4540 case kBRegSize: 4541 op = PRFB_i_p_bi_s; 4542 break; 4543 case kHRegSize: 4544 op = PRFH_i_p_bi_s; 4545 break; 4546 case kSRegSize: 4547 op = PRFW_i_p_bi_s; 4548 break; 4549 case kDRegSize: 4550 op = PRFD_i_p_bi_s; 4551 break; 4552 default: 4553 VIXL_UNIMPLEMENTED(); 4554 break; 4555 } 4556 4557 Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | 4558 RnSP(addr.GetScalarBase()) | ImmField<21, 16>(imm6)); 4559} 4560 4561void Assembler::SVEContiguousPrefetchScalarPlusScalarHelper( 4562 PrefetchOperation prfop, 4563 const PRegister& pg, 4564 const SVEMemOperand& addr, 4565 int prefetch_size) { 4566 VIXL_ASSERT(addr.IsScalarPlusScalar()); 4567 Instr op = 0xffffffff; 4568 4569 switch (prefetch_size) { 4570 case kBRegSize: 4571 VIXL_ASSERT(addr.GetOffsetModifier() == NO_SVE_OFFSET_MODIFIER); 4572 op = PRFB_i_p_br_s; 4573 break; 4574 case kHRegSize: 4575 VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL); 4576 VIXL_ASSERT(addr.GetShiftAmount() == kHRegSizeInBytesLog2); 4577 op = PRFH_i_p_br_s; 4578 break; 4579 case kSRegSize: 4580 VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL); 4581 VIXL_ASSERT(addr.GetShiftAmount() == kSRegSizeInBytesLog2); 4582 op = PRFW_i_p_br_s; 4583 break; 4584 case kDRegSize: 4585 VIXL_ASSERT(addr.GetOffsetModifier() == SVE_LSL); 4586 VIXL_ASSERT(addr.GetShiftAmount() == kDRegSizeInBytesLog2); 4587 op = PRFD_i_p_br_s; 4588 break; 4589 default: 4590 VIXL_UNIMPLEMENTED(); 4591 break; 4592 } 4593 4594 VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); 4595 Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | 4596 RnSP(addr.GetScalarBase()) | Rm(addr.GetScalarOffset())); 4597} 4598 4599void Assembler::SVEContiguousPrefetchScalarPlusVectorHelper( 4600 PrefetchOperation prfop, 4601 const PRegister& pg, 4602 const SVEMemOperand& addr, 4603 int prefetch_size) { 4604 VIXL_ASSERT(addr.IsScalarPlusVector()); 4605 ZRegister zm = addr.GetVectorOffset(); 4606 SVEOffsetModifier mod = addr.GetOffsetModifier(); 4607 4608 // All prefetch scalar-plus-vector addressing modes use a shift corresponding 4609 // to the element size. 4610 switch (prefetch_size) { 4611 case kBRegSize: 4612 VIXL_ASSERT(addr.GetShiftAmount() == kBRegSizeInBytesLog2); 4613 break; 4614 case kHRegSize: 4615 VIXL_ASSERT(addr.GetShiftAmount() == kHRegSizeInBytesLog2); 4616 break; 4617 case kSRegSize: 4618 VIXL_ASSERT(addr.GetShiftAmount() == kSRegSizeInBytesLog2); 4619 break; 4620 case kDRegSize: 4621 VIXL_ASSERT(addr.GetShiftAmount() == kDRegSizeInBytesLog2); 4622 break; 4623 default: 4624 VIXL_UNIMPLEMENTED(); 4625 break; 4626 } 4627 4628 Instr sx = 0; 4629 Instr op = 0xffffffff; 4630 if ((mod == NO_SVE_OFFSET_MODIFIER) || (mod == SVE_LSL)) { 4631 VIXL_ASSERT(zm.IsLaneSizeD()); 4632 4633 switch (prefetch_size) { 4634 case kBRegSize: 4635 VIXL_ASSERT(mod == NO_SVE_OFFSET_MODIFIER); 4636 op = PRFB_i_p_bz_d_64_scaled; 4637 break; 4638 case kHRegSize: 4639 VIXL_ASSERT(mod == SVE_LSL); 4640 op = PRFH_i_p_bz_d_64_scaled; 4641 break; 4642 case kSRegSize: 4643 VIXL_ASSERT(mod == SVE_LSL); 4644 op = PRFW_i_p_bz_d_64_scaled; 4645 break; 4646 case kDRegSize: 4647 VIXL_ASSERT(mod == SVE_LSL); 4648 op = PRFD_i_p_bz_d_64_scaled; 4649 break; 4650 default: 4651 VIXL_UNIMPLEMENTED(); 4652 break; 4653 } 4654 } else { 4655 VIXL_ASSERT((mod == SVE_SXTW) || (mod == SVE_UXTW)); 4656 VIXL_ASSERT(zm.IsLaneSizeS() || zm.IsLaneSizeD()); 4657 4658 switch (prefetch_size) { 4659 case kBRegSize: 4660 op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFB_i_p_bz_s_x32_scaled) 4661 : static_cast<Instr>(PRFB_i_p_bz_d_x32_scaled); 4662 break; 4663 case kHRegSize: 4664 op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFH_i_p_bz_s_x32_scaled) 4665 : static_cast<Instr>(PRFH_i_p_bz_d_x32_scaled); 4666 break; 4667 case kSRegSize: 4668 op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFW_i_p_bz_s_x32_scaled) 4669 : static_cast<Instr>(PRFW_i_p_bz_d_x32_scaled); 4670 break; 4671 case kDRegSize: 4672 op = zm.IsLaneSizeS() ? static_cast<Instr>(PRFD_i_p_bz_s_x32_scaled) 4673 : static_cast<Instr>(PRFD_i_p_bz_d_x32_scaled); 4674 break; 4675 default: 4676 VIXL_UNIMPLEMENTED(); 4677 break; 4678 } 4679 4680 if (mod == SVE_SXTW) { 4681 sx = 1 << 22; 4682 } 4683 } 4684 4685 Emit(op | SVEImmPrefetchOperation(prfop) | PgLow8(pg) | sx | 4686 RnSP(addr.GetScalarBase()) | Rm(zm)); 4687} 4688 4689void Assembler::SVEPrefetchHelper(PrefetchOperation prfop, 4690 const PRegister& pg, 4691 const SVEMemOperand& addr, 4692 int prefetch_size) { 4693 if (addr.IsVectorPlusImmediate()) { 4694 // For example: 4695 // [z0.s, #0] 4696 SVEGatherPrefetchVectorPlusImmediateHelper(prfop, pg, addr, prefetch_size); 4697 4698 } else if (addr.IsScalarPlusImmediate()) { 4699 // For example: 4700 // [x0, #42, mul vl] 4701 SVEGatherPrefetchScalarPlusImmediateHelper(prfop, pg, addr, prefetch_size); 4702 4703 } else if (addr.IsScalarPlusVector()) { 4704 // For example: 4705 // [x0, z0.s, sxtw] 4706 SVEContiguousPrefetchScalarPlusVectorHelper(prfop, pg, addr, prefetch_size); 4707 4708 } else if (addr.IsScalarPlusScalar()) { 4709 // For example: 4710 // [x0, x1] 4711 SVEContiguousPrefetchScalarPlusScalarHelper(prfop, pg, addr, prefetch_size); 4712 4713 } else { 4714 VIXL_UNIMPLEMENTED(); 4715 } 4716} 4717 4718void Assembler::prfb(PrefetchOperation prfop, 4719 const PRegister& pg, 4720 const SVEMemOperand& addr) { 4721 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4722 SVEPrefetchHelper(prfop, pg, addr, kBRegSize); 4723} 4724 4725void Assembler::prfd(PrefetchOperation prfop, 4726 const PRegister& pg, 4727 const SVEMemOperand& addr) { 4728 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4729 SVEPrefetchHelper(prfop, pg, addr, kDRegSize); 4730} 4731 4732void Assembler::prfh(PrefetchOperation prfop, 4733 const PRegister& pg, 4734 const SVEMemOperand& addr) { 4735 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4736 SVEPrefetchHelper(prfop, pg, addr, kHRegSize); 4737} 4738 4739void Assembler::prfw(PrefetchOperation prfop, 4740 const PRegister& pg, 4741 const SVEMemOperand& addr) { 4742 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4743 SVEPrefetchHelper(prfop, pg, addr, kSRegSize); 4744} 4745 4746void Assembler::SVELd1St1ScaImmHelper(const ZRegister& zt, 4747 const PRegister& pg, 4748 const SVEMemOperand& addr, 4749 Instr regoffset_op, 4750 Instr immoffset_op, 4751 int imm_divisor) { 4752 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4753 VIXL_ASSERT(addr.IsScalarPlusScalar() || addr.IsScalarPlusImmediate()); 4754 4755 Instr op; 4756 if (addr.IsScalarPlusScalar()) { 4757 op = regoffset_op | Rm(addr.GetScalarOffset()); 4758 } else { 4759 int64_t imm = addr.GetImmediateOffset(); 4760 VIXL_ASSERT(((imm % imm_divisor) == 0) && IsInt4(imm / imm_divisor)); 4761 op = immoffset_op | ImmField<19, 16>(imm / imm_divisor); 4762 } 4763 Emit(op | Rt(zt) | PgLow8(pg) | RnSP(addr.GetScalarBase())); 4764} 4765 4766void Assembler::SVELd1VecScaHelper(const ZRegister& zt, 4767 const PRegister& pg, 4768 const SVEMemOperand& addr, 4769 uint32_t msize_bytes_log2, 4770 bool is_signed) { 4771 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 4772 VIXL_ASSERT(addr.IsVectorPlusScalar()); 4773 ZRegister zn = addr.GetVectorBase(); 4774 VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD()); 4775 VIXL_ASSERT(AreSameLaneSize(zn, zt)); 4776 4777 uint32_t esize = zn.GetLaneSizeInBytesLog2(); 4778 uint32_t b14_13 = 0; 4779 if (!is_signed) b14_13 = zn.IsLaneSizeS() ? 0x1 : 0x2; 4780 4781 Instr op = 0x04008000; // LDNT1 with vector plus scalar addressing mode. 4782 op |= (esize << 30) | (msize_bytes_log2 << 23) | (b14_13 << 13); 4783 Emit(op | Rt(zt) | PgLow8(pg) | 4784 SVEMemOperandHelper(msize_bytes_log2, 1, addr, true)); 4785} 4786 4787void Assembler::SVESt1VecScaHelper(const ZRegister& zt, 4788 const PRegister& pg, 4789 const SVEMemOperand& addr, 4790 uint32_t msize_bytes_log2) { 4791 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 4792 VIXL_ASSERT(addr.IsVectorPlusScalar()); 4793 ZRegister zn = addr.GetVectorBase(); 4794 VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD()); 4795 VIXL_ASSERT(AreSameLaneSize(zn, zt)); 4796 4797 uint32_t bit22 = zn.IsLaneSizeS() ? (1 << 22) : 0; 4798 Instr op = 0xe4002000; // STNT1 with vector plus scalar addressing mode. 4799 op |= bit22 | (msize_bytes_log2 << 23); 4800 Emit(op | Rt(zt) | PgLow8(pg) | 4801 SVEMemOperandHelper(msize_bytes_log2, 1, addr, true)); 4802} 4803 4804#define VIXL_SVE_LD1R_LIST(V) \ 4805 V(qb, 0, B, LD1RQB_z_p_br_contiguous, LD1RQB_z_p_bi_u8, 16) \ 4806 V(qh, 1, H, LD1RQH_z_p_br_contiguous, LD1RQH_z_p_bi_u16, 16) \ 4807 V(qw, 2, S, LD1RQW_z_p_br_contiguous, LD1RQW_z_p_bi_u32, 16) \ 4808 V(qd, 3, D, LD1RQD_z_p_br_contiguous, LD1RQD_z_p_bi_u64, 16) \ 4809 V(ob, 0, B, 0xa4200000, 0xa4202000, 32) \ 4810 V(oh, 1, H, 0xa4a00000, 0xa4a02000, 32) \ 4811 V(ow, 2, S, 0xa5200000, 0xa5202000, 32) \ 4812 V(od, 3, D, 0xa5a00000, 0xa5a02000, 32) 4813 4814#define VIXL_DEFINE_ASM_FUNC(FN, SH, SZ, SCA, IMM, BYTES) \ 4815 void Assembler::ld1r##FN(const ZRegister& zt, \ 4816 const PRegisterZ& pg, \ 4817 const SVEMemOperand& addr) { \ 4818 VIXL_ASSERT((BYTES == 16) || \ 4819 ((BYTES == 32) && (CPUHas(CPUFeatures::kSVEF64MM)))); \ 4820 VIXL_ASSERT(addr.IsScalarPlusImmediate() || addr.IsEquivalentToLSL(SH)); \ 4821 VIXL_ASSERT(zt.IsLaneSize##SZ()); \ 4822 SVELd1St1ScaImmHelper(zt, pg, addr, SCA, IMM, BYTES); \ 4823 } 4824VIXL_SVE_LD1R_LIST(VIXL_DEFINE_ASM_FUNC) 4825#undef VIXL_DEFINE_ASM_FUNC 4826#undef VIXL_SVE_LD1R_LIST 4827 4828#define VIXL_DEFINE_LDFF1(MSZ, LANE_SIZE) \ 4829 void Assembler::ldff1##MSZ(const ZRegister& zt, \ 4830 const PRegisterZ& pg, \ 4831 const SVEMemOperand& addr) { \ 4832 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 4833 SVELdff1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, false); \ 4834 } 4835VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_LDFF1) 4836 4837#define VIXL_DEFINE_LDFF1S(MSZ, LANE_SIZE) \ 4838 void Assembler::ldff1s##MSZ(const ZRegister& zt, \ 4839 const PRegisterZ& pg, \ 4840 const SVEMemOperand& addr) { \ 4841 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 4842 SVELdff1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr, true); \ 4843 } 4844VIXL_SVE_LOAD_STORE_SIGNED_VARIANT_LIST(VIXL_DEFINE_LDFF1S) 4845 4846void Assembler::ldnf1b(const ZRegister& zt, 4847 const PRegisterZ& pg, 4848 const SVEMemOperand& addr) { 4849 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4850 VIXL_ASSERT(addr.IsPlainRegister() || 4851 (addr.IsScalarPlusImmediate() && 4852 (addr.GetOffsetModifier() == SVE_MUL_VL))); 4853 4854 SVELdSt1Helper(0, 4855 zt, 4856 pg, 4857 addr, 4858 /* is_signed = */ false, 4859 SVEContiguousNonFaultLoad_ScalarPlusImmFixed); 4860} 4861 4862void Assembler::ldnf1d(const ZRegister& zt, 4863 const PRegisterZ& pg, 4864 const SVEMemOperand& addr) { 4865 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4866 VIXL_ASSERT(addr.IsPlainRegister() || 4867 (addr.IsScalarPlusImmediate() && 4868 (addr.GetOffsetModifier() == SVE_MUL_VL))); 4869 4870 SVELdSt1Helper(3, 4871 zt, 4872 pg, 4873 addr, 4874 /* is_signed = */ false, 4875 SVEContiguousNonFaultLoad_ScalarPlusImmFixed); 4876} 4877 4878void Assembler::ldnf1h(const ZRegister& zt, 4879 const PRegisterZ& pg, 4880 const SVEMemOperand& addr) { 4881 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4882 VIXL_ASSERT(addr.IsPlainRegister() || 4883 (addr.IsScalarPlusImmediate() && 4884 (addr.GetOffsetModifier() == SVE_MUL_VL))); 4885 4886 SVELdSt1Helper(1, 4887 zt, 4888 pg, 4889 addr, 4890 /* is_signed = */ false, 4891 SVEContiguousNonFaultLoad_ScalarPlusImmFixed); 4892} 4893 4894void Assembler::ldnf1sb(const ZRegister& zt, 4895 const PRegisterZ& pg, 4896 const SVEMemOperand& addr) { 4897 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4898 VIXL_ASSERT(addr.IsPlainRegister() || 4899 (addr.IsScalarPlusImmediate() && 4900 (addr.GetOffsetModifier() == SVE_MUL_VL))); 4901 4902 SVELdSt1Helper(0, 4903 zt, 4904 pg, 4905 addr, 4906 /* is_signed = */ true, 4907 SVEContiguousNonFaultLoad_ScalarPlusImmFixed); 4908} 4909 4910void Assembler::ldnf1sh(const ZRegister& zt, 4911 const PRegisterZ& pg, 4912 const SVEMemOperand& addr) { 4913 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4914 VIXL_ASSERT(addr.IsPlainRegister() || 4915 (addr.IsScalarPlusImmediate() && 4916 (addr.GetOffsetModifier() == SVE_MUL_VL))); 4917 4918 SVELdSt1Helper(1, 4919 zt, 4920 pg, 4921 addr, 4922 /* is_signed = */ true, 4923 SVEContiguousNonFaultLoad_ScalarPlusImmFixed); 4924} 4925 4926void Assembler::ldnf1sw(const ZRegister& zt, 4927 const PRegisterZ& pg, 4928 const SVEMemOperand& addr) { 4929 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4930 VIXL_ASSERT(addr.IsPlainRegister() || 4931 (addr.IsScalarPlusImmediate() && 4932 (addr.GetOffsetModifier() == SVE_MUL_VL))); 4933 4934 SVELdSt1Helper(2, 4935 zt, 4936 pg, 4937 addr, 4938 /* is_signed = */ true, 4939 SVEContiguousNonFaultLoad_ScalarPlusImmFixed); 4940} 4941 4942void Assembler::ldnf1w(const ZRegister& zt, 4943 const PRegisterZ& pg, 4944 const SVEMemOperand& addr) { 4945 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 4946 VIXL_ASSERT(addr.IsPlainRegister() || 4947 (addr.IsScalarPlusImmediate() && 4948 (addr.GetOffsetModifier() == SVE_MUL_VL))); 4949 4950 SVELdSt1Helper(2, 4951 zt, 4952 pg, 4953 addr, 4954 /* is_signed = */ false, 4955 SVEContiguousNonFaultLoad_ScalarPlusImmFixed); 4956} 4957 4958void Assembler::ldnt1b(const ZRegister& zt, 4959 const PRegisterZ& pg, 4960 const SVEMemOperand& addr) { 4961 VIXL_ASSERT(addr.IsPlainScalar() || 4962 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || 4963 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(0)) || 4964 (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2))); 4965 if (addr.IsVectorPlusScalar()) { 4966 SVELd1VecScaHelper(zt, pg, addr, 0, /* is_signed = */ false); 4967 } else { 4968 SVELd1St1ScaImmHelper(zt, 4969 pg, 4970 addr, 4971 LDNT1B_z_p_br_contiguous, 4972 LDNT1B_z_p_bi_contiguous); 4973 } 4974} 4975 4976void Assembler::ldnt1d(const ZRegister& zt, 4977 const PRegisterZ& pg, 4978 const SVEMemOperand& addr) { 4979 VIXL_ASSERT(addr.IsPlainScalar() || 4980 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || 4981 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(3)) || 4982 (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2))); 4983 if (addr.IsVectorPlusScalar()) { 4984 SVELd1VecScaHelper(zt, pg, addr, 3, /* is_signed = */ false); 4985 } else { 4986 SVELd1St1ScaImmHelper(zt, 4987 pg, 4988 addr, 4989 LDNT1D_z_p_br_contiguous, 4990 LDNT1D_z_p_bi_contiguous); 4991 } 4992} 4993 4994void Assembler::ldnt1h(const ZRegister& zt, 4995 const PRegisterZ& pg, 4996 const SVEMemOperand& addr) { 4997 VIXL_ASSERT(addr.IsPlainScalar() || 4998 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || 4999 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(1)) || 5000 (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2))); 5001 if (addr.IsVectorPlusScalar()) { 5002 SVELd1VecScaHelper(zt, pg, addr, 1, /* is_signed = */ false); 5003 } else { 5004 SVELd1St1ScaImmHelper(zt, 5005 pg, 5006 addr, 5007 LDNT1H_z_p_br_contiguous, 5008 LDNT1H_z_p_bi_contiguous); 5009 } 5010} 5011 5012void Assembler::ldnt1w(const ZRegister& zt, 5013 const PRegisterZ& pg, 5014 const SVEMemOperand& addr) { 5015 VIXL_ASSERT(addr.IsPlainScalar() || 5016 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || 5017 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(2)) || 5018 (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2))); 5019 if (addr.IsVectorPlusScalar()) { 5020 SVELd1VecScaHelper(zt, pg, addr, 2, /* is_signed = */ false); 5021 } else { 5022 SVELd1St1ScaImmHelper(zt, 5023 pg, 5024 addr, 5025 LDNT1W_z_p_br_contiguous, 5026 LDNT1W_z_p_bi_contiguous); 5027 } 5028} 5029 5030void Assembler::ldnt1sb(const ZRegister& zt, 5031 const PRegisterZ& pg, 5032 const SVEMemOperand& addr) { 5033 VIXL_ASSERT(addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)); 5034 SVELd1VecScaHelper(zt, pg, addr, 0, /* is_signed = */ true); 5035} 5036 5037void Assembler::ldnt1sh(const ZRegister& zt, 5038 const PRegisterZ& pg, 5039 const SVEMemOperand& addr) { 5040 VIXL_ASSERT(addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)); 5041 SVELd1VecScaHelper(zt, pg, addr, 1, /* is_signed = */ true); 5042} 5043 5044void Assembler::ldnt1sw(const ZRegister& zt, 5045 const PRegisterZ& pg, 5046 const SVEMemOperand& addr) { 5047 VIXL_ASSERT(addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2)); 5048 SVELd1VecScaHelper(zt, pg, addr, 2, /* is_signed = */ true); 5049} 5050 5051Instr Assembler::SVEMemOperandHelper(unsigned msize_in_bytes_log2, 5052 int num_regs, 5053 const SVEMemOperand& addr, 5054 bool is_load) { 5055 VIXL_ASSERT((num_regs >= 1) && (num_regs <= 4)); 5056 5057 Instr op = 0xfffffff; 5058 if (addr.IsScalarPlusImmediate()) { 5059 VIXL_ASSERT((addr.GetImmediateOffset() == 0) || addr.IsMulVl()); 5060 int64_t imm = addr.GetImmediateOffset(); 5061 VIXL_ASSERT((imm % num_regs) == 0); 5062 op = RnSP(addr.GetScalarBase()) | ImmField<19, 16>(imm / num_regs); 5063 5064 } else if (addr.IsScalarPlusScalar()) { 5065 VIXL_ASSERT(addr.GetScalarOffset().IsZero() || 5066 addr.IsEquivalentToLSL(msize_in_bytes_log2)); 5067 op = RnSP(addr.GetScalarBase()) | Rm(addr.GetScalarOffset()); 5068 5069 } else if (addr.IsVectorPlusImmediate()) { 5070 ZRegister zn = addr.GetVectorBase(); 5071 uint64_t imm = addr.GetImmediateOffset(); 5072 VIXL_ASSERT(num_regs == 1); 5073 VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD()); 5074 VIXL_ASSERT(IsMultiple(imm, (1 << msize_in_bytes_log2))); 5075 op = Rn(zn) | ImmUnsignedField<20, 16>(imm >> msize_in_bytes_log2); 5076 } else if (addr.IsVectorPlusScalar()) { 5077 VIXL_ASSERT(addr.GetOffsetModifier() == NO_SVE_OFFSET_MODIFIER); 5078 VIXL_ASSERT(addr.GetShiftAmount() == 0); 5079 ZRegister zn = addr.GetVectorBase(); 5080 VIXL_ASSERT(zn.IsLaneSizeS() || zn.IsLaneSizeD()); 5081 Register xm = addr.GetScalarOffset(); 5082 op = Rn(zn) | Rm(xm); 5083 } else if (addr.IsScalarPlusVector()) { 5084 // We have to support several different addressing modes. Some instructions 5085 // support a subset of these, but the SVEMemOperand encoding is consistent. 5086 Register xn = addr.GetScalarBase(); 5087 ZRegister zm = addr.GetVectorOffset(); 5088 SVEOffsetModifier mod = addr.GetOffsetModifier(); 5089 Instr modifier_bit = 1 << (is_load ? 22 : 14); 5090 Instr xs = (mod == SVE_SXTW) ? modifier_bit : 0; 5091 VIXL_ASSERT(num_regs == 1); 5092 5093 if (mod == SVE_LSL) { 5094 // 64-bit scaled offset: [<Xn|SP>, <Zm>.D, LSL #<shift>] 5095 VIXL_ASSERT(zm.IsLaneSizeD()); 5096 VIXL_ASSERT(addr.GetShiftAmount() == msize_in_bytes_log2); 5097 } else if (mod == NO_SVE_OFFSET_MODIFIER) { 5098 // 64-bit unscaled offset: [<Xn|SP>, <Zm>.D] 5099 VIXL_ASSERT(zm.IsLaneSizeD()); 5100 VIXL_ASSERT(addr.GetShiftAmount() == 0); 5101 } else { 5102 // 32-bit scaled offset: [<Xn|SP>, <Zm>.S, <mod> #<shift>] 5103 // 32-bit unscaled offset: [<Xn|SP>, <Zm>.S, <mod>] 5104 // 32-bit unpacked scaled offset: [<Xn|SP>, <Zm>.D, <mod> #<shift>] 5105 // 32-bit unpacked unscaled offset: [<Xn|SP>, <Zm>.D, <mod>] 5106 VIXL_ASSERT(zm.IsLaneSizeS() || zm.IsLaneSizeD()); 5107 VIXL_ASSERT((mod == SVE_SXTW) || (mod == SVE_UXTW)); 5108 VIXL_ASSERT((addr.GetShiftAmount() == 0) || 5109 (addr.GetShiftAmount() == msize_in_bytes_log2)); 5110 } 5111 5112 // The form itself is encoded in the instruction opcode. 5113 op = RnSP(xn) | Rm(zm) | xs; 5114 } else { 5115 VIXL_UNIMPLEMENTED(); 5116 } 5117 5118 return op; 5119} 5120 5121// SVEMemStore. 5122 5123void Assembler::SVESt1Helper(unsigned msize_in_bytes_log2, 5124 const ZRegister& zt, 5125 const PRegister& pg, 5126 const SVEMemOperand& addr) { 5127 if (addr.IsScalarPlusScalar()) { 5128 // Rm must not be xzr. 5129 VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); 5130 } 5131 5132 if (addr.IsScatterGather()) { 5133 bool is_load = false; 5134 bool is_signed = false; 5135 bool is_ff = false; 5136 SVEScatterGatherHelper(msize_in_bytes_log2, 5137 zt, 5138 pg, 5139 addr, 5140 is_load, 5141 is_signed, 5142 is_ff); 5143 return; 5144 } 5145 5146 Instr op; 5147 if (addr.IsScalarPlusImmediate()) { 5148 op = SVEContiguousStore_ScalarPlusImmFixed; 5149 } else if (addr.IsScalarPlusScalar()) { 5150 op = SVEContiguousStore_ScalarPlusScalarFixed; 5151 } else { 5152 VIXL_UNIMPLEMENTED(); 5153 op = 0xffffffff; 5154 } 5155 SVELdSt1Helper(msize_in_bytes_log2, zt, pg, addr, false, op); 5156} 5157 5158void Assembler::SVESt234Helper(int num_regs, 5159 const ZRegister& zt1, 5160 const PRegister& pg, 5161 const SVEMemOperand& addr) { 5162 if (addr.IsScalarPlusScalar()) { 5163 // Rm must not be xzr. 5164 VIXL_ASSERT(!addr.GetScalarOffset().IsZero()); 5165 } 5166 5167 Instr op; 5168 if (addr.IsScalarPlusImmediate()) { 5169 op = SVEStoreMultipleStructures_ScalarPlusImmFixed; 5170 } else if (addr.IsScalarPlusScalar()) { 5171 op = SVEStoreMultipleStructures_ScalarPlusScalarFixed; 5172 } else { 5173 // These instructions don't support any other addressing modes. 5174 VIXL_ABORT(); 5175 } 5176 SVELdSt234Helper(num_regs, zt1, pg, addr, op); 5177} 5178 5179#define VIXL_DEFINE_ST1(MSZ, LANE_SIZE) \ 5180 void Assembler::st1##MSZ(const ZRegister& zt, \ 5181 const PRegister& pg, \ 5182 const SVEMemOperand& addr) { \ 5183 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 5184 SVESt1Helper(k##LANE_SIZE##RegSizeInBytesLog2, zt, pg, addr); \ 5185 } 5186#define VIXL_DEFINE_ST2(MSZ, LANE_SIZE) \ 5187 void Assembler::st2##MSZ(const ZRegister& zt1, \ 5188 const ZRegister& zt2, \ 5189 const PRegister& pg, \ 5190 const SVEMemOperand& addr) { \ 5191 USE(zt2); \ 5192 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 5193 VIXL_ASSERT(AreConsecutive(zt1, zt2)); \ 5194 VIXL_ASSERT(AreSameFormat(zt1, zt2)); \ 5195 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ 5196 SVESt234Helper(2, zt1, pg, addr); \ 5197 } 5198#define VIXL_DEFINE_ST3(MSZ, LANE_SIZE) \ 5199 void Assembler::st3##MSZ(const ZRegister& zt1, \ 5200 const ZRegister& zt2, \ 5201 const ZRegister& zt3, \ 5202 const PRegister& pg, \ 5203 const SVEMemOperand& addr) { \ 5204 USE(zt2, zt3); \ 5205 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 5206 VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3)); \ 5207 VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3)); \ 5208 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ 5209 SVESt234Helper(3, zt1, pg, addr); \ 5210 } 5211#define VIXL_DEFINE_ST4(MSZ, LANE_SIZE) \ 5212 void Assembler::st4##MSZ(const ZRegister& zt1, \ 5213 const ZRegister& zt2, \ 5214 const ZRegister& zt3, \ 5215 const ZRegister& zt4, \ 5216 const PRegister& pg, \ 5217 const SVEMemOperand& addr) { \ 5218 USE(zt2, zt3, zt4); \ 5219 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); \ 5220 VIXL_ASSERT(AreConsecutive(zt1, zt2, zt3, zt4)); \ 5221 VIXL_ASSERT(AreSameFormat(zt1, zt2, zt3, zt4)); \ 5222 VIXL_ASSERT(zt1.IsLaneSize##LANE_SIZE()); \ 5223 SVESt234Helper(4, zt1, pg, addr); \ 5224 } 5225 5226VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST1) 5227VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST2) 5228VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST3) 5229VIXL_SVE_LOAD_STORE_VARIANT_LIST(VIXL_DEFINE_ST4) 5230 5231void Assembler::stnt1b(const ZRegister& zt, 5232 const PRegister& pg, 5233 const SVEMemOperand& addr) { 5234 VIXL_ASSERT(addr.IsPlainScalar() || 5235 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || 5236 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(0)) || 5237 (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2))); 5238 if (addr.IsVectorPlusScalar()) { 5239 SVESt1VecScaHelper(zt, pg, addr, 0); 5240 } else { 5241 SVELd1St1ScaImmHelper(zt, 5242 pg, 5243 addr, 5244 STNT1B_z_p_br_contiguous, 5245 STNT1B_z_p_bi_contiguous); 5246 } 5247} 5248 5249void Assembler::stnt1d(const ZRegister& zt, 5250 const PRegister& pg, 5251 const SVEMemOperand& addr) { 5252 VIXL_ASSERT(addr.IsPlainScalar() || 5253 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || 5254 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(3)) || 5255 (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2))); 5256 if (addr.IsVectorPlusScalar()) { 5257 SVESt1VecScaHelper(zt, pg, addr, 3); 5258 } else { 5259 SVELd1St1ScaImmHelper(zt, 5260 pg, 5261 addr, 5262 STNT1D_z_p_br_contiguous, 5263 STNT1D_z_p_bi_contiguous); 5264 } 5265} 5266 5267void Assembler::stnt1h(const ZRegister& zt, 5268 const PRegister& pg, 5269 const SVEMemOperand& addr) { 5270 VIXL_ASSERT(addr.IsPlainScalar() || 5271 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || 5272 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(1)) || 5273 (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2))); 5274 if (addr.IsVectorPlusScalar()) { 5275 SVESt1VecScaHelper(zt, pg, addr, 1); 5276 } else { 5277 SVELd1St1ScaImmHelper(zt, 5278 pg, 5279 addr, 5280 STNT1H_z_p_br_contiguous, 5281 STNT1H_z_p_bi_contiguous); 5282 } 5283} 5284 5285void Assembler::stnt1w(const ZRegister& zt, 5286 const PRegister& pg, 5287 const SVEMemOperand& addr) { 5288 VIXL_ASSERT(addr.IsPlainScalar() || 5289 (addr.IsScalarPlusImmediate() && addr.IsMulVl()) || 5290 (addr.IsScalarPlusScalar() && addr.IsEquivalentToLSL(2)) || 5291 (addr.IsVectorPlusScalar() && CPUHas(CPUFeatures::kSVE2))); 5292 if (addr.IsVectorPlusScalar()) { 5293 SVESt1VecScaHelper(zt, pg, addr, 2); 5294 } else { 5295 SVELd1St1ScaImmHelper(zt, 5296 pg, 5297 addr, 5298 STNT1W_z_p_br_contiguous, 5299 STNT1W_z_p_bi_contiguous); 5300 } 5301} 5302 5303void Assembler::str(const CPURegister& rt, const SVEMemOperand& addr) { 5304 // STR <Pt/Zt>, [<Xn|SP>{, #<imm>, MUL VL}] 5305 5306 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5307 VIXL_ASSERT(rt.IsPRegister() || rt.IsZRegister()); 5308 VIXL_ASSERT(addr.IsPlainScalar() || 5309 (addr.IsScalarPlusImmediate() && 5310 (addr.GetOffsetModifier() == SVE_MUL_VL))); 5311 int64_t imm9 = addr.GetImmediateOffset(); 5312 VIXL_ASSERT(IsInt9(imm9)); 5313 Instr imm9l = ExtractUnsignedBitfield32(2, 0, imm9) << 10; 5314 Instr imm9h = ExtractUnsignedBitfield32(8, 3, imm9) << 16; 5315 5316 Instr op = STR_z_bi; 5317 if (rt.IsPRegister()) { 5318 op = STR_p_bi; 5319 } 5320 Emit(op | Rt(rt) | RnSP(addr.GetScalarBase()) | imm9h | imm9l); 5321} 5322 5323// SVEMulIndex. 5324 5325void Assembler::sdot(const ZRegister& zda, 5326 const ZRegister& zn, 5327 const ZRegister& zm, 5328 int index) { 5329 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5330 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); 5331 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 5332 5333 Instr op = 0xffffffff; 5334 switch (zda.GetLaneSizeInBits()) { 5335 case kSRegSize: 5336 VIXL_ASSERT(IsUint2(index)); 5337 op = SDOT_z_zzzi_s | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn); 5338 break; 5339 case kDRegSize: 5340 VIXL_ASSERT(IsUint1(index)); 5341 op = SDOT_z_zzzi_d | Rx<19, 16>(zm) | (index << 20) | Rd(zda) | Rn(zn); 5342 break; 5343 default: 5344 VIXL_UNIMPLEMENTED(); 5345 break; 5346 } 5347 5348 Emit(op); 5349} 5350 5351void Assembler::udot(const ZRegister& zda, 5352 const ZRegister& zn, 5353 const ZRegister& zm, 5354 int index) { 5355 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5356 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); 5357 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 5358 5359 Instr op = 0xffffffff; 5360 switch (zda.GetLaneSizeInBits()) { 5361 case kSRegSize: 5362 VIXL_ASSERT(IsUint2(index)); 5363 op = UDOT_z_zzzi_s | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn); 5364 break; 5365 case kDRegSize: 5366 VIXL_ASSERT(IsUint1(index)); 5367 op = UDOT_z_zzzi_d | Rx<19, 16>(zm) | (index << 20) | Rd(zda) | Rn(zn); 5368 break; 5369 default: 5370 VIXL_UNIMPLEMENTED(); 5371 break; 5372 } 5373 5374 Emit(op); 5375} 5376 5377// SVEPartitionBreak. 5378 5379void Assembler::brka(const PRegisterWithLaneSize& pd, 5380 const PRegister& pg, 5381 const PRegisterWithLaneSize& pn) { 5382 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5383 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); 5384 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); 5385 5386 Instr m = pg.IsMerging() ? 0x00000010 : 0x00000000; 5387 Emit(BRKA_p_p_p | Pd(pd) | Pg<13, 10>(pg) | m | Pn(pn)); 5388} 5389 5390void Assembler::brkas(const PRegisterWithLaneSize& pd, 5391 const PRegisterZ& pg, 5392 const PRegisterWithLaneSize& pn) { 5393 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5394 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); 5395 5396 Emit(BRKAS_p_p_p_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn)); 5397} 5398 5399void Assembler::brkb(const PRegisterWithLaneSize& pd, 5400 const PRegister& pg, 5401 const PRegisterWithLaneSize& pn) { 5402 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5403 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); 5404 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); 5405 5406 Instr m = pg.IsMerging() ? 0x00000010 : 0x00000000; 5407 Emit(BRKB_p_p_p | Pd(pd) | Pg<13, 10>(pg) | m | Pn(pn)); 5408} 5409 5410void Assembler::brkbs(const PRegisterWithLaneSize& pd, 5411 const PRegisterZ& pg, 5412 const PRegisterWithLaneSize& pn) { 5413 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5414 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); 5415 5416 Emit(BRKBS_p_p_p_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn)); 5417} 5418 5419void Assembler::brkn(const PRegisterWithLaneSize& pd, 5420 const PRegisterZ& pg, 5421 const PRegisterWithLaneSize& pn, 5422 const PRegisterWithLaneSize& pm) { 5423 USE(pm); 5424 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5425 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); 5426 VIXL_ASSERT(pd.Is(pm)); 5427 5428 Emit(BRKN_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn)); 5429} 5430 5431void Assembler::brkns(const PRegisterWithLaneSize& pd, 5432 const PRegisterZ& pg, 5433 const PRegisterWithLaneSize& pn, 5434 const PRegisterWithLaneSize& pm) { 5435 USE(pm); 5436 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5437 VIXL_ASSERT(pd.IsLaneSizeB() && pn.IsLaneSizeB()); 5438 VIXL_ASSERT(pd.Is(pm)); 5439 5440 Emit(BRKNS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn)); 5441} 5442 5443// SVEPermutePredicate. 5444 5445void Assembler::punpkhi(const PRegisterWithLaneSize& pd, 5446 const PRegisterWithLaneSize& pn) { 5447 // PUNPKHI <Pd>.H, <Pn>.B 5448 // 0000 0101 0011 0001 0100 000. ...0 .... 5449 // H<16> = 1 | Pn<8:5> | Pd<3:0> 5450 5451 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5452 VIXL_ASSERT(pd.IsLaneSizeH()); 5453 VIXL_ASSERT(pn.IsLaneSizeB()); 5454 5455 Emit(PUNPKHI_p_p | Pd(pd) | Pn(pn)); 5456} 5457 5458void Assembler::punpklo(const PRegisterWithLaneSize& pd, 5459 const PRegisterWithLaneSize& pn) { 5460 // PUNPKLO <Pd>.H, <Pn>.B 5461 // 0000 0101 0011 0000 0100 000. ...0 .... 5462 // H<16> = 0 | Pn<8:5> | Pd<3:0> 5463 5464 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5465 VIXL_ASSERT(pd.IsLaneSizeH()); 5466 VIXL_ASSERT(pn.IsLaneSizeB()); 5467 5468 Emit(PUNPKLO_p_p | Pd(pd) | Pn(pn)); 5469} 5470 5471void Assembler::rev(const PRegisterWithLaneSize& pd, 5472 const PRegisterWithLaneSize& pn) { 5473 // REV <Pd>.<T>, <Pn>.<T> 5474 // 0000 0101 ..11 0100 0100 000. ...0 .... 5475 // size<23:22> | Pn<8:5> | Pd<3:0> 5476 5477 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5478 VIXL_ASSERT(AreSameLaneSize(pd, pn)); 5479 5480 Emit(REV_p_p | SVESize(pd) | Pd(pd) | Rx<8, 5>(pn)); 5481} 5482 5483void Assembler::trn1(const PRegisterWithLaneSize& pd, 5484 const PRegisterWithLaneSize& pn, 5485 const PRegisterWithLaneSize& pm) { 5486 // TRN1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> 5487 // 0000 0101 ..10 .... 0101 000. ...0 .... 5488 // size<23:22> | Pm<19:16> | opc<12:11> = 10 | H<10> = 0 | Pn<8:5> | Pd<3:0> 5489 5490 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5491 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); 5492 5493 Emit(TRN1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); 5494} 5495 5496void Assembler::trn2(const PRegisterWithLaneSize& pd, 5497 const PRegisterWithLaneSize& pn, 5498 const PRegisterWithLaneSize& pm) { 5499 // TRN2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> 5500 // 0000 0101 ..10 .... 0101 010. ...0 .... 5501 // size<23:22> | Pm<19:16> | opc<12:11> = 10 | H<10> = 1 | Pn<8:5> | Pd<3:0> 5502 5503 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5504 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); 5505 5506 Emit(TRN2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); 5507} 5508 5509void Assembler::uzp1(const PRegisterWithLaneSize& pd, 5510 const PRegisterWithLaneSize& pn, 5511 const PRegisterWithLaneSize& pm) { 5512 // UZP1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> 5513 // 0000 0101 ..10 .... 0100 100. ...0 .... 5514 // size<23:22> | Pm<19:16> | opc<12:11> = 01 | H<10> = 0 | Pn<8:5> | Pd<3:0> 5515 5516 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5517 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); 5518 5519 Emit(UZP1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); 5520} 5521 5522void Assembler::uzp2(const PRegisterWithLaneSize& pd, 5523 const PRegisterWithLaneSize& pn, 5524 const PRegisterWithLaneSize& pm) { 5525 // UZP2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> 5526 // 0000 0101 ..10 .... 0100 110. ...0 .... 5527 // size<23:22> | Pm<19:16> | opc<12:11> = 01 | H<10> = 1 | Pn<8:5> | Pd<3:0> 5528 5529 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5530 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); 5531 5532 Emit(UZP2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); 5533} 5534 5535void Assembler::zip1(const PRegisterWithLaneSize& pd, 5536 const PRegisterWithLaneSize& pn, 5537 const PRegisterWithLaneSize& pm) { 5538 // ZIP1 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> 5539 // 0000 0101 ..10 .... 0100 000. ...0 .... 5540 // size<23:22> | Pm<19:16> | opc<12:11> = 00 | H<10> = 0 | Pn<8:5> | Pd<3:0> 5541 5542 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5543 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); 5544 5545 Emit(ZIP1_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); 5546} 5547 5548void Assembler::zip2(const PRegisterWithLaneSize& pd, 5549 const PRegisterWithLaneSize& pn, 5550 const PRegisterWithLaneSize& pm) { 5551 // ZIP2 <Pd>.<T>, <Pn>.<T>, <Pm>.<T> 5552 // 0000 0101 ..10 .... 0100 010. ...0 .... 5553 // size<23:22> | Pm<19:16> | opc<12:11> = 00 | H<10> = 1 | Pn<8:5> | Pd<3:0> 5554 5555 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5556 VIXL_ASSERT(AreSameLaneSize(pd, pn, pm)); 5557 5558 Emit(ZIP2_p_pp | SVESize(pd) | Pd(pd) | Pn(pn) | Pm(pm)); 5559} 5560 5561// SVEPermuteVectorExtract. 5562 5563void Assembler::ext(const ZRegister& zd, 5564 const ZRegister& zn, 5565 const ZRegister& zm, 5566 unsigned offset) { 5567 // EXT <Zdn>.B, <Zdn>.B, <Zm>.B, #<imm> 5568 // 0000 0101 001. .... 000. .... .... .... 5569 // imm8h<20:16> | imm8l<12:10> | Zm<9:5> | Zdn<4:0> 5570 5571 // EXT <Zd>.B, { <Zn1>.B, <Zn2>.B }, #<imm> 5572 // 0000 0101 011. .... 000. .... .... .... 5573 // imm8h<20:16> | imm8l<12:10> | Zn<9:5> | Zd<4:0> 5574 5575 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5576 VIXL_ASSERT(IsUint8(offset)); 5577 5578 int imm8h = ExtractUnsignedBitfield32(7, 3, offset); 5579 int imm8l = ExtractUnsignedBitfield32(2, 0, offset); 5580 5581 Instr op; 5582 if (zd.Is(zn)) { 5583 // Destructive form. 5584 op = EXT_z_zi_des | Rn(zm); 5585 } else { 5586 // Constructive form (requires SVE2). 5587 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2) && AreConsecutive(zn, zm)); 5588 op = 0x05600000 | Rn(zn); 5589 } 5590 5591 Emit(op | Rd(zd) | ImmUnsignedField<20, 16>(imm8h) | 5592 ImmUnsignedField<12, 10>(imm8l)); 5593} 5594 5595// SVEPermuteVectorInterleaving. 5596 5597void Assembler::trn1(const ZRegister& zd, 5598 const ZRegister& zn, 5599 const ZRegister& zm) { 5600 // TRN1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 5601 // 0000 0101 ..1. .... 0111 00.. .... .... 5602 // size<23:22> | Zm<20:16> | opc<12:10> = 100 | Zn<9:5> | Zd<4:0> 5603 5604 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5605 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 5606 5607 Emit(TRN1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 5608} 5609 5610void Assembler::trn2(const ZRegister& zd, 5611 const ZRegister& zn, 5612 const ZRegister& zm) { 5613 // TRN2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 5614 // 0000 0101 ..1. .... 0111 01.. .... .... 5615 // size<23:22> | Zm<20:16> | opc<12:10> = 101 | Zn<9:5> | Zd<4:0> 5616 5617 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5618 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 5619 5620 Emit(TRN2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 5621} 5622 5623void Assembler::uzp1(const ZRegister& zd, 5624 const ZRegister& zn, 5625 const ZRegister& zm) { 5626 // UZP1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 5627 // 0000 0101 ..1. .... 0110 10.. .... .... 5628 // size<23:22> | Zm<20:16> | opc<12:10> = 010 | Zn<9:5> | Zd<4:0> 5629 5630 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5631 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 5632 5633 Emit(UZP1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 5634} 5635 5636void Assembler::uzp2(const ZRegister& zd, 5637 const ZRegister& zn, 5638 const ZRegister& zm) { 5639 // UZP2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 5640 // 0000 0101 ..1. .... 0110 11.. .... .... 5641 // size<23:22> | Zm<20:16> | opc<12:10> = 011 | Zn<9:5> | Zd<4:0> 5642 5643 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5644 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 5645 5646 Emit(UZP2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 5647} 5648 5649void Assembler::zip1(const ZRegister& zd, 5650 const ZRegister& zn, 5651 const ZRegister& zm) { 5652 // ZIP1 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 5653 // 0000 0101 ..1. .... 0110 00.. .... .... 5654 // size<23:22> | Zm<20:16> | opc<12:10> = 000 | Zn<9:5> | Zd<4:0> 5655 5656 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5657 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 5658 5659 Emit(ZIP1_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 5660} 5661 5662void Assembler::zip2(const ZRegister& zd, 5663 const ZRegister& zn, 5664 const ZRegister& zm) { 5665 // ZIP2 <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 5666 // 0000 0101 ..1. .... 0110 01.. .... .... 5667 // size<23:22> | Zm<20:16> | opc<12:10> = 001 | Zn<9:5> | Zd<4:0> 5668 5669 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5670 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 5671 5672 Emit(ZIP2_z_zz | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 5673} 5674 5675// SVEPermuteVectorPredicated. 5676 5677void Assembler::clasta(const Register& rd, 5678 const PRegister& pg, 5679 const Register& rn, 5680 const ZRegister& zm) { 5681 // CLASTA <R><dn>, <Pg>, <R><dn>, <Zm>.<T> 5682 // 0000 0101 ..11 0000 101. .... .... .... 5683 // size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Rdn<4:0> 5684 5685 USE(rn); 5686 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5687 VIXL_ASSERT(rd.Is(rn)); 5688 5689 Emit(CLASTA_r_p_z | SVESize(zm) | Rd(rd) | PgLow8(pg) | Rn(zm)); 5690} 5691 5692void Assembler::clasta(const VRegister& vd, 5693 const PRegister& pg, 5694 const VRegister& vn, 5695 const ZRegister& zm) { 5696 // CLASTA <V><dn>, <Pg>, <V><dn>, <Zm>.<T> 5697 // 0000 0101 ..10 1010 100. .... .... .... 5698 // size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Vdn<4:0> 5699 5700 USE(vn); 5701 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5702 VIXL_ASSERT(vd.Is(vn)); 5703 VIXL_ASSERT(vd.IsScalar()); 5704 VIXL_ASSERT(AreSameLaneSize(vd, zm)); 5705 5706 Emit(CLASTA_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm)); 5707} 5708 5709void Assembler::clasta(const ZRegister& zd, 5710 const PRegister& pg, 5711 const ZRegister& zn, 5712 const ZRegister& zm) { 5713 // CLASTA <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T> 5714 // 0000 0101 ..10 1000 100. .... .... .... 5715 // size<23:22> | B<16> = 0 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 5716 5717 USE(zn); 5718 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5719 VIXL_ASSERT(zd.Is(zn)); 5720 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 5721 5722 Emit(CLASTA_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 5723} 5724 5725void Assembler::clastb(const Register& rd, 5726 const PRegister& pg, 5727 const Register& rn, 5728 const ZRegister& zm) { 5729 // CLASTB <R><dn>, <Pg>, <R><dn>, <Zm>.<T> 5730 // 0000 0101 ..11 0001 101. .... .... .... 5731 // size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Rdn<4:0> 5732 5733 USE(rn); 5734 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5735 VIXL_ASSERT(rd.Is(rn)); 5736 5737 Emit(CLASTB_r_p_z | SVESize(zm) | Rd(rd) | PgLow8(pg) | Rn(zm)); 5738} 5739 5740void Assembler::clastb(const VRegister& vd, 5741 const PRegister& pg, 5742 const VRegister& vn, 5743 const ZRegister& zm) { 5744 // CLASTB <V><dn>, <Pg>, <V><dn>, <Zm>.<T> 5745 // 0000 0101 ..10 1011 100. .... .... .... 5746 // size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Vdn<4:0> 5747 5748 USE(vn); 5749 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5750 VIXL_ASSERT(vd.Is(vn)); 5751 VIXL_ASSERT(vd.IsScalar()); 5752 VIXL_ASSERT(AreSameLaneSize(vd, zm)); 5753 5754 Emit(CLASTB_v_p_z | SVESize(zm) | Rd(vd) | PgLow8(pg) | Rn(zm)); 5755} 5756 5757void Assembler::clastb(const ZRegister& zd, 5758 const PRegister& pg, 5759 const ZRegister& zn, 5760 const ZRegister& zm) { 5761 // CLASTB <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T> 5762 // 0000 0101 ..10 1001 100. .... .... .... 5763 // size<23:22> | B<16> = 1 | Pg<12:10> | Zm<9:5> | Zdn<4:0> 5764 5765 USE(zn); 5766 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5767 VIXL_ASSERT(zd.Is(zn)); 5768 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 5769 5770 Emit(CLASTB_z_p_zz | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 5771} 5772 5773void Assembler::compact(const ZRegister& zd, 5774 const PRegister& pg, 5775 const ZRegister& zn) { 5776 // COMPACT <Zd>.<T>, <Pg>, <Zn>.<T> 5777 // 0000 0101 1.10 0001 100. .... .... .... 5778 // sz<22> | Pg<12:10> | Zn<9:5> | Zd<4:0> 5779 5780 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5781 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 5782 VIXL_ASSERT((zd.GetLaneSizeInBits() == kSRegSize) || 5783 (zd.GetLaneSizeInBits() == kDRegSize)); 5784 5785 Instr sz = (zd.GetLaneSizeInBits() == kDRegSize) ? (1 << 22) : 0; 5786 Emit(COMPACT_z_p_z | sz | Rd(zd) | PgLow8(pg) | Rn(zn)); 5787} 5788 5789void Assembler::cpy(const ZRegister& zd, 5790 const PRegisterM& pg, 5791 const Register& rn) { 5792 // CPY <Zd>.<T>, <Pg>/M, <R><n|SP> 5793 // 0000 0101 ..10 1000 101. .... .... .... 5794 // size<23:22> | Pg<12:10> | Rn<9:5> | Zd<4:0> 5795 5796 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5797 VIXL_ASSERT(static_cast<unsigned>(rn.GetSizeInBits()) >= 5798 zd.GetLaneSizeInBits()); 5799 5800 Emit(CPY_z_p_r | SVESize(zd) | Rd(zd) | PgLow8(pg) | RnSP(rn)); 5801} 5802 5803void Assembler::cpy(const ZRegister& zd, 5804 const PRegisterM& pg, 5805 const VRegister& vn) { 5806 // CPY <Zd>.<T>, <Pg>/M, <V><n> 5807 // 0000 0101 ..10 0000 100. .... .... .... 5808 // size<23:22> | Pg<12:10> | Vn<9:5> | Zd<4:0> 5809 5810 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5811 VIXL_ASSERT(vn.IsScalar()); 5812 VIXL_ASSERT(static_cast<unsigned>(vn.GetSizeInBits()) == 5813 zd.GetLaneSizeInBits()); 5814 5815 Emit(CPY_z_p_v | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(vn)); 5816} 5817 5818void Assembler::lasta(const Register& rd, 5819 const PRegister& pg, 5820 const ZRegister& zn) { 5821 // LASTA <R><d>, <Pg>, <Zn>.<T> 5822 // 0000 0101 ..10 0000 101. .... .... .... 5823 // size<23:22> | B<16> = 0 | Pg<12:10> | Zn<9:5> | Rd<4:0> 5824 5825 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5826 5827 Emit(LASTA_r_p_z | SVESize(zn) | Rd(rd) | PgLow8(pg) | Rn(zn)); 5828} 5829 5830void Assembler::lasta(const VRegister& vd, 5831 const PRegister& pg, 5832 const ZRegister& zn) { 5833 // LASTA <V><d>, <Pg>, <Zn>.<T> 5834 // 0000 0101 ..10 0010 100. .... .... .... 5835 // size<23:22> | B<16> = 0 | Pg<12:10> | Zn<9:5> | Vd<4:0> 5836 5837 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5838 VIXL_ASSERT(vd.IsScalar()); 5839 5840 Emit(LASTA_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 5841} 5842 5843void Assembler::lastb(const Register& rd, 5844 const PRegister& pg, 5845 const ZRegister& zn) { 5846 // LASTB <R><d>, <Pg>, <Zn>.<T> 5847 // 0000 0101 ..10 0001 101. .... .... .... 5848 // size<23:22> | B<16> = 1 | Pg<12:10> | Zn<9:5> | Rd<4:0> 5849 5850 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5851 5852 Emit(LASTB_r_p_z | SVESize(zn) | Rd(rd) | PgLow8(pg) | Rn(zn)); 5853} 5854 5855void Assembler::lastb(const VRegister& vd, 5856 const PRegister& pg, 5857 const ZRegister& zn) { 5858 // LASTB <V><d>, <Pg>, <Zn>.<T> 5859 // 0000 0101 ..10 0011 100. .... .... .... 5860 // size<23:22> | B<16> = 1 | Pg<12:10> | Zn<9:5> | Vd<4:0> 5861 5862 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5863 VIXL_ASSERT(vd.IsScalar()); 5864 5865 Emit(LASTB_v_p_z | SVESize(zn) | Rd(vd) | PgLow8(pg) | Rn(zn)); 5866} 5867 5868void Assembler::rbit(const ZRegister& zd, 5869 const PRegisterM& pg, 5870 const ZRegister& zn) { 5871 // RBIT <Zd>.<T>, <Pg>/M, <Zn>.<T> 5872 // 0000 0101 ..10 0111 100. .... .... .... 5873 // size<23:22> | opc<17:16> = 11 | Pg<12:10> | Zn<9:5> | Zd<4:0> 5874 5875 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5876 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 5877 5878 Emit(RBIT_z_p_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 5879} 5880 5881void Assembler::revb(const ZRegister& zd, 5882 const PRegisterM& pg, 5883 const ZRegister& zn) { 5884 // REVB <Zd>.<T>, <Pg>/M, <Zn>.<T> 5885 // 0000 0101 ..10 0100 100. .... .... .... 5886 // size<23:22> | opc<17:16> = 00 | Pg<12:10> | Zn<9:5> | Zd<4:0> 5887 5888 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5889 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 5890 VIXL_ASSERT(zd.IsLaneSizeH() || zd.IsLaneSizeS() || zd.IsLaneSizeD()); 5891 5892 Emit(REVB_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 5893} 5894 5895void Assembler::revh(const ZRegister& zd, 5896 const PRegisterM& pg, 5897 const ZRegister& zn) { 5898 // REVH <Zd>.<T>, <Pg>/M, <Zn>.<T> 5899 // 0000 0101 ..10 0101 100. .... .... .... 5900 // size<23:22> | opc<17:16> = 01 | Pg<12:10> | Zn<9:5> | Zd<4:0> 5901 5902 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5903 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 5904 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); 5905 5906 Emit(REVH_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 5907} 5908 5909void Assembler::revw(const ZRegister& zd, 5910 const PRegisterM& pg, 5911 const ZRegister& zn) { 5912 // REVW <Zd>.D, <Pg>/M, <Zn>.D 5913 // 0000 0101 ..10 0110 100. .... .... .... 5914 // size<23:22> | opc<17:16> = 10 | Pg<12:10> | Zn<9:5> | Zd<4:0> 5915 5916 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5917 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 5918 VIXL_ASSERT(zd.IsLaneSizeD()); 5919 5920 Emit(REVW_z_z | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 5921} 5922 5923void Assembler::splice(const ZRegister& zd, 5924 const PRegister& pg, 5925 const ZRegister& zn, 5926 const ZRegister& zm) { 5927 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 5928 5929 if (zd.Aliases(zn)) { 5930 // SPLICE <Zdn>.<T>, <Pg>, <Zdn>.<T>, <Zm>.<T> 5931 // 0000 0101 ..10 1100 100. .... .... .... 5932 // size<23:22> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 5933 5934 USE(zn); 5935 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5936 VIXL_ASSERT(zd.Is(zn)); 5937 5938 Emit(SPLICE_z_p_zz_des | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 5939 } else { 5940 splice_con(zd, pg, zn, zm); 5941 } 5942} 5943 5944void Assembler::splice_con(const ZRegister& zd, 5945 const PRegister& pg, 5946 const ZRegister& zn1, 5947 const ZRegister& zn2) { 5948 // SPLICE <Zd>.<T>, <Pg>, { <Zn1>.<T>, <Zn2>.<T> } 5949 // 0000 0101 ..10 1101 100. .... .... .... 5950 // size<23:22> | Pg<12:10> | Zn<9:5> | Zd<4:0> 5951 5952 USE(zn2); 5953 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 5954 VIXL_ASSERT(AreConsecutive(zn1, zn2)); 5955 VIXL_ASSERT(AreSameLaneSize(zd, zn1, zn2)); 5956 5957 Emit(0x052d8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn1)); 5958} 5959 5960// SVEPermuteVectorUnpredicated. 5961 5962void Assembler::dup(const ZRegister& zd, const Register& xn) { 5963 // DUP <Zd>.<T>, <R><n|SP> 5964 // 0000 0101 ..10 0000 0011 10.. .... .... 5965 // size<23:22> | Rn<9:5> | Zd<4:0> 5966 5967 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5968 5969 Emit(DUP_z_r | SVESize(zd) | Rd(zd) | RnSP(xn)); 5970} 5971 5972void Assembler::dup(const ZRegister& zd, const ZRegister& zn, unsigned index) { 5973 // DUP <Zd>.<T>, <Zn>.<T>[<imm>] 5974 // 0000 0101 ..1. .... 0010 00.. .... .... 5975 // imm2<23:22> | tsz<20:16> | Zn<9:5> | Zd<4:0> 5976 5977 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5978 VIXL_ASSERT(AreSameFormat(zd, zn)); 5979 VIXL_ASSERT((index * zd.GetLaneSizeInBits()) < 512); 5980 int n = zd.GetLaneSizeInBytesLog2(); 5981 unsigned imm_7 = (index << (n + 1)) | (1 << n); 5982 VIXL_ASSERT(IsUint7(imm_7)); 5983 unsigned imm_2 = ExtractUnsignedBitfield32(6, 5, imm_7); 5984 unsigned tsz_5 = ExtractUnsignedBitfield32(4, 0, imm_7); 5985 5986 Emit(DUP_z_zi | ImmUnsignedField<23, 22>(imm_2) | 5987 ImmUnsignedField<20, 16>(tsz_5) | Rd(zd) | Rn(zn)); 5988} 5989 5990void Assembler::insr(const ZRegister& zdn, const Register& rm) { 5991 // INSR <Zdn>.<T>, <R><m> 5992 // 0000 0101 ..10 0100 0011 10.. .... .... 5993 // size<23:22> | Rm<9:5> | Zdn<4:0> 5994 5995 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 5996 5997 Emit(INSR_z_r | SVESize(zdn) | Rd(zdn) | Rn(rm)); 5998} 5999 6000void Assembler::insr(const ZRegister& zdn, const VRegister& vm) { 6001 // INSR <Zdn>.<T>, <V><m> 6002 // 0000 0101 ..11 0100 0011 10.. .... .... 6003 // size<23:22> | Vm<9:5> | Zdn<4:0> 6004 6005 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6006 VIXL_ASSERT(vm.IsScalar()); 6007 6008 Emit(INSR_z_v | SVESize(zdn) | Rd(zdn) | Rn(vm)); 6009} 6010 6011void Assembler::rev(const ZRegister& zd, const ZRegister& zn) { 6012 // REV <Zd>.<T>, <Zn>.<T> 6013 // 0000 0101 ..11 1000 0011 10.. .... .... 6014 // size<23:22> | Zn<9:5> | Zd<4:0> 6015 6016 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6017 VIXL_ASSERT(AreSameFormat(zd, zn)); 6018 6019 Emit(REV_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); 6020} 6021 6022void Assembler::sunpkhi(const ZRegister& zd, const ZRegister& zn) { 6023 // SUNPKHI <Zd>.<T>, <Zn>.<Tb> 6024 // 0000 0101 ..11 0001 0011 10.. .... .... 6025 // size<23:22> | U<17> = 0 | H<16> = 1 | Zn<9:5> | Zd<4:0> 6026 6027 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6028 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 6029 VIXL_ASSERT(!zd.IsLaneSizeB()); 6030 6031 Emit(SUNPKHI_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); 6032} 6033 6034void Assembler::sunpklo(const ZRegister& zd, const ZRegister& zn) { 6035 // SUNPKLO <Zd>.<T>, <Zn>.<Tb> 6036 // 0000 0101 ..11 0000 0011 10.. .... .... 6037 // size<23:22> | U<17> = 0 | H<16> = 0 | Zn<9:5> | Zd<4:0> 6038 6039 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6040 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 6041 VIXL_ASSERT(!zd.IsLaneSizeB()); 6042 6043 Emit(SUNPKLO_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); 6044} 6045 6046void Assembler::tbl(const ZRegister& zd, 6047 const ZRegister& zn, 6048 const ZRegister& zm) { 6049 // TBL <Zd>.<T>, { <Zn>.<T> }, <Zm>.<T> 6050 // 0000 0101 ..1. .... 0011 00.. .... .... 6051 // size<23:22> | Zm<20:16> | Zn<9:5> | Zd<4:0> 6052 6053 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6054 VIXL_ASSERT(AreSameFormat(zd, zn, zm)); 6055 6056 Emit(TBL_z_zz_1 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 6057} 6058 6059void Assembler::uunpkhi(const ZRegister& zd, const ZRegister& zn) { 6060 // UUNPKHI <Zd>.<T>, <Zn>.<Tb> 6061 // 0000 0101 ..11 0011 0011 10.. .... .... 6062 // size<23:22> | U<17> = 1 | H<16> = 1 | Zn<9:5> | Zd<4:0> 6063 6064 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6065 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 6066 VIXL_ASSERT(!zd.IsLaneSizeB()); 6067 6068 Emit(UUNPKHI_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); 6069} 6070 6071void Assembler::uunpklo(const ZRegister& zd, const ZRegister& zn) { 6072 // UUNPKLO <Zd>.<T>, <Zn>.<Tb> 6073 // 0000 0101 ..11 0010 0011 10.. .... .... 6074 // size<23:22> | U<17> = 1 | H<16> = 0 | Zn<9:5> | Zd<4:0> 6075 6076 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6077 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 6078 VIXL_ASSERT(!zd.IsLaneSizeB()); 6079 6080 Emit(UUNPKLO_z_z | SVESize(zd) | Rd(zd) | Rn(zn)); 6081} 6082 6083// SVEPredicateCount. 6084 6085void Assembler::cntp(const Register& xd, 6086 const PRegister& pg, 6087 const PRegisterWithLaneSize& pn) { 6088 // CNTP <Xd>, <Pg>, <Pn>.<T> 6089 // 0010 0101 ..10 0000 10.. ..0. .... .... 6090 // size<23:22> | opc<18:16> = 000 | Pg<13:10> | o2<9> = 0 | Pn<8:5> | Rd<4:0> 6091 6092 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6093 VIXL_ASSERT(xd.IsX()); 6094 VIXL_ASSERT(pg.IsUnqualified()); 6095 if (pg.HasLaneSize()) VIXL_ASSERT(AreSameFormat(pg, pn)); 6096 6097 Emit(CNTP_r_p_p | SVESize(pn) | Rd(xd) | Pg<13, 10>(pg) | Pn(pn)); 6098} 6099 6100// SVEPredicateLogicalOp. 6101void Assembler::and_(const PRegisterWithLaneSize& pd, 6102 const PRegisterZ& pg, 6103 const PRegisterWithLaneSize& pn, 6104 const PRegisterWithLaneSize& pm) { 6105 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6106 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6107 VIXL_ASSERT(pd.IsLaneSizeB()); 6108 Emit(AND_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6109} 6110 6111void Assembler::ands(const PRegisterWithLaneSize& pd, 6112 const PRegisterZ& pg, 6113 const PRegisterWithLaneSize& pn, 6114 const PRegisterWithLaneSize& pm) { 6115 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6116 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6117 VIXL_ASSERT(pd.IsLaneSizeB()); 6118 Emit(ANDS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6119} 6120 6121void Assembler::bic(const PRegisterWithLaneSize& pd, 6122 const PRegisterZ& pg, 6123 const PRegisterWithLaneSize& pn, 6124 const PRegisterWithLaneSize& pm) { 6125 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6126 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6127 VIXL_ASSERT(pd.IsLaneSizeB()); 6128 Emit(BIC_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6129} 6130 6131void Assembler::bics(const PRegisterWithLaneSize& pd, 6132 const PRegisterZ& pg, 6133 const PRegisterWithLaneSize& pn, 6134 const PRegisterWithLaneSize& pm) { 6135 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6136 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6137 VIXL_ASSERT(pd.IsLaneSizeB()); 6138 Emit(BICS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6139} 6140 6141void Assembler::eor(const PRegisterWithLaneSize& pd, 6142 const PRegisterZ& pg, 6143 const PRegisterWithLaneSize& pn, 6144 const PRegisterWithLaneSize& pm) { 6145 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6146 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6147 VIXL_ASSERT(pd.IsLaneSizeB()); 6148 Emit(EOR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6149} 6150 6151void Assembler::eors(const PRegisterWithLaneSize& pd, 6152 const PRegisterZ& pg, 6153 const PRegisterWithLaneSize& pn, 6154 const PRegisterWithLaneSize& pm) { 6155 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6156 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6157 VIXL_ASSERT(pd.IsLaneSizeB()); 6158 Emit(EORS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6159} 6160 6161void Assembler::nand(const PRegisterWithLaneSize& pd, 6162 const PRegisterZ& pg, 6163 const PRegisterWithLaneSize& pn, 6164 const PRegisterWithLaneSize& pm) { 6165 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6166 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6167 VIXL_ASSERT(pd.IsLaneSizeB()); 6168 Emit(NAND_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6169} 6170 6171void Assembler::nands(const PRegisterWithLaneSize& pd, 6172 const PRegisterZ& pg, 6173 const PRegisterWithLaneSize& pn, 6174 const PRegisterWithLaneSize& pm) { 6175 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6176 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6177 VIXL_ASSERT(pd.IsLaneSizeB()); 6178 Emit(NANDS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6179} 6180 6181void Assembler::nor(const PRegisterWithLaneSize& pd, 6182 const PRegisterZ& pg, 6183 const PRegisterWithLaneSize& pn, 6184 const PRegisterWithLaneSize& pm) { 6185 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6186 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6187 VIXL_ASSERT(pd.IsLaneSizeB()); 6188 Emit(NOR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6189} 6190 6191void Assembler::nors(const PRegisterWithLaneSize& pd, 6192 const PRegisterZ& pg, 6193 const PRegisterWithLaneSize& pn, 6194 const PRegisterWithLaneSize& pm) { 6195 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6196 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6197 VIXL_ASSERT(pd.IsLaneSizeB()); 6198 Emit(NORS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6199} 6200 6201void Assembler::orn(const PRegisterWithLaneSize& pd, 6202 const PRegisterZ& pg, 6203 const PRegisterWithLaneSize& pn, 6204 const PRegisterWithLaneSize& pm) { 6205 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6206 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6207 VIXL_ASSERT(pd.IsLaneSizeB()); 6208 Emit(ORN_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6209} 6210 6211void Assembler::orns(const PRegisterWithLaneSize& pd, 6212 const PRegisterZ& pg, 6213 const PRegisterWithLaneSize& pn, 6214 const PRegisterWithLaneSize& pm) { 6215 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6216 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6217 VIXL_ASSERT(pd.IsLaneSizeB()); 6218 Emit(ORNS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6219} 6220 6221void Assembler::orr(const PRegisterWithLaneSize& pd, 6222 const PRegisterZ& pg, 6223 const PRegisterWithLaneSize& pn, 6224 const PRegisterWithLaneSize& pm) { 6225 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6226 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6227 VIXL_ASSERT(pd.IsLaneSizeB()); 6228 Emit(ORR_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6229} 6230 6231void Assembler::orrs(const PRegisterWithLaneSize& pd, 6232 const PRegisterZ& pg, 6233 const PRegisterWithLaneSize& pn, 6234 const PRegisterWithLaneSize& pm) { 6235 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6236 VIXL_ASSERT(AreSameFormat(pd, pn, pm)); 6237 VIXL_ASSERT(pd.IsLaneSizeB()); 6238 Emit(ORRS_p_p_pp_z | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6239} 6240 6241void Assembler::sel(const PRegisterWithLaneSize& pd, 6242 const PRegister& pg, 6243 const PRegisterWithLaneSize& pn, 6244 const PRegisterWithLaneSize& pm) { 6245 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6246 Emit(SEL_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6247} 6248 6249// SVEPredicateMisc. 6250 6251void Assembler::pfalse(const PRegisterWithLaneSize& pd) { 6252 // PFALSE <Pd>.B 6253 // 0010 0101 0001 1000 1110 0100 0000 .... 6254 // op<23> = 0 | S<22> = 0 | Pd<3:0> 6255 6256 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6257 // Ignore the lane size, since it makes no difference to the operation. 6258 6259 Emit(PFALSE_p | Pd(pd)); 6260} 6261 6262void Assembler::pfirst(const PRegisterWithLaneSize& pd, 6263 const PRegister& pg, 6264 const PRegisterWithLaneSize& pn) { 6265 // PFIRST <Pdn>.B, <Pg>, <Pdn>.B 6266 // 0010 0101 0101 1000 1100 000. ...0 .... 6267 // op<23> = 0 | S<22> = 1 | Pg<8:5> | Pdn<3:0> 6268 6269 USE(pn); 6270 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6271 VIXL_ASSERT(pd.Is(pn)); 6272 VIXL_ASSERT(pd.IsLaneSizeB()); 6273 6274 Emit(PFIRST_p_p_p | Pd(pd) | Pg<8, 5>(pg)); 6275} 6276 6277void Assembler::pnext(const PRegisterWithLaneSize& pd, 6278 const PRegister& pg, 6279 const PRegisterWithLaneSize& pn) { 6280 // PNEXT <Pdn>.<T>, <Pg>, <Pdn>.<T> 6281 // 0010 0101 ..01 1001 1100 010. ...0 .... 6282 // size<23:22> | Pg<8:5> | Pdn<3:0> 6283 6284 USE(pn); 6285 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6286 VIXL_ASSERT(pd.Is(pn)); 6287 6288 Emit(PNEXT_p_p_p | SVESize(pd) | Pd(pd) | Pg<8, 5>(pg)); 6289} 6290 6291void Assembler::ptest(const PRegister& pg, const PRegisterWithLaneSize& pn) { 6292 // PTEST <Pg>, <Pn>.B 6293 // 0010 0101 0101 0000 11.. ..0. ...0 0000 6294 // op<23> = 0 | S<22> = 1 | Pg<13:10> | Pn<8:5> | opc2<3:0> = 0000 6295 6296 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6297 VIXL_ASSERT(pn.IsLaneSizeB()); 6298 6299 Emit(PTEST_p_p | Pg<13, 10>(pg) | Rx<8, 5>(pn)); 6300} 6301 6302void Assembler::ptrue(const PRegisterWithLaneSize& pd, int pattern) { 6303 // PTRUE <Pd>.<T>{, <pattern>} 6304 // 0010 0101 ..01 1000 1110 00.. ...0 .... 6305 // size<23:22> | S<16> = 0 | pattern<9:5> | Pd<3:0> 6306 6307 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6308 6309 Emit(PTRUE_p_s | SVESize(pd) | Pd(pd) | ImmSVEPredicateConstraint(pattern)); 6310} 6311 6312void Assembler::ptrues(const PRegisterWithLaneSize& pd, int pattern) { 6313 // PTRUES <Pd>.<T>{, <pattern>} 6314 // 0010 0101 ..01 1001 1110 00.. ...0 .... 6315 // size<23:22> | S<16> = 1 | pattern<9:5> | Pd<3:0> 6316 6317 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6318 6319 Emit(PTRUES_p_s | SVESize(pd) | Pd(pd) | ImmSVEPredicateConstraint(pattern)); 6320} 6321 6322void Assembler::rdffr(const PRegisterWithLaneSize& pd) { 6323 // RDFFR <Pd>.B 6324 // 0010 0101 0001 1001 1111 0000 0000 .... 6325 // op<23> = 0 | S<22> = 0 | Pd<3:0> 6326 6327 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6328 6329 Emit(RDFFR_p_f | Pd(pd)); 6330} 6331 6332void Assembler::rdffr(const PRegisterWithLaneSize& pd, const PRegisterZ& pg) { 6333 // RDFFR <Pd>.B, <Pg>/Z 6334 // 0010 0101 0001 1000 1111 000. ...0 .... 6335 // op<23> = 0 | S<22> = 0 | Pg<8:5> | Pd<3:0> 6336 6337 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6338 6339 Emit(RDFFR_p_p_f | Pd(pd) | Pg<8, 5>(pg)); 6340} 6341 6342void Assembler::rdffrs(const PRegisterWithLaneSize& pd, const PRegisterZ& pg) { 6343 // RDFFRS <Pd>.B, <Pg>/Z 6344 // 0010 0101 0101 1000 1111 000. ...0 .... 6345 // op<23> = 0 | S<22> = 1 | Pg<8:5> | Pd<3:0> 6346 6347 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6348 6349 Emit(RDFFRS_p_p_f | Pd(pd) | Pg<8, 5>(pg)); 6350} 6351 6352// SVEPropagateBreak. 6353 6354void Assembler::brkpa(const PRegisterWithLaneSize& pd, 6355 const PRegisterZ& pg, 6356 const PRegisterWithLaneSize& pn, 6357 const PRegisterWithLaneSize& pm) { 6358 // BRKPA <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B 6359 // 0010 0101 0000 .... 11.. ..0. ...0 .... 6360 // op<23> = 0 | S<22> = 0 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 0 | 6361 // Pd<3:0> 6362 6363 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6364 6365 Emit(BRKPA_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6366} 6367 6368void Assembler::brkpas(const PRegisterWithLaneSize& pd, 6369 const PRegisterZ& pg, 6370 const PRegisterWithLaneSize& pn, 6371 const PRegisterWithLaneSize& pm) { 6372 // BRKPAS <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B 6373 // 0010 0101 0100 .... 11.. ..0. ...0 .... 6374 // op<23> = 0 | S<22> = 1 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 0 | 6375 // Pd<3:0> 6376 6377 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6378 6379 Emit(BRKPAS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6380} 6381 6382void Assembler::brkpb(const PRegisterWithLaneSize& pd, 6383 const PRegisterZ& pg, 6384 const PRegisterWithLaneSize& pn, 6385 const PRegisterWithLaneSize& pm) { 6386 // BRKPB <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B 6387 // 0010 0101 0000 .... 11.. ..0. ...1 .... 6388 // op<23> = 0 | S<22> = 0 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 1 | 6389 // Pd<3:0> 6390 6391 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6392 6393 Emit(BRKPB_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6394} 6395 6396void Assembler::brkpbs(const PRegisterWithLaneSize& pd, 6397 const PRegisterZ& pg, 6398 const PRegisterWithLaneSize& pn, 6399 const PRegisterWithLaneSize& pm) { 6400 // BRKPBS <Pd>.B, <Pg>/Z, <Pn>.B, <Pm>.B 6401 // 0010 0101 0100 .... 11.. ..0. ...1 .... 6402 // op<23> = 0 | S<22> = 1 | Pm<19:16> | Pg<13:10> | Pn<8:5> | B<4> = 1 | 6403 // Pd<3:0> 6404 6405 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6406 6407 Emit(BRKPBS_p_p_pp | Pd(pd) | Pg<13, 10>(pg) | Pn(pn) | Pm(pm)); 6408} 6409 6410// SVEStackFrameAdjustment. 6411 6412void Assembler::addpl(const Register& xd, const Register& xn, int imm6) { 6413 // ADDPL <Xd|SP>, <Xn|SP>, #<imm> 6414 // 0000 0100 011. .... 0101 0... .... .... 6415 // op<22> = 1 | Rn<20:16> | imm6<10:5> | Rd<4:0> 6416 6417 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6418 VIXL_ASSERT(xd.IsX()); 6419 VIXL_ASSERT(xn.IsX()); 6420 6421 Emit(ADDPL_r_ri | RdSP(xd) | RmSP(xn) | ImmField<10, 5>(imm6)); 6422} 6423 6424void Assembler::addvl(const Register& xd, const Register& xn, int imm6) { 6425 // ADDVL <Xd|SP>, <Xn|SP>, #<imm> 6426 // 0000 0100 001. .... 0101 0... .... .... 6427 // op<22> = 0 | Rn<20:16> | imm6<10:5> | Rd<4:0> 6428 6429 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6430 VIXL_ASSERT(xd.IsX()); 6431 VIXL_ASSERT(xn.IsX()); 6432 6433 Emit(ADDVL_r_ri | RdSP(xd) | RmSP(xn) | ImmField<10, 5>(imm6)); 6434} 6435 6436// SVEStackFrameSize. 6437 6438void Assembler::rdvl(const Register& xd, int imm6) { 6439 // RDVL <Xd>, #<imm> 6440 // 0000 0100 1011 1111 0101 0... .... .... 6441 // op<22> = 0 | opc2<20:16> = 11111 | imm6<10:5> | Rd<4:0> 6442 6443 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6444 VIXL_ASSERT(xd.IsX()); 6445 6446 Emit(RDVL_r_i | Rd(xd) | ImmField<10, 5>(imm6)); 6447} 6448 6449// SVEVectorSelect. 6450 6451void Assembler::sel(const ZRegister& zd, 6452 const PRegister& pg, 6453 const ZRegister& zn, 6454 const ZRegister& zm) { 6455 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6456 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 6457 6458 Emit(SEL_z_p_zz | SVESize(zd) | Rd(zd) | Pg<13, 10>(pg) | Rn(zn) | Rm(zm)); 6459} 6460 6461// SVEWriteFFR. 6462 6463void Assembler::setffr() { 6464 // SETFFR 6465 // 0010 0101 0010 1100 1001 0000 0000 0000 6466 // opc<23:22> = 00 6467 6468 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6469 6470 Emit(SETFFR_f); 6471} 6472 6473void Assembler::wrffr(const PRegisterWithLaneSize& pn) { 6474 // WRFFR <Pn>.B 6475 // 0010 0101 0010 1000 1001 000. ...0 0000 6476 // opc<23:22> = 00 | Pn<8:5> 6477 6478 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6479 6480 Emit(WRFFR_f_p | Rx<8, 5>(pn)); 6481} 6482 6483// Aliases. 6484 6485void Assembler::bic(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { 6486 and_(zd, zn, ~imm); 6487} 6488 6489void Assembler::eon(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { 6490 eor(zd, zn, ~imm); 6491} 6492 6493void Assembler::orn(const ZRegister& zd, const ZRegister& zn, uint64_t imm) { 6494 orr(zd, zn, ~imm); 6495} 6496 6497 6498void Assembler::fmov(const ZRegister& zd, const PRegisterM& pg, double imm) { 6499 if (IsPositiveZero(imm)) { 6500 cpy(zd, pg, 0); 6501 } else { 6502 fcpy(zd, pg, imm); 6503 } 6504} 6505 6506void Assembler::fmov(const ZRegister& zd, double imm) { 6507 if (IsPositiveZero(imm)) { 6508 dup(zd, 0); 6509 } else { 6510 fdup(zd, imm); 6511 } 6512} 6513 6514void Assembler::mov(const PRegister& pd, const PRegister& pn) { 6515 // If the inputs carry a lane size, they must match. 6516 VIXL_ASSERT((!pd.HasLaneSize() && !pn.HasLaneSize()) || 6517 AreSameLaneSize(pd, pn)); 6518 orr(pd.VnB(), pn.Zeroing(), pn.VnB(), pn.VnB()); 6519} 6520 6521void Assembler::mov(const PRegisterWithLaneSize& pd, 6522 const PRegisterM& pg, 6523 const PRegisterWithLaneSize& pn) { 6524 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6525 sel(pd, pg, pn, pd); 6526} 6527 6528void Assembler::mov(const PRegisterWithLaneSize& pd, 6529 const PRegisterZ& pg, 6530 const PRegisterWithLaneSize& pn) { 6531 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6532 and_(pd, pg, pn, pn); 6533} 6534 6535void Assembler::mov(const ZRegister& zd, 6536 const PRegister& pg, 6537 int imm8, 6538 int shift) { 6539 VIXL_ASSERT(pg.IsMerging() || pg.IsZeroing()); 6540 cpy(zd, pg, imm8, shift); 6541} 6542 6543void Assembler::mov(const ZRegister& zd, const Register& xn) { dup(zd, xn); } 6544 6545void Assembler::mov(const ZRegister& zd, const VRegister& vn) { 6546 VIXL_ASSERT(vn.IsScalar()); 6547 VIXL_ASSERT(AreSameLaneSize(zd, vn)); 6548 dup(zd, vn.Z().WithSameLaneSizeAs(vn), 0); 6549} 6550 6551void Assembler::mov(const ZRegister& zd, const ZRegister& zn) { 6552 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 6553 orr(zd.VnD(), zn.VnD(), zn.VnD()); 6554} 6555 6556void Assembler::mov(const ZRegister& zd, const ZRegister& zn, unsigned index) { 6557 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 6558 dup(zd, zn, index); 6559} 6560 6561void Assembler::mov(const ZRegister& zd, 6562 const PRegisterM& pg, 6563 const Register& rn) { 6564 cpy(zd, pg, rn); 6565} 6566 6567void Assembler::mov(const ZRegister& zd, 6568 const PRegisterM& pg, 6569 const VRegister& vn) { 6570 VIXL_ASSERT(vn.IsScalar()); 6571 VIXL_ASSERT(AreSameLaneSize(zd, vn)); 6572 cpy(zd, pg, vn); 6573} 6574 6575void Assembler::mov(const ZRegister& zd, 6576 const PRegisterM& pg, 6577 const ZRegister& zn) { 6578 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 6579 sel(zd, pg, zn, zd); 6580} 6581 6582void Assembler::mov(const ZRegister& zd, uint64_t imm) { 6583 // Mov is an alias of dupm for certain values of imm. Whilst this matters in 6584 // the disassembler, for the assembler, we don't distinguish between the 6585 // two mnemonics, and simply call dupm. 6586 dupm(zd, imm); 6587} 6588 6589void Assembler::mov(const ZRegister& zd, int imm8, int shift) { 6590 dup(zd, imm8, shift); 6591} 6592 6593void Assembler::movs(const PRegister& pd, const PRegister& pn) { 6594 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6595 orrs(pd.VnB(), pn.Zeroing(), pn.VnB(), pn.VnB()); 6596} 6597 6598void Assembler::movs(const PRegisterWithLaneSize& pd, 6599 const PRegisterZ& pg, 6600 const PRegisterWithLaneSize& pn) { 6601 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6602 ands(pd, pg, pn, pn); 6603} 6604 6605void Assembler::not_(const PRegisterWithLaneSize& pd, 6606 const PRegisterZ& pg, 6607 const PRegisterWithLaneSize& pn) { 6608 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6609 eor(pd, pg, pn, pg.VnB()); 6610} 6611 6612void Assembler::nots(const PRegisterWithLaneSize& pd, 6613 const PRegisterZ& pg, 6614 const PRegisterWithLaneSize& pn) { 6615 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 6616 eors(pd, pg, pn, pg.VnB()); 6617} 6618 6619// SVE2 6620 6621void Assembler::adclb(const ZRegister& zda, 6622 const ZRegister& zn, 6623 const ZRegister& zm) { 6624 // ADCLB <Zda>.<T>, <Zn>.<T>, <Zm>.<T> 6625 // 0100 0101 0.0. .... 1101 00.. .... .... 6626 // size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0> 6627 6628 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6629 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 6630 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 6631 6632 Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0; 6633 Emit(0x4500d000 | sz | Rd(zda) | Rn(zn) | Rm(zm)); 6634} 6635 6636void Assembler::adclt(const ZRegister& zda, 6637 const ZRegister& zn, 6638 const ZRegister& zm) { 6639 // ADCLT <Zda>.<T>, <Zn>.<T>, <Zm>.<T> 6640 // 0100 0101 0.0. .... 1101 01.. .... .... 6641 // size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0> 6642 6643 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6644 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 6645 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 6646 6647 Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0; 6648 Emit(0x4500d400 | sz | Rd(zda) | Rn(zn) | Rm(zm)); 6649} 6650 6651void Assembler::addhnb(const ZRegister& zd, 6652 const ZRegister& zn, 6653 const ZRegister& zm) { 6654 // ADDHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 6655 // 0100 0101 ..1. .... 0110 00.. .... .... 6656 // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> 6657 6658 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6659 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 6660 VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); 6661 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 6662 6663 Emit(0x45206000 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); 6664} 6665 6666void Assembler::addhnt(const ZRegister& zd, 6667 const ZRegister& zn, 6668 const ZRegister& zm) { 6669 // ADDHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 6670 // 0100 0101 ..1. .... 0110 01.. .... .... 6671 // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> 6672 6673 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6674 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 6675 VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); 6676 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 6677 6678 Emit(0x45206400 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); 6679} 6680 6681void Assembler::addp(const ZRegister& zd, 6682 const PRegisterM& pg, 6683 const ZRegister& zn, 6684 const ZRegister& zm) { 6685 // ADDP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 6686 // 0100 0100 ..01 0001 101. .... .... .... 6687 // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 6688 6689 USE(zn); 6690 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6691 VIXL_ASSERT(zd.Is(zn)); 6692 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 6693 6694 Emit(0x4411a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 6695} 6696 6697void Assembler::bcax(const ZRegister& zd, 6698 const ZRegister& zn, 6699 const ZRegister& zm, 6700 const ZRegister& zk) { 6701 // BCAX <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D 6702 // 0000 0100 011. .... 0011 10.. .... .... 6703 // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> 6704 6705 USE(zn); 6706 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6707 VIXL_ASSERT(zd.Is(zn)); 6708 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk)); 6709 VIXL_ASSERT(zd.IsLaneSizeD()); 6710 6711 Emit(0x04603800 | Rd(zd) | Rm(zm) | Rn(zk)); 6712} 6713 6714void Assembler::bdep(const ZRegister& zd, 6715 const ZRegister& zn, 6716 const ZRegister& zm) { 6717 // BDEP <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 6718 // 0100 0101 ..0. .... 1011 01.. .... .... 6719 // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> 6720 6721 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6722 VIXL_ASSERT(CPUHas(CPUFeatures::kSVEBitPerm)); 6723 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 6724 6725 Emit(0x4500b400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 6726} 6727 6728void Assembler::bext(const ZRegister& zd, 6729 const ZRegister& zn, 6730 const ZRegister& zm) { 6731 // BEXT <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 6732 // 0100 0101 ..0. .... 1011 00.. .... .... 6733 // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> 6734 6735 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6736 VIXL_ASSERT(CPUHas(CPUFeatures::kSVEBitPerm)); 6737 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 6738 6739 Emit(0x4500b000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 6740} 6741 6742void Assembler::bgrp(const ZRegister& zd, 6743 const ZRegister& zn, 6744 const ZRegister& zm) { 6745 // BGRP <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 6746 // 0100 0101 ..0. .... 1011 10.. .... .... 6747 // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> 6748 6749 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6750 VIXL_ASSERT(CPUHas(CPUFeatures::kSVEBitPerm)); 6751 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 6752 6753 Emit(0x4500b800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 6754} 6755 6756void Assembler::bsl(const ZRegister& zd, 6757 const ZRegister& zn, 6758 const ZRegister& zm, 6759 const ZRegister& zk) { 6760 // BSL <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D 6761 // 0000 0100 001. .... 0011 11.. .... .... 6762 // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> 6763 6764 USE(zn); 6765 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6766 VIXL_ASSERT(zd.Is(zn)); 6767 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk)); 6768 VIXL_ASSERT(zd.IsLaneSizeD()); 6769 6770 Emit(0x04203c00 | Rd(zd) | Rm(zm) | Rn(zk)); 6771} 6772 6773void Assembler::bsl1n(const ZRegister& zd, 6774 const ZRegister& zn, 6775 const ZRegister& zm, 6776 const ZRegister& zk) { 6777 // BSL1N <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D 6778 // 0000 0100 011. .... 0011 11.. .... .... 6779 // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> 6780 6781 USE(zn); 6782 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6783 VIXL_ASSERT(zd.Is(zn)); 6784 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk)); 6785 VIXL_ASSERT(zd.IsLaneSizeD()); 6786 6787 Emit(0x04603c00 | Rd(zd) | Rm(zm) | Rn(zk)); 6788} 6789 6790void Assembler::bsl2n(const ZRegister& zd, 6791 const ZRegister& zn, 6792 const ZRegister& zm, 6793 const ZRegister& zk) { 6794 // BSL2N <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D 6795 // 0000 0100 101. .... 0011 11.. .... .... 6796 // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> 6797 6798 USE(zn); 6799 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6800 VIXL_ASSERT(zd.Is(zn)); 6801 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk)); 6802 VIXL_ASSERT(zd.IsLaneSizeD()); 6803 6804 Emit(0x04a03c00 | Rd(zd) | Rm(zm) | Rn(zk)); 6805} 6806 6807void Assembler::cadd(const ZRegister& zd, 6808 const ZRegister& zn, 6809 const ZRegister& zm, 6810 int rot) { 6811 // CADD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, <const> 6812 // 0100 0101 ..00 0000 1101 1... .... .... 6813 // size<23:22> | op<16> | rot<10> | Zm<9:5> | Zdn<4:0> 6814 6815 USE(zn); 6816 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6817 VIXL_ASSERT(zd.Is(zn)); 6818 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 6819 VIXL_ASSERT((rot == 90) || (rot == 270)); 6820 6821 Instr rotate_bit = (rot == 90) ? 0 : (1 << 10); 6822 Emit(0x4500d800 | rotate_bit | SVESize(zd) | Rd(zd) | Rn(zm)); 6823} 6824 6825void Assembler::cdot(const ZRegister& zda, 6826 const ZRegister& zn, 6827 const ZRegister& zm, 6828 int index, 6829 int rot) { 6830 // CDOT <Zda>.D, <Zn>.H, <Zm>.H[<imm>], <const> 6831 // 0100 0100 111. .... 0100 .... .... .... 6832 // size<23:22> | opc<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0> 6833 6834 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6835 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); 6836 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 6837 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 6838 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); 6839 VIXL_ASSERT(index >= 0); 6840 6841 Instr zm_and_idx = 0; 6842 if (zm.IsLaneSizeB()) { 6843 // Zm<18:16> | i2<20:19> 6844 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 3)); 6845 zm_and_idx = (index << 19) | Rx<18, 16>(zm); 6846 } else { 6847 // Zm<19:16> | i1<20> 6848 VIXL_ASSERT(zm.IsLaneSizeH()); 6849 VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 1)); 6850 zm_and_idx = (index << 20) | Rx<19, 16>(zm); 6851 } 6852 6853 Instr rotate_bits = (rot / 90) << 10; 6854 Emit(0x44a04000 | zm_and_idx | rotate_bits | SVESize(zda) | Rd(zda) | Rn(zn)); 6855} 6856 6857void Assembler::cdot(const ZRegister& zda, 6858 const ZRegister& zn, 6859 const ZRegister& zm, 6860 int rot) { 6861 // CDOT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>, <const> 6862 // 0100 0100 ..0. .... 0001 .... .... .... 6863 // size<23:22> | Zm<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0> 6864 6865 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6866 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); 6867 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 6868 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 6869 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 4)); 6870 6871 Instr rotate_bits = (rot / 90) << 10; 6872 Emit(0x44001000 | rotate_bits | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 6873} 6874 6875void Assembler::cmla(const ZRegister& zda, 6876 const ZRegister& zn, 6877 const ZRegister& zm, 6878 int index, 6879 int rot) { 6880 // CMLA <Zda>.H, <Zn>.H, <Zm>.H[<imm>], <const> 6881 // 0100 0100 101. .... 0110 .... .... .... 6882 // size<23:22> | opc<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0> 6883 6884 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6885 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 6886 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); 6887 6888 Instr rotate_bit = (rot / 90) << 10; 6889 Emit(0x44a06000 | SVEMulComplexIndexHelper(zm, index) | rotate_bit | Rd(zda) | 6890 Rn(zn)); 6891} 6892 6893void Assembler::cmla(const ZRegister& zda, 6894 const ZRegister& zn, 6895 const ZRegister& zm, 6896 int rot) { 6897 // CMLA <Zda>.<T>, <Zn>.<T>, <Zm>.<T>, <const> 6898 // 0100 0100 ..0. .... 0010 .... .... .... 6899 // size<23:22> | Zm<20:16> | op<12> | rot<11:10> | Zn<9:5> | Zda<4:0> 6900 6901 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6902 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 6903 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); 6904 6905 Instr rotate_bit = (rot / 90) << 10; 6906 Emit(0x44002000 | rotate_bit | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 6907} 6908 6909void Assembler::eor3(const ZRegister& zd, 6910 const ZRegister& zn, 6911 const ZRegister& zm, 6912 const ZRegister& zk) { 6913 // EOR3 <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D 6914 // 0000 0100 001. .... 0011 10.. .... .... 6915 // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> 6916 6917 USE(zn); 6918 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6919 VIXL_ASSERT(zd.Is(zn)); 6920 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk)); 6921 VIXL_ASSERT(zd.IsLaneSizeD()); 6922 6923 Emit(0x04203800 | Rd(zd) | Rm(zm) | Rn(zk)); 6924} 6925 6926void Assembler::eorbt(const ZRegister& zd, 6927 const ZRegister& zn, 6928 const ZRegister& zm) { 6929 // EORBT <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 6930 // 0100 0101 ..0. .... 1001 00.. .... .... 6931 // size<23:22> | Zm<20:16> | tb<10> | Zn<9:5> | Zd<4:0> 6932 6933 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6934 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 6935 6936 Emit(0x45009000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 6937} 6938 6939void Assembler::eortb(const ZRegister& zd, 6940 const ZRegister& zn, 6941 const ZRegister& zm) { 6942 // EORTB <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 6943 // 0100 0101 ..0. .... 1001 01.. .... .... 6944 // size<23:22> | Zm<20:16> | tb<10> | Zn<9:5> | Zd<4:0> 6945 6946 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6947 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 6948 6949 Emit(0x45009400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 6950} 6951 6952void Assembler::faddp(const ZRegister& zd, 6953 const PRegisterM& pg, 6954 const ZRegister& zn, 6955 const ZRegister& zm) { 6956 // FADDP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 6957 // 0110 0100 ..01 0000 100. .... .... .... 6958 // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 6959 6960 USE(zn); 6961 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6962 VIXL_ASSERT(zd.Is(zn)); 6963 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 6964 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 6965 6966 Emit(0x64108000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 6967} 6968 6969void Assembler::fcvtlt(const ZRegister& zd, 6970 const PRegisterM& pg, 6971 const ZRegister& zn) { 6972 // FCVTLT <Zd>.S, <Pg>/M, <Zn>.H 6973 // 0110 0100 1000 1001 101. .... .... .... 6974 // opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 6975 6976 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6977 6978 Instr op; 6979 if (zd.IsLaneSizeD() && zn.IsLaneSizeS()) { 6980 op = 0x64cba000; 6981 } else { 6982 VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeH()); 6983 op = 0x6489a000; 6984 } 6985 6986 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); 6987} 6988 6989void Assembler::fcvtnt(const ZRegister& zd, 6990 const PRegisterM& pg, 6991 const ZRegister& zn) { 6992 // FCVTNT <Zd>.S, <Pg>/M, <Zn>.D 6993 // 0110 0100 1100 1010 101. .... .... .... 6994 // opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 6995 6996 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 6997 6998 Instr op; 6999 if (zd.IsLaneSizeS() && zn.IsLaneSizeD()) { 7000 op = 0x64caa000; 7001 } else { 7002 VIXL_ASSERT(zd.IsLaneSizeH() && zn.IsLaneSizeS()); 7003 op = 0x6488a000; 7004 } 7005 Emit(op | Rd(zd) | PgLow8(pg) | Rn(zn)); 7006} 7007 7008void Assembler::fcvtx(const ZRegister& zd, 7009 const PRegisterM& pg, 7010 const ZRegister& zn) { 7011 // FCVTX <Zd>.S, <Pg>/M, <Zn>.D 7012 // 0110 0101 0000 1010 101. .... .... .... 7013 // opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 7014 7015 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7016 VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeD()); 7017 7018 Emit(0x650aa000 | Rd(zd) | PgLow8(pg) | Rn(zn)); 7019} 7020 7021void Assembler::fcvtxnt(const ZRegister& zd, 7022 const PRegisterM& pg, 7023 const ZRegister& zn) { 7024 // FCVTXNT <Zd>.S, <Pg>/M, <Zn>.D 7025 // 0110 0100 0000 1010 101. .... .... .... 7026 // opc<23:22> | opc2<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 7027 7028 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7029 7030 Emit(0x640aa000 | Rd(zd) | PgLow8(pg) | Rn(zn)); 7031} 7032 7033void Assembler::flogb(const ZRegister& zd, 7034 const PRegisterM& pg, 7035 const ZRegister& zn) { 7036 // FLOGB <Zd>.<T>, <Pg>/M, <Zn>.<T> 7037 // 0110 0101 0001 1..0 101. .... .... .... 7038 // opc<23:22> | opc2<18:17> | U<16> | Pg<12:10> | Zn<9:5> | Zd<4:0> | size<> 7039 7040 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7041 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 7042 VIXL_ASSERT(!zd.IsLaneSizeB()); 7043 7044 // Size field is encoded in bits <18:17> rather than <23:22>. 7045 Instr size = SVESize(zd) >> 5; 7046 Emit(0x6518a000 | size | Rd(zd) | PgLow8(pg) | Rn(zn)); 7047} 7048 7049void Assembler::fmaxnmp(const ZRegister& zd, 7050 const PRegisterM& pg, 7051 const ZRegister& zn, 7052 const ZRegister& zm) { 7053 // FMAXNMP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7054 // 0110 0100 ..01 0100 100. .... .... .... 7055 // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7056 7057 USE(zn); 7058 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7059 VIXL_ASSERT(zd.Is(zn)); 7060 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7061 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7062 7063 Emit(0x64148000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7064} 7065 7066void Assembler::fmaxp(const ZRegister& zd, 7067 const PRegisterM& pg, 7068 const ZRegister& zn, 7069 const ZRegister& zm) { 7070 // FMAXP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7071 // 0110 0100 ..01 0110 100. .... .... .... 7072 // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7073 7074 USE(zn); 7075 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7076 VIXL_ASSERT(zd.Is(zn)); 7077 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7078 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7079 7080 Emit(0x64168000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7081} 7082 7083void Assembler::fminnmp(const ZRegister& zd, 7084 const PRegisterM& pg, 7085 const ZRegister& zn, 7086 const ZRegister& zm) { 7087 // FMINNMP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7088 // 0110 0100 ..01 0101 100. .... .... .... 7089 // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7090 7091 USE(zn); 7092 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7093 VIXL_ASSERT(zd.Is(zn)); 7094 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7095 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7096 7097 Emit(0x64158000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7098} 7099 7100void Assembler::fminp(const ZRegister& zd, 7101 const PRegisterM& pg, 7102 const ZRegister& zn, 7103 const ZRegister& zm) { 7104 // FMINP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7105 // 0110 0100 ..01 0111 100. .... .... .... 7106 // size<23:22> | opc<18:16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7107 7108 USE(zn); 7109 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7110 VIXL_ASSERT(zd.Is(zn)); 7111 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7112 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7113 7114 Emit(0x64178000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7115} 7116 7117void Assembler::fmlalb(const ZRegister& zda, 7118 const ZRegister& zn, 7119 const ZRegister& zm) { 7120 // FMLALB <Zda>.S, <Zn>.H, <Zm>.H 7121 // 0110 0100 101. .... 1000 00.. .... .... 7122 // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> 7123 7124 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7125 VIXL_ASSERT(zda.IsLaneSizeS()); 7126 VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH()); 7127 7128 Emit(0x64a08000 | Rd(zda) | Rn(zn) | Rm(zm)); 7129} 7130 7131void Assembler::fmlalb(const ZRegister& zda, 7132 const ZRegister& zn, 7133 const ZRegister& zm, 7134 int index) { 7135 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7136 VIXL_ASSERT(zda.IsLaneSizeS()); 7137 VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH()); 7138 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7)); 7139 Instr zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) | 7140 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm); 7141 7142 Emit(0x64a04000 | Rd(zda) | Rn(zn) | zm_and_idx); 7143} 7144 7145void Assembler::fmlalt(const ZRegister& zda, 7146 const ZRegister& zn, 7147 const ZRegister& zm) { 7148 // FMLALT <Zda>.S, <Zn>.H, <Zm>.H 7149 // 0110 0100 101. .... 1000 01.. .... .... 7150 // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> 7151 7152 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7153 VIXL_ASSERT(zda.IsLaneSizeS()); 7154 VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH()); 7155 7156 Emit(0x64a08400 | Rd(zda) | Rn(zn) | Rm(zm)); 7157} 7158 7159void Assembler::fmlalt(const ZRegister& zda, 7160 const ZRegister& zn, 7161 const ZRegister& zm, 7162 int index) { 7163 // FMLALT <Zda>.S, <Zn>.H, <Zm>.H 7164 // 0110 0100 101. .... 1000 01.. .... .... 7165 // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> 7166 7167 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7168 VIXL_ASSERT(zda.IsLaneSizeS()); 7169 VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH()); 7170 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7)); 7171 Instr zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) | 7172 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm); 7173 7174 Emit(0x64a04400 | Rd(zda) | Rn(zn) | zm_and_idx); 7175} 7176 7177void Assembler::fmlslb(const ZRegister& zda, 7178 const ZRegister& zn, 7179 const ZRegister& zm) { 7180 // FMLSLB <Zda>.S, <Zn>.H, <Zm>.H 7181 // 0110 0100 101. .... 1010 00.. .... .... 7182 // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> 7183 7184 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7185 VIXL_ASSERT(zda.IsLaneSizeS()); 7186 VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH()); 7187 7188 Emit(0x64a0a000 | Rd(zda) | Rn(zn) | Rm(zm)); 7189} 7190 7191void Assembler::fmlslb(const ZRegister& zda, 7192 const ZRegister& zn, 7193 const ZRegister& zm, 7194 int index) { 7195 // FMLSLB <Zda>.S, <Zn>.H, <Zm>.H 7196 // 0110 0100 101. .... 1010 00.. .... .... 7197 // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> 7198 7199 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7200 VIXL_ASSERT(zda.IsLaneSizeS()); 7201 VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH()); 7202 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7)); 7203 Instr zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) | 7204 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm); 7205 7206 Emit(0x64a06000 | Rd(zda) | Rn(zn) | zm_and_idx); 7207} 7208 7209void Assembler::fmlslt(const ZRegister& zda, 7210 const ZRegister& zn, 7211 const ZRegister& zm) { 7212 // FMLSLT <Zda>.S, <Zn>.H, <Zm>.H 7213 // 0110 0100 101. .... 1010 01.. .... .... 7214 // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> 7215 7216 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7217 VIXL_ASSERT(zda.IsLaneSizeS()); 7218 VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH()); 7219 7220 Emit(0x64a0a400 | Rd(zda) | Rn(zn) | Rm(zm)); 7221} 7222 7223void Assembler::fmlslt(const ZRegister& zda, 7224 const ZRegister& zn, 7225 const ZRegister& zm, 7226 int index) { 7227 // FMLSLT <Zda>.S, <Zn>.H, <Zm>.H 7228 // 0110 0100 101. .... 1010 01.. .... .... 7229 // o2<22> | Zm<20:16> | op<13> | T<10> | Zn<9:5> | Zda<4:0> 7230 7231 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7232 VIXL_ASSERT(zda.IsLaneSizeS()); 7233 VIXL_ASSERT(zn.IsLaneSizeH() && zm.IsLaneSizeH()); 7234 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7)); 7235 Instr zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) | 7236 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm); 7237 7238 Emit(0x64a06400 | Rd(zda) | Rn(zn) | zm_and_idx); 7239} 7240 7241void Assembler::histcnt(const ZRegister& zd, 7242 const PRegisterZ& pg, 7243 const ZRegister& zn, 7244 const ZRegister& zm) { 7245 // HISTCNT <Zd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 7246 // 0100 0101 ..1. .... 110. .... .... .... 7247 // size<23:22> | Zm<20:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 7248 7249 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7250 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7251 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeD()); 7252 7253 Emit(0x4520c000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 7254} 7255 7256void Assembler::histseg(const ZRegister& zd, 7257 const ZRegister& zn, 7258 const ZRegister& zm) { 7259 // HISTSEG <Zd>.B, <Zn>.B, <Zm>.B 7260 // 0100 0101 ..1. .... 1010 00.. .... .... 7261 // size<23:22> | Zm<20:16> | Zn<9:5> | Zd<4:0> 7262 7263 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7264 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7265 VIXL_ASSERT(zd.IsLaneSizeB()); 7266 7267 Emit(0x4520a000 | Rd(zd) | Rn(zn) | Rm(zm)); 7268} 7269 7270void Assembler::match(const PRegisterWithLaneSize& pd, 7271 const PRegisterZ& pg, 7272 const ZRegister& zn, 7273 const ZRegister& zm) { 7274 // MATCH <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 7275 // 0100 0101 ..1. .... 100. .... ...0 .... 7276 // size<23:22> | Zm<20:16> | Pg<12:10> | Zn<9:5> | op<4> | Pd<3:0> 7277 7278 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7279 VIXL_ASSERT(AreSameLaneSize(pd, zn, zm)); 7280 VIXL_ASSERT(zm.IsLaneSizeB() || zm.IsLaneSizeH()); 7281 7282 Emit(0x45208000 | SVESize(zm) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 7283} 7284 7285void Assembler::mla(const ZRegister& zda, 7286 const ZRegister& zn, 7287 const ZRegister& zm, 7288 int index) { 7289 // MLA <Zda>.D, <Zn>.D, <Zm>.D[<imm>] 7290 // 0100 0100 111. .... 0000 10.. .... .... 7291 // size<23:22> | opc<20:16> | S<10> | Zn<9:5> | Zda<4:0> 7292 7293 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7294 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 7295 7296 Instr synthesised_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(), 7297 zm, 7298 index, 7299 0x44200800, 7300 0x44a00800, 7301 0x44e00800); 7302 7303 Emit(synthesised_op | Rd(zda) | Rn(zn)); 7304} 7305 7306void Assembler::mls(const ZRegister& zda, 7307 const ZRegister& zn, 7308 const ZRegister& zm, 7309 int index) { 7310 // MLS <Zda>.D, <Zn>.D, <Zm>.D[<imm>] 7311 // 0100 0100 111. .... 0000 11.. .... .... 7312 // size<23:22> | opc<20:16> | S<10> | Zn<9:5> | Zda<4:0> 7313 7314 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7315 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 7316 7317 Instr synthesised_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(), 7318 zm, 7319 index, 7320 0x44200c00, 7321 0x44a00c00, 7322 0x44e00c00); 7323 7324 Emit(synthesised_op | Rd(zda) | Rn(zn)); 7325} 7326 7327void Assembler::mul(const ZRegister& zd, 7328 const ZRegister& zn, 7329 const ZRegister& zm, 7330 int index) { 7331 // MUL <Zd>.D, <Zn>.D, <Zm>.D[<imm>] 7332 // 0100 0100 111. .... 1111 10.. .... .... 7333 // size<23:22> | opc<20:16> | Zn<9:5> | Zd<4:0> 7334 7335 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7336 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7337 7338 Instr synthesised_op = SVEMulIndexHelper(zd.GetLaneSizeInBytesLog2(), 7339 zm, 7340 index, 7341 0x4420f800, 7342 0x44a0f800, 7343 0x44e0f800); 7344 7345 Emit(synthesised_op | Rd(zd) | Rn(zn)); 7346} 7347 7348void Assembler::mul(const ZRegister& zd, 7349 const ZRegister& zn, 7350 const ZRegister& zm) { 7351 // MUL <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 7352 // 0000 0100 ..1. .... 0110 00.. .... .... 7353 // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> 7354 7355 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7356 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7357 7358 Emit(0x04206000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7359} 7360 7361void Assembler::nbsl(const ZRegister& zd, 7362 const ZRegister& zn, 7363 const ZRegister& zm, 7364 const ZRegister& zk) { 7365 // NBSL <Zdn>.D, <Zdn>.D, <Zm>.D, <Zk>.D 7366 // 0000 0100 111. .... 0011 11.. .... .... 7367 // opc<23:22> | Zm<20:16> | o2<10> | Zk<9:5> | Zdn<4:0> 7368 7369 USE(zn); 7370 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7371 VIXL_ASSERT(zd.Is(zn)); 7372 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm, zk)); 7373 VIXL_ASSERT(zd.IsLaneSizeD()); 7374 7375 Emit(0x04e03c00 | Rd(zd) | Rm(zm) | Rn(zk)); 7376} 7377 7378void Assembler::nmatch(const PRegisterWithLaneSize& pd, 7379 const PRegisterZ& pg, 7380 const ZRegister& zn, 7381 const ZRegister& zm) { 7382 // NMATCH <Pd>.<T>, <Pg>/Z, <Zn>.<T>, <Zm>.<T> 7383 // 0100 0101 ..1. .... 100. .... ...1 .... 7384 // size<23:22> | Zm<20:16> | Pg<12:10> | Zn<9:5> | op<4> | Pd<3:0> 7385 7386 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7387 VIXL_ASSERT(AreSameLaneSize(pd, zn, zm)); 7388 VIXL_ASSERT(zm.IsLaneSizeB() || zm.IsLaneSizeH()); 7389 7390 Emit(0x45208010 | SVESize(zm) | Pd(pd) | PgLow8(pg) | Rn(zn) | Rm(zm)); 7391} 7392 7393void Assembler::pmul(const ZRegister& zd, 7394 const ZRegister& zn, 7395 const ZRegister& zm) { 7396 // PMUL <Zd>.B, <Zn>.B, <Zm>.B 7397 // 0000 0100 001. .... 0110 01.. .... .... 7398 // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> 7399 7400 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7401 7402 Emit(0x04206400 | Rd(zd) | Rn(zn) | Rm(zm)); 7403} 7404 7405void Assembler::pmullb(const ZRegister& zd, 7406 const ZRegister& zn, 7407 const ZRegister& zm) { 7408 // PMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7409 // 0100 0101 ..0. .... 0110 10.. .... .... 7410 // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 7411 7412 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7413 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7414 VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeS()); 7415 VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); 7416 // SVEPmull128 is not supported 7417 VIXL_ASSERT(!zd.IsLaneSizeQ()); 7418 7419 Emit(0x45006800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7420} 7421 7422void Assembler::pmullt(const ZRegister& zd, 7423 const ZRegister& zn, 7424 const ZRegister& zm) { 7425 // PMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7426 // 0100 0101 ..0. .... 0110 11.. .... .... 7427 // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 7428 7429 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7430 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7431 VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeS()); 7432 VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); 7433 // SVEPmull128 is not supported 7434 VIXL_ASSERT(!zd.IsLaneSizeQ()); 7435 7436 Emit(0x45006c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7437} 7438 7439void Assembler::raddhnb(const ZRegister& zd, 7440 const ZRegister& zn, 7441 const ZRegister& zm) { 7442 // RADDHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7443 // 0100 0101 ..1. .... 0110 10.. .... .... 7444 // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> 7445 7446 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7447 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7448 VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); 7449 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 7450 7451 Emit(0x45206800 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); 7452} 7453 7454void Assembler::raddhnt(const ZRegister& zd, 7455 const ZRegister& zn, 7456 const ZRegister& zm) { 7457 // RADDHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7458 // 0100 0101 ..1. .... 0110 11.. .... .... 7459 // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> 7460 7461 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7462 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7463 VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); 7464 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 7465 7466 Emit(0x45206c00 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); 7467} 7468 7469#define VIXL_SVE_SHR_LIST(V) \ 7470 V(rshrnb, 0x45201800) \ 7471 V(rshrnt, 0x45201c00) \ 7472 V(shrnb, 0x45201000) \ 7473 V(shrnt, 0x45201400) \ 7474 V(sqrshrnb, 0x45202800) \ 7475 V(sqrshrnt, 0x45202c00) \ 7476 V(sqrshrunb, 0x45200800) \ 7477 V(sqrshrunt, 0x45200c00) \ 7478 V(sqshrnb, 0x45202000) \ 7479 V(sqshrnt, 0x45202400) \ 7480 V(sqshrunb, 0x45200000) \ 7481 V(sqshrunt, 0x45200400) \ 7482 V(uqrshrnb, 0x45203800) \ 7483 V(uqrshrnt, 0x45203c00) \ 7484 V(uqshrnb, 0x45203000) \ 7485 V(uqshrnt, 0x45203400) 7486 7487#define VIXL_DEFINE_ASM_FUNC(MNE, X) \ 7488 void Assembler::MNE(const ZRegister& zd, const ZRegister& zn, int shift) { \ 7489 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); \ 7490 VIXL_ASSERT(!zd.IsLaneSizeD() && !zd.IsLaneSizeQ()); \ 7491 VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); \ 7492 Instr encoded_imm = \ 7493 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); \ 7494 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, X); \ 7495 } 7496VIXL_SVE_SHR_LIST(VIXL_DEFINE_ASM_FUNC) 7497#undef VIXL_DEFINE_ASM_FUNC 7498 7499void Assembler::rsubhnb(const ZRegister& zd, 7500 const ZRegister& zn, 7501 const ZRegister& zm) { 7502 // RSUBHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7503 // 0100 0101 ..1. .... 0111 10.. .... .... 7504 // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> 7505 7506 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7507 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7508 VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); 7509 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 7510 7511 Emit(0x45207800 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); 7512} 7513 7514void Assembler::rsubhnt(const ZRegister& zd, 7515 const ZRegister& zn, 7516 const ZRegister& zm) { 7517 // RSUBHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7518 // 0100 0101 ..1. .... 0111 11.. .... .... 7519 // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> 7520 7521 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7522 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7523 VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); 7524 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 7525 7526 Emit(0x45207c00 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); 7527} 7528 7529void Assembler::saba(const ZRegister& zda, 7530 const ZRegister& zn, 7531 const ZRegister& zm) { 7532 // SABA <Zda>.<T>, <Zn>.<T>, <Zm>.<T> 7533 // 0100 0101 ..0. .... 1111 10.. .... .... 7534 // size<23:22> | Zm<20:16> | U<10> | Zn<9:5> | Zda<4:0> 7535 7536 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7537 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 7538 7539 Emit(0x4500f800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 7540} 7541 7542void Assembler::sabalb(const ZRegister& zda, 7543 const ZRegister& zn, 7544 const ZRegister& zm) { 7545 // SABALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7546 // 0100 0101 ..0. .... 1100 00.. .... .... 7547 // size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 7548 7549 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7550 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7551 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7552 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 7553 7554 Emit(0x4500c000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 7555} 7556 7557void Assembler::sabalt(const ZRegister& zda, 7558 const ZRegister& zn, 7559 const ZRegister& zm) { 7560 // SABALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7561 // 0100 0101 ..0. .... 1100 01.. .... .... 7562 // size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 7563 7564 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7565 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7566 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7567 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 7568 7569 Emit(0x4500c400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 7570} 7571 7572void Assembler::sabdlb(const ZRegister& zd, 7573 const ZRegister& zn, 7574 const ZRegister& zm) { 7575 // SABDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7576 // 0100 0101 ..0. .... 0011 00.. .... .... 7577 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 7578 // Zd<4:0> 7579 7580 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7581 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7582 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7583 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7584 7585 Emit(0x45003000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7586} 7587 7588void Assembler::sabdlt(const ZRegister& zd, 7589 const ZRegister& zn, 7590 const ZRegister& zm) { 7591 // SABDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7592 // 0100 0101 ..0. .... 0011 01.. .... .... 7593 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 7594 // Zd<4:0> 7595 7596 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7597 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7598 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7599 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7600 7601 Emit(0x45003400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7602} 7603 7604void Assembler::sadalp(const ZRegister& zda, 7605 const PRegisterM& pg, 7606 const ZRegister& zn) { 7607 // SADALP <Zda>.<T>, <Pg>/M, <Zn>.<Tb> 7608 // 0100 0100 ..00 0100 101. .... .... .... 7609 // size<23:22> | U<16> | Pg<12:10> | Zn<9:5> | Zda<4:0> 7610 7611 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7612 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7613 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 7614 7615 Emit(0x4404a000 | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn)); 7616} 7617 7618void Assembler::saddlb(const ZRegister& zd, 7619 const ZRegister& zn, 7620 const ZRegister& zm) { 7621 // SADDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7622 // 0100 0101 ..0. .... 0000 00.. .... .... 7623 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 7624 // Zd<4:0> 7625 7626 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7627 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7628 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7629 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7630 7631 Emit(0x45000000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7632} 7633 7634void Assembler::saddlbt(const ZRegister& zd, 7635 const ZRegister& zn, 7636 const ZRegister& zm) { 7637 // SADDLBT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7638 // 0100 0101 ..0. .... 1000 00.. .... .... 7639 // size<23:22> | Zm<20:16> | S<11> | tb<10> | Zn<9:5> | Zd<4:0> 7640 7641 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7642 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7643 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7644 VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeH() || zd.IsLaneSizeS()); 7645 7646 Emit(0x45008000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7647} 7648 7649void Assembler::saddlt(const ZRegister& zd, 7650 const ZRegister& zn, 7651 const ZRegister& zm) { 7652 // SADDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7653 // 0100 0101 ..0. .... 0000 01.. .... .... 7654 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 7655 // Zd<4:0> 7656 7657 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7658 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7659 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7660 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7661 7662 Emit(0x45000400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7663} 7664 7665void Assembler::saddwb(const ZRegister& zd, 7666 const ZRegister& zn, 7667 const ZRegister& zm) { 7668 // SADDWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> 7669 // 0100 0101 ..0. .... 0100 00.. .... .... 7670 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 7671 7672 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7673 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 7674 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); 7675 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7676 7677 Emit(0x45004000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7678} 7679 7680void Assembler::saddwt(const ZRegister& zd, 7681 const ZRegister& zn, 7682 const ZRegister& zm) { 7683 // SADDWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> 7684 // 0100 0101 ..0. .... 0100 01.. .... .... 7685 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 7686 7687 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7688 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 7689 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); 7690 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 7691 7692 Emit(0x45004400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7693} 7694 7695void Assembler::sbclb(const ZRegister& zda, 7696 const ZRegister& zn, 7697 const ZRegister& zm) { 7698 // SBCLB <Zda>.<T>, <Zn>.<T>, <Zm>.<T> 7699 // 0100 0101 1.0. .... 1101 00.. .... .... 7700 // size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0> 7701 7702 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7703 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 7704 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 7705 7706 Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0; 7707 Emit(0x4580d000 | sz | Rd(zda) | Rn(zn) | Rm(zm)); 7708} 7709 7710void Assembler::sbclt(const ZRegister& zda, 7711 const ZRegister& zn, 7712 const ZRegister& zm) { 7713 // SBCLT <Zda>.<T>, <Zn>.<T>, <Zm>.<T> 7714 // 0100 0101 1.0. .... 1101 01.. .... .... 7715 // size<23:22> | Zm<20:16> | T<10> | Zn<9:5> | Zda<4:0> 7716 7717 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7718 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 7719 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 7720 7721 Instr sz = zda.IsLaneSizeD() ? (1 << 22) : 0; 7722 Emit(0x4580d400 | sz | Rd(zda) | Rn(zn) | Rm(zm)); 7723} 7724 7725void Assembler::shadd(const ZRegister& zd, 7726 const PRegisterM& pg, 7727 const ZRegister& zn, 7728 const ZRegister& zm) { 7729 // SHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7730 // 0100 0100 ..01 0000 100. .... .... .... 7731 // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7732 7733 USE(zn); 7734 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7735 VIXL_ASSERT(zd.Is(zn)); 7736 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7737 7738 Emit(0x44108000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7739} 7740 7741void Assembler::shsub(const ZRegister& zd, 7742 const PRegisterM& pg, 7743 const ZRegister& zn, 7744 const ZRegister& zm) { 7745 // SHSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7746 // 0100 0100 ..01 0010 100. .... .... .... 7747 // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7748 7749 USE(zn); 7750 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7751 VIXL_ASSERT(zd.Is(zn)); 7752 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7753 7754 Emit(0x44128000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7755} 7756 7757void Assembler::shsubr(const ZRegister& zd, 7758 const PRegisterM& pg, 7759 const ZRegister& zn, 7760 const ZRegister& zm) { 7761 // SHSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7762 // 0100 0100 ..01 0110 100. .... .... .... 7763 // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7764 7765 USE(zn); 7766 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7767 VIXL_ASSERT(zd.Is(zn)); 7768 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7769 7770 Emit(0x44168000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7771} 7772 7773void Assembler::sli(const ZRegister& zd, const ZRegister& zn, int shift) { 7774 // SLI <Zd>.<T>, <Zn>.<T>, #<const> 7775 // 0100 0101 ..0. .... 1111 01.. .... .... 7776 // tszh<23:22> | tszl<20:19> | imm3<18:16> | op<10> | Zn<9:5> | Zd<4:0> 7777 7778 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7779 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 7780 Instr encoded_imm = 7781 EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); 7782 7783 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500f400); 7784} 7785 7786void Assembler::smaxp(const ZRegister& zd, 7787 const PRegisterM& pg, 7788 const ZRegister& zn, 7789 const ZRegister& zm) { 7790 // SMAXP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7791 // 0100 0100 ..01 0100 101. .... .... .... 7792 // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7793 7794 USE(zn); 7795 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7796 VIXL_ASSERT(zd.Is(zn)); 7797 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7798 7799 Emit(0x4414a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7800} 7801 7802void Assembler::sminp(const ZRegister& zd, 7803 const PRegisterM& pg, 7804 const ZRegister& zn, 7805 const ZRegister& zm) { 7806 // SMINP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7807 // 0100 0100 ..01 0110 101. .... .... .... 7808 // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7809 7810 USE(zn); 7811 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7812 VIXL_ASSERT(zd.Is(zn)); 7813 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7814 7815 Emit(0x4416a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7816} 7817 7818#define VIXL_SVE_MULL_INDEX_LIST(V) \ 7819 V(smullb, 0x44a0c000) \ 7820 V(smullt, 0x44a0c400) \ 7821 V(umullb, 0x44a0d000) \ 7822 V(umullt, 0x44a0d400) \ 7823 V(smlalb, 0x44a08000) \ 7824 V(smlalt, 0x44a08400) \ 7825 V(smlslb, 0x44a0a000) \ 7826 V(smlslt, 0x44a0a400) \ 7827 V(umlalb, 0x44a09000) \ 7828 V(umlalt, 0x44a09400) \ 7829 V(umlslb, 0x44a0b000) \ 7830 V(umlslt, 0x44a0b400) \ 7831 V(sqdmullb, 0x44a0e000) \ 7832 V(sqdmullt, 0x44a0e400) 7833 7834#define VIXL_DEFINE_ASM_FUNC(MNE, OP) \ 7835 void Assembler::MNE(const ZRegister& zda, \ 7836 const ZRegister& zn, \ 7837 const ZRegister& zm, \ 7838 int index) { \ 7839 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); \ 7840 VIXL_ASSERT(AreSameLaneSize(zn, zm)); \ 7841 VIXL_ASSERT(zda.IsLaneSizeD() || zda.IsLaneSizeS()); \ 7842 VIXL_ASSERT(zda.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); \ 7843 Instr zm_with_index = SVEMulLongIndexHelper(zm, index); \ 7844 Emit(OP | SVESize(zda) | Rd(zda) | Rn(zn) | zm_with_index); \ 7845 } 7846VIXL_SVE_MULL_INDEX_LIST(VIXL_DEFINE_ASM_FUNC) 7847#undef VIXL_DEFINE_ASM_FuNC 7848 7849void Assembler::smlalb(const ZRegister& zda, 7850 const ZRegister& zn, 7851 const ZRegister& zm) { 7852 // SMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7853 // 0100 0100 ..0. .... 0100 00.. .... .... 7854 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 7855 7856 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7857 VIXL_ASSERT(!zda.IsLaneSizeB()); 7858 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7859 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7860 7861 Emit(0x44004000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 7862} 7863 7864void Assembler::smlalt(const ZRegister& zda, 7865 const ZRegister& zn, 7866 const ZRegister& zm) { 7867 // SMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7868 // 0100 0100 ..0. .... 0100 01.. .... .... 7869 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 7870 7871 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7872 VIXL_ASSERT(!zda.IsLaneSizeB()); 7873 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7874 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7875 7876 Emit(0x44004400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 7877} 7878 7879void Assembler::smlslb(const ZRegister& zda, 7880 const ZRegister& zn, 7881 const ZRegister& zm) { 7882 // SMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7883 // 0100 0100 ..0. .... 0101 00.. .... .... 7884 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 7885 7886 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7887 VIXL_ASSERT(!zda.IsLaneSizeB()); 7888 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7889 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7890 7891 Emit(0x44005000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 7892} 7893 7894void Assembler::smlslt(const ZRegister& zda, 7895 const ZRegister& zn, 7896 const ZRegister& zm) { 7897 // SMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7898 // 0100 0100 ..0. .... 0101 01.. .... .... 7899 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 7900 7901 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7902 VIXL_ASSERT(!zda.IsLaneSizeB()); 7903 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7904 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 7905 7906 Emit(0x44005400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 7907} 7908 7909void Assembler::smulh(const ZRegister& zd, 7910 const ZRegister& zn, 7911 const ZRegister& zm) { 7912 // SMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 7913 // 0000 0100 ..1. .... 0110 10.. .... .... 7914 // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> 7915 7916 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7917 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7918 7919 Emit(0x04206800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7920} 7921 7922void Assembler::smullb(const ZRegister& zd, 7923 const ZRegister& zn, 7924 const ZRegister& zm) { 7925 // SMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7926 // 0100 0101 ..0. .... 0111 00.. .... .... 7927 // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 7928 7929 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7930 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7931 VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); 7932 VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); 7933 7934 Emit(0x45007000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7935} 7936 7937void Assembler::smullt(const ZRegister& zd, 7938 const ZRegister& zn, 7939 const ZRegister& zm) { 7940 // SMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 7941 // 0100 0101 ..0. .... 0111 01.. .... .... 7942 // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 7943 7944 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7945 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 7946 VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); 7947 VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); 7948 7949 Emit(0x45007400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 7950} 7951 7952void Assembler::sqabs(const ZRegister& zd, 7953 const PRegisterM& pg, 7954 const ZRegister& zn) { 7955 // SQABS <Zd>.<T>, <Pg>/M, <Zn>.<T> 7956 // 0100 0100 ..00 1000 101. .... .... .... 7957 // size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 7958 7959 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7960 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 7961 7962 Emit(0x4408a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 7963} 7964 7965void Assembler::sqadd(const ZRegister& zd, 7966 const PRegisterM& pg, 7967 const ZRegister& zn, 7968 const ZRegister& zm) { 7969 // SQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 7970 // 0100 0100 ..01 1000 100. .... .... .... 7971 // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 7972 7973 USE(zn); 7974 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7975 VIXL_ASSERT(zd.Is(zn)); 7976 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7977 7978 Emit(0x44188000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 7979} 7980 7981void Assembler::sqcadd(const ZRegister& zd, 7982 const ZRegister& zn, 7983 const ZRegister& zm, 7984 int rot) { 7985 // SQCADD <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, <const> 7986 // 0100 0101 ..00 0001 1101 1... .... .... 7987 // size<23:22> | op<16> | rot<10> | Zm<9:5> | Zdn<4:0> 7988 7989 USE(zn); 7990 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 7991 VIXL_ASSERT(zd.Is(zn)); 7992 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 7993 VIXL_ASSERT((rot == 90) || (rot == 270)); 7994 7995 Instr rotate_bit = (rot == 90) ? 0 : (1 << 10); 7996 Emit(0x4501d800 | rotate_bit | SVESize(zd) | Rd(zd) | Rn(zm)); 7997} 7998 7999// This prototype maps to 2 instruction encodings: 8000// sqdmlalb_z_zzzi_d 8001// sqdmlalb_z_zzzi_s 8002void Assembler::sqdmlalb(const ZRegister& zda, 8003 const ZRegister& zn, 8004 const ZRegister& zm, 8005 int index) { 8006 // SQDMLALB <Zda>.D, <Zn>.S, <Zm>.S[<imm>] 8007 // 0100 0100 111. .... 0010 .0.. .... .... 8008 // size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0> 8009 8010 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8011 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8012 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 8013 VIXL_ASSERT(index >= 0); 8014 8015 Instr zm_and_idx = 0; 8016 if (zm.IsLaneSizeH()) { 8017 // Zm<18:16> | i3h<20:19> | i3l<11> 8018 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7)); 8019 zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) | 8020 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm); 8021 } else { 8022 // Zm<19:16> | i2h<20> | i2l<11> 8023 VIXL_ASSERT(zm.IsLaneSizeS()); 8024 VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 3)); 8025 zm_and_idx = (ExtractBit(index, 1) << 20) | (ExtractBit(index, 0) << 11) | 8026 Rx<19, 16>(zm); 8027 } 8028 8029 Emit(0x44202000 | zm_and_idx | SVESize(zda) | Rd(zda) | Rn(zn)); 8030} 8031 8032void Assembler::sqdmlalb(const ZRegister& zda, 8033 const ZRegister& zn, 8034 const ZRegister& zm) { 8035 // SQDMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8036 // 0100 0100 ..0. .... 0110 00.. .... .... 8037 // size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0> 8038 8039 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8040 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8041 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8042 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 8043 8044 Emit(0x44006000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 8045} 8046 8047void Assembler::sqdmlalbt(const ZRegister& zda, 8048 const ZRegister& zn, 8049 const ZRegister& zm) { 8050 // SQDMLALBT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8051 // 0100 0100 ..0. .... 0000 10.. .... .... 8052 // size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0> 8053 8054 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8055 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8056 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8057 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 8058 8059 Emit(0x44000800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 8060} 8061 8062// This prototype maps to 2 instruction encodings: 8063// sqdmlalt_z_zzzi_d 8064// sqdmlalt_z_zzzi_s 8065void Assembler::sqdmlalt(const ZRegister& zda, 8066 const ZRegister& zn, 8067 const ZRegister& zm, 8068 int index) { 8069 // SQDMLALT <Zda>.D, <Zn>.S, <Zm>.S[<imm>] 8070 // 0100 0100 111. .... 0010 .1.. .... .... 8071 // size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0> 8072 8073 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8074 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8075 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 8076 VIXL_ASSERT(index >= 0); 8077 8078 Instr zm_and_idx = 0; 8079 if (zm.IsLaneSizeH()) { 8080 // Zm<18:16> | i3h<20:19> | i3l<11> 8081 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7)); 8082 zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) | 8083 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm); 8084 } else { 8085 // Zm<19:16> | i2h<20> | i2l<11> 8086 VIXL_ASSERT(zm.IsLaneSizeS()); 8087 VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 3)); 8088 zm_and_idx = (ExtractBit(index, 1) << 20) | (ExtractBit(index, 0) << 11) | 8089 Rx<19, 16>(zm); 8090 } 8091 8092 Emit(0x44202400 | zm_and_idx | SVESize(zda) | Rd(zda) | Rn(zn)); 8093} 8094 8095void Assembler::sqdmlalt(const ZRegister& zda, 8096 const ZRegister& zn, 8097 const ZRegister& zm) { 8098 // SQDMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8099 // 0100 0100 ..0. .... 0110 01.. .... .... 8100 // size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0> 8101 8102 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8103 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8104 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8105 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 8106 8107 Emit(0x44006400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 8108} 8109 8110// This prototype maps to 2 instruction encodings: 8111// sqdmlslb_z_zzzi_d 8112// sqdmlslb_z_zzzi_s 8113void Assembler::sqdmlslb(const ZRegister& zda, 8114 const ZRegister& zn, 8115 const ZRegister& zm, 8116 int index) { 8117 // SQDMLSLB <Zda>.D, <Zn>.S, <Zm>.S[<imm>] 8118 // 0100 0100 111. .... 0011 .0.. .... .... 8119 // size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0> 8120 8121 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8122 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8123 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 8124 VIXL_ASSERT(index >= 0); 8125 8126 Instr zm_and_idx = 0; 8127 if (zm.IsLaneSizeH()) { 8128 // Zm<18:16> | i3h<20:19> | i3l<11> 8129 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7)); 8130 zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) | 8131 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm); 8132 } else { 8133 // Zm<19:16> | i2h<20> | i2l<11> 8134 VIXL_ASSERT(zm.IsLaneSizeS()); 8135 VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 3)); 8136 zm_and_idx = (ExtractBit(index, 1) << 20) | (ExtractBit(index, 0) << 11) | 8137 Rx<19, 16>(zm); 8138 } 8139 8140 Emit(0x44203000 | zm_and_idx | SVESize(zda) | Rd(zda) | Rn(zn)); 8141} 8142 8143void Assembler::sqdmlslb(const ZRegister& zda, 8144 const ZRegister& zn, 8145 const ZRegister& zm) { 8146 // SQDMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8147 // 0100 0100 ..0. .... 0110 10.. .... .... 8148 // size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0> 8149 8150 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8151 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8152 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8153 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 8154 8155 Emit(0x44006800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 8156} 8157 8158void Assembler::sqdmlslbt(const ZRegister& zda, 8159 const ZRegister& zn, 8160 const ZRegister& zm) { 8161 // SQDMLSLBT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8162 // 0100 0100 ..0. .... 0000 11.. .... .... 8163 // size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0> 8164 8165 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8166 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8167 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8168 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 8169 8170 Emit(0x44000c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 8171} 8172 8173// This prototype maps to 2 instruction encodings: 8174// sqdmlslt_z_zzzi_d 8175// sqdmlslt_z_zzzi_s 8176void Assembler::sqdmlslt(const ZRegister& zda, 8177 const ZRegister& zn, 8178 const ZRegister& zm, 8179 int index) { 8180 // SQDMLSLT <Zda>.D, <Zn>.S, <Zm>.S[<imm>] 8181 // 0100 0100 111. .... 0011 .1.. .... .... 8182 // size<23:22> | opc<20:16> | S<12> | il<11> | T<10> | Zn<9:5> | Zda<4:0> 8183 8184 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8185 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8186 VIXL_ASSERT(zda.IsLaneSizeS() || zda.IsLaneSizeD()); 8187 VIXL_ASSERT(index >= 0); 8188 8189 Instr zm_and_idx = 0; 8190 if (zm.IsLaneSizeH()) { 8191 // Zm<18:16> | i3h<20:19> | i3l<11> 8192 VIXL_ASSERT((zm.GetCode() <= 7) && (index <= 7)); 8193 zm_and_idx = (ExtractUnsignedBitfield32(2, 1, index) << 19) | 8194 (ExtractBit(index, 0) << 11) | Rx<18, 16>(zm); 8195 } else { 8196 // Zm<19:16> | i2h<20> | i2l<11> 8197 VIXL_ASSERT(zm.IsLaneSizeS()); 8198 VIXL_ASSERT((zm.GetCode() <= 15) && (index <= 3)); 8199 zm_and_idx = (ExtractBit(index, 1) << 20) | (ExtractBit(index, 0) << 11) | 8200 Rx<19, 16>(zm); 8201 } 8202 8203 Emit(0x44203400 | zm_and_idx | SVESize(zda) | Rd(zda) | Rn(zn)); 8204} 8205 8206void Assembler::sqdmlslt(const ZRegister& zda, 8207 const ZRegister& zn, 8208 const ZRegister& zm) { 8209 // SQDMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8210 // 0100 0100 ..0. .... 0110 11.. .... .... 8211 // size<23:22> | Zm<20:16> | S<11> | T<10> | Zn<9:5> | Zda<4:0> 8212 8213 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8214 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8215 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8216 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 8217 8218 Emit(0x44006c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 8219} 8220 8221void Assembler::sqdmulh(const ZRegister& zd, 8222 const ZRegister& zn, 8223 const ZRegister& zm, 8224 int index) { 8225 // SQDMULH <Zd>.D, <Zn>.D, <Zm>.D[<imm>] 8226 // 0100 0100 111. .... 1111 00.. .... .... 8227 // size<23:22> | opc<20:16> | R<10> | Zn<9:5> | Zd<4:0> 8228 8229 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8230 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8231 8232 Instr synthesised_op = SVEMulIndexHelper(zd.GetLaneSizeInBytesLog2(), 8233 zm, 8234 index, 8235 0x4420f000, 8236 0x44a0f000, 8237 0x44e0f000); 8238 8239 Emit(synthesised_op | Rd(zd) | Rn(zn)); 8240} 8241 8242void Assembler::sqdmulh(const ZRegister& zd, 8243 const ZRegister& zn, 8244 const ZRegister& zm) { 8245 // SQDMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 8246 // 0000 0100 ..1. .... 0111 00.. .... .... 8247 // size<23:22> | Zm<20:16> | R<10> | Zn<9:5> | Zd<4:0> 8248 8249 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8250 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8251 8252 Emit(0x04207000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8253} 8254 8255void Assembler::sqdmullb(const ZRegister& zd, 8256 const ZRegister& zn, 8257 const ZRegister& zm) { 8258 // SQDMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8259 // 0100 0101 ..0. .... 0110 00.. .... .... 8260 // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 8261 8262 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8263 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8264 VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); 8265 VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); 8266 8267 Emit(0x45006000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8268} 8269 8270void Assembler::sqdmullt(const ZRegister& zd, 8271 const ZRegister& zn, 8272 const ZRegister& zm) { 8273 // SQDMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8274 // 0100 0101 ..0. .... 0110 01.. .... .... 8275 // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 8276 8277 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8278 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8279 VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); 8280 VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); 8281 8282 Emit(0x45006400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8283} 8284 8285void Assembler::sqneg(const ZRegister& zd, 8286 const PRegisterM& pg, 8287 const ZRegister& zn) { 8288 // SQNEG <Zd>.<T>, <Pg>/M, <Zn>.<T> 8289 // 0100 0100 ..00 1001 101. .... .... .... 8290 // size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 8291 8292 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8293 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 8294 8295 Emit(0x4409a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 8296} 8297 8298void Assembler::sqrdcmlah(const ZRegister& zda, 8299 const ZRegister& zn, 8300 const ZRegister& zm, 8301 int index, 8302 int rot) { 8303 // SQRDCMLAH <Zda>.H, <Zn>.H, <Zm>.H[<imm>], <const> 8304 // 0100 0100 101. .... 0111 .... .... .... 8305 // size<23:22> | opc<20:16> | rot<11:10> | Zn<9:5> | Zda<4:0> 8306 8307 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8308 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 8309 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); 8310 8311 Instr rotate_bit = (rot / 90) << 10; 8312 Emit(0x44a07000 | SVEMulComplexIndexHelper(zm, index) | rotate_bit | Rd(zda) | 8313 Rn(zn)); 8314} 8315 8316void Assembler::sqrdcmlah(const ZRegister& zda, 8317 const ZRegister& zn, 8318 const ZRegister& zm, 8319 int rot) { 8320 // SQRDCMLAH <Zda>.<T>, <Zn>.<T>, <Zm>.<T>, <const> 8321 // 0100 0100 ..0. .... 0011 .... .... .... 8322 // size<23:22> | Zm<20:16> | op<12> | rot<11:10> | Zn<9:5> | Zda<4:0> 8323 8324 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8325 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 8326 VIXL_ASSERT((rot == 0) || (rot == 90) || (rot == 180) || (rot == 270)); 8327 8328 Instr rotate_bit = (rot / 90) << 10; 8329 Emit(0x44003000 | rotate_bit | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 8330} 8331 8332// This prototype maps to 3 instruction encodings: 8333// sqrdmlah_z_zzzi_d 8334// sqrdmlah_z_zzzi_h 8335// sqrdmlah_z_zzzi_s 8336void Assembler::sqrdmlah(const ZRegister& zda, 8337 const ZRegister& zn, 8338 const ZRegister& zm, 8339 int index) { 8340 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8341 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 8342 8343 Instr op_h = 0x44201000; 8344 Instr op_s = op_h | (1 << 23); 8345 Instr op_d = op_h | (3 << 22); 8346 // The encoding of opcode, index, Zm, and size are synthesized in this 8347 // variable. 8348 Instr synthesized_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(), 8349 zm, 8350 index, 8351 op_h, 8352 op_s, 8353 op_d); 8354 8355 Emit(synthesized_op | Rd(zda) | Rn(zn)); 8356} 8357 8358void Assembler::sqrdmlah(const ZRegister& zda, 8359 const ZRegister& zn, 8360 const ZRegister& zm) { 8361 // SQRDMLAH <Zda>.<T>, <Zn>.<T>, <Zm>.<T> 8362 // 0100 0100 ..0. .... 0111 00.. .... .... 8363 // size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0> 8364 8365 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8366 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 8367 8368 Emit(0x44007000 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 8369} 8370 8371// This prototype maps to 3 instruction encodings: 8372// sqrdmlsh_z_zzzi_d 8373// sqrdmlsh_z_zzzi_h 8374// sqrdmlsh_z_zzzi_s 8375void Assembler::sqrdmlsh(const ZRegister& zda, 8376 const ZRegister& zn, 8377 const ZRegister& zm, 8378 int index) { 8379 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8380 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 8381 8382 Instr op_h = 0x44201400; 8383 Instr op_s = op_h | (1 << 23); 8384 Instr op_d = op_h | (3 << 22); 8385 // The encoding of opcode, index, Zm, and size are synthesized in this 8386 // variable. 8387 Instr synthesized_op = SVEMulIndexHelper(zda.GetLaneSizeInBytesLog2(), 8388 zm, 8389 index, 8390 op_h, 8391 op_s, 8392 op_d); 8393 8394 Emit(synthesized_op | Rd(zda) | Rn(zn)); 8395} 8396 8397void Assembler::sqrdmlsh(const ZRegister& zda, 8398 const ZRegister& zn, 8399 const ZRegister& zm) { 8400 // SQRDMLSH <Zda>.<T>, <Zn>.<T>, <Zm>.<T> 8401 // 0100 0100 ..0. .... 0111 01.. .... .... 8402 // size<23:22> | Zm<20:16> | S<10> | Zn<9:5> | Zda<4:0> 8403 8404 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8405 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 8406 8407 Emit(0x44007400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 8408} 8409 8410void Assembler::sqrdmulh(const ZRegister& zd, 8411 const ZRegister& zn, 8412 const ZRegister& zm, 8413 int index) { 8414 // SQRDMULH <Zd>.D, <Zn>.D, <Zm>.D[<imm>] 8415 // 0100 0100 111. .... 1111 01.. .... .... 8416 // size<23:22> | opc<20:16> | R<10> | Zn<9:5> | Zd<4:0> 8417 8418 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8419 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8420 8421 Instr synthesised_op = SVEMulIndexHelper(zd.GetLaneSizeInBytesLog2(), 8422 zm, 8423 index, 8424 0x4420f400, 8425 0x44a0f400, 8426 0x44e0f400); 8427 8428 Emit(synthesised_op | Rd(zd) | Rn(zn)); 8429} 8430 8431void Assembler::sqrdmulh(const ZRegister& zd, 8432 const ZRegister& zn, 8433 const ZRegister& zm) { 8434 // SQRDMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 8435 // 0000 0100 ..1. .... 0111 01.. .... .... 8436 // size<23:22> | Zm<20:16> | R<10> | Zn<9:5> | Zd<4:0> 8437 8438 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8439 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8440 8441 Emit(0x04207400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8442} 8443 8444void Assembler::sqrshl(const ZRegister& zd, 8445 const PRegisterM& pg, 8446 const ZRegister& zn, 8447 const ZRegister& zm) { 8448 // SQRSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8449 // 0100 0100 ..00 1010 100. .... .... .... 8450 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 8451 // Zdn<4:0> 8452 8453 USE(zn); 8454 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8455 VIXL_ASSERT(zd.Is(zn)); 8456 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8457 8458 Emit(0x440a8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8459} 8460 8461void Assembler::sqrshlr(const ZRegister& zd, 8462 const PRegisterM& pg, 8463 const ZRegister& zn, 8464 const ZRegister& zm) { 8465 // SQRSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8466 // 0100 0100 ..00 1110 100. .... .... .... 8467 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 8468 // Zdn<4:0> 8469 8470 USE(zn); 8471 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8472 VIXL_ASSERT(zd.Is(zn)); 8473 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8474 8475 Emit(0x440e8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8476} 8477 8478void Assembler::sqshl(const ZRegister& zd, 8479 const PRegisterM& pg, 8480 const ZRegister& zn, 8481 int shift) { 8482 // SQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> 8483 // 0000 0100 ..00 0110 100. .... .... .... 8484 // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | 8485 // imm3<7:5> | Zdn<4:0> 8486 8487 USE(zn); 8488 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8489 VIXL_ASSERT(zd.Is(zn)); 8490 Instr encoded_imm = 8491 EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); 8492 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x04068000); 8493} 8494 8495void Assembler::sqshl(const ZRegister& zd, 8496 const PRegisterM& pg, 8497 const ZRegister& zn, 8498 const ZRegister& zm) { 8499 // SQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8500 // 0100 0100 ..00 1000 100. .... .... .... 8501 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 8502 // Zdn<4:0> 8503 8504 USE(zn); 8505 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8506 VIXL_ASSERT(zd.Is(zn)); 8507 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8508 8509 Emit(0x44088000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8510} 8511 8512void Assembler::sqshlr(const ZRegister& zd, 8513 const PRegisterM& pg, 8514 const ZRegister& zn, 8515 const ZRegister& zm) { 8516 // SQSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8517 // 0100 0100 ..00 1100 100. .... .... .... 8518 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 8519 // Zdn<4:0> 8520 8521 USE(zn); 8522 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8523 VIXL_ASSERT(zd.Is(zn)); 8524 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8525 8526 Emit(0x440c8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8527} 8528 8529void Assembler::sqshlu(const ZRegister& zd, 8530 const PRegisterM& pg, 8531 const ZRegister& zn, 8532 int shift) { 8533 // SQSHLU <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> 8534 // 0000 0100 ..00 1111 100. .... .... .... 8535 // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | 8536 // imm3<7:5> | Zdn<4:0> 8537 8538 USE(zn); 8539 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8540 VIXL_ASSERT(zd.Is(zn)); 8541 8542 Instr encoded_imm = 8543 EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); 8544 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x040f8000); 8545} 8546 8547void Assembler::sqsub(const ZRegister& zd, 8548 const PRegisterM& pg, 8549 const ZRegister& zn, 8550 const ZRegister& zm) { 8551 // SQSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8552 // 0100 0100 ..01 1010 100. .... .... .... 8553 // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 8554 8555 USE(zn); 8556 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8557 VIXL_ASSERT(zd.Is(zn)); 8558 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8559 8560 Emit(0x441a8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8561} 8562 8563void Assembler::sqsubr(const ZRegister& zd, 8564 const PRegisterM& pg, 8565 const ZRegister& zn, 8566 const ZRegister& zm) { 8567 // SQSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8568 // 0100 0100 ..01 1110 100. .... .... .... 8569 // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 8570 8571 USE(zn); 8572 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8573 VIXL_ASSERT(zd.Is(zn)); 8574 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8575 8576 Emit(0x441e8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8577} 8578 8579void Assembler::sqxtnb(const ZRegister& zd, const ZRegister& zn) { 8580 // SQXTNB <Zd>.<T>, <Zn>.<Tb> 8581 // 0100 0101 0.1. .000 0100 00.. .... .... 8582 // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> 8583 8584 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8585 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); 8586 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); 8587 8588 // XTN instructions look like immediate shifts with zero shift distance. 8589 Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); 8590 SVEBitwiseShiftImmediate(zd, zn, size, 0x45204000); 8591} 8592 8593void Assembler::sqxtnt(const ZRegister& zd, const ZRegister& zn) { 8594 // SQXTNT <Zd>.<T>, <Zn>.<Tb> 8595 // 0100 0101 0.1. .000 0100 01.. .... .... 8596 // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> 8597 8598 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8599 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); 8600 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); 8601 8602 // XTN instructions look like immediate shifts with zero shift distance. 8603 Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); 8604 SVEBitwiseShiftImmediate(zd, zn, size, 0x45204400); 8605} 8606 8607void Assembler::sqxtunb(const ZRegister& zd, const ZRegister& zn) { 8608 // SQXTUNB <Zd>.<T>, <Zn>.<Tb> 8609 // 0100 0101 0.1. .000 0101 00.. .... .... 8610 // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> 8611 8612 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8613 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); 8614 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); 8615 8616 // XTN instructions look like immediate shifts with zero shift distance. 8617 Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); 8618 SVEBitwiseShiftImmediate(zd, zn, size, 0x45205000); 8619} 8620 8621void Assembler::sqxtunt(const ZRegister& zd, const ZRegister& zn) { 8622 // SQXTUNT <Zd>.<T>, <Zn>.<Tb> 8623 // 0100 0101 0.1. .000 0101 01.. .... .... 8624 // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> 8625 8626 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8627 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); 8628 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); 8629 8630 // XTN instructions look like immediate shifts with zero shift distance. 8631 Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); 8632 SVEBitwiseShiftImmediate(zd, zn, size, 0x45205400); 8633} 8634 8635void Assembler::srhadd(const ZRegister& zd, 8636 const PRegisterM& pg, 8637 const ZRegister& zn, 8638 const ZRegister& zm) { 8639 // SRHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8640 // 0100 0100 ..01 0100 100. .... .... .... 8641 // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 8642 8643 USE(zn); 8644 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8645 VIXL_ASSERT(zd.Is(zn)); 8646 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8647 8648 Emit(0x44148000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8649} 8650 8651void Assembler::sri(const ZRegister& zd, const ZRegister& zn, int shift) { 8652 // SRI <Zd>.<T>, <Zn>.<T>, #<const> 8653 // 0100 0101 ..0. .... 1111 00.. .... .... 8654 // tszh<23:22> | tszl<20:19> | imm3<18:16> | op<10> | Zn<9:5> | Zd<4:0> 8655 8656 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8657 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 8658 Instr encoded_imm = 8659 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); 8660 8661 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500f000); 8662} 8663 8664void Assembler::srshl(const ZRegister& zd, 8665 const PRegisterM& pg, 8666 const ZRegister& zn, 8667 const ZRegister& zm) { 8668 // SRSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8669 // 0100 0100 ..00 0010 100. .... .... .... 8670 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 8671 // Zdn<4:0> 8672 8673 USE(zn); 8674 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8675 VIXL_ASSERT(zd.Is(zn)); 8676 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8677 8678 Emit(0x44028000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8679} 8680 8681void Assembler::srshlr(const ZRegister& zd, 8682 const PRegisterM& pg, 8683 const ZRegister& zn, 8684 const ZRegister& zm) { 8685 // SRSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8686 // 0100 0100 ..00 0110 100. .... .... .... 8687 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 8688 // Zdn<4:0> 8689 8690 USE(zn); 8691 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8692 VIXL_ASSERT(zd.Is(zn)); 8693 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8694 8695 Emit(0x44068000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8696} 8697 8698void Assembler::srshr(const ZRegister& zd, 8699 const PRegisterM& pg, 8700 const ZRegister& zn, 8701 int shift) { 8702 // SRSHR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> 8703 // 0000 0100 ..00 1100 100. .... .... .... 8704 // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | 8705 // imm3<7:5> | Zdn<4:0> 8706 8707 USE(zn); 8708 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8709 VIXL_ASSERT(zd.Is(zn)); 8710 Instr encoded_imm = 8711 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); 8712 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x040c8000); 8713} 8714 8715void Assembler::srsra(const ZRegister& zda, const ZRegister& zn, int shift) { 8716 // SRSRA <Zda>.<T>, <Zn>.<T>, #<const> 8717 // 0100 0101 ..0. .... 1110 10.. .... .... 8718 // tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> | 8719 // Zda<4:0> 8720 8721 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8722 VIXL_ASSERT(AreSameLaneSize(zda, zn)); 8723 Instr encoded_imm = 8724 EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits()); 8725 8726 SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500e800); 8727} 8728 8729void Assembler::sshllb(const ZRegister& zd, const ZRegister& zn, int shift) { 8730 // SSHLLB <Zd>.<T>, <Zn>.<Tb>, #<const> 8731 // 0100 0101 0.0. .... 1010 00.. .... .... 8732 // tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 8733 8734 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8735 VIXL_ASSERT(!zd.IsLaneSizeB()); 8736 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8737 8738 Instr encoded_imm = 8739 EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits()); 8740 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500a000); 8741} 8742 8743void Assembler::sshllt(const ZRegister& zd, const ZRegister& zn, int shift) { 8744 // SSHLLT <Zd>.<T>, <Zn>.<Tb>, #<const> 8745 // 0100 0101 0.0. .... 1010 01.. .... .... 8746 // tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 8747 8748 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8749 VIXL_ASSERT(!zd.IsLaneSizeB()); 8750 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8751 8752 Instr encoded_imm = 8753 EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits()); 8754 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500a400); 8755} 8756 8757void Assembler::ssra(const ZRegister& zda, const ZRegister& zn, int shift) { 8758 // SSRA <Zda>.<T>, <Zn>.<T>, #<const> 8759 // 0100 0101 ..0. .... 1110 00.. .... .... 8760 // tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> | 8761 // Zda<4:0> 8762 8763 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8764 VIXL_ASSERT(AreSameLaneSize(zda, zn)); 8765 Instr encoded_imm = 8766 EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits()); 8767 8768 SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500e000); 8769} 8770 8771void Assembler::ssublb(const ZRegister& zd, 8772 const ZRegister& zn, 8773 const ZRegister& zm) { 8774 // SSUBLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8775 // 0100 0101 ..0. .... 0001 00.. .... .... 8776 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 8777 // Zd<4:0> 8778 8779 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8780 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8781 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8782 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 8783 8784 Emit(0x45001000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8785} 8786 8787void Assembler::ssublbt(const ZRegister& zd, 8788 const ZRegister& zn, 8789 const ZRegister& zm) { 8790 // SSUBLBT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8791 // 0100 0101 ..0. .... 1000 10.. .... .... 8792 // size<23:22> | Zm<20:16> | S<11> | tb<10> | Zn<9:5> | Zd<4:0> 8793 8794 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8795 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8796 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8797 VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeH() || zd.IsLaneSizeS()); 8798 8799 Emit(0x45008800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8800} 8801 8802void Assembler::ssublt(const ZRegister& zd, 8803 const ZRegister& zn, 8804 const ZRegister& zm) { 8805 // SSUBLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8806 // 0100 0101 ..0. .... 0001 01.. .... .... 8807 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 8808 // Zd<4:0> 8809 8810 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8811 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8812 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8813 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 8814 8815 Emit(0x45001400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8816} 8817 8818void Assembler::ssubltb(const ZRegister& zd, 8819 const ZRegister& zn, 8820 const ZRegister& zm) { 8821 // SSUBLTB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8822 // 0100 0101 ..0. .... 1000 11.. .... .... 8823 // size<23:22> | Zm<20:16> | S<11> | tb<10> | Zn<9:5> | Zd<4:0> 8824 8825 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8826 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8827 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 8828 VIXL_ASSERT(zd.IsLaneSizeD() || zd.IsLaneSizeH() || zd.IsLaneSizeS()); 8829 8830 Emit(0x45008c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8831} 8832 8833void Assembler::ssubwb(const ZRegister& zd, 8834 const ZRegister& zn, 8835 const ZRegister& zm) { 8836 // SSUBWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> 8837 // 0100 0101 ..0. .... 0101 00.. .... .... 8838 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 8839 8840 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8841 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 8842 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); 8843 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 8844 8845 Emit(0x45005000 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8846} 8847 8848void Assembler::ssubwt(const ZRegister& zd, 8849 const ZRegister& zn, 8850 const ZRegister& zm) { 8851 // SSUBWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> 8852 // 0100 0101 ..0. .... 0101 01.. .... .... 8853 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 8854 8855 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8856 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 8857 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); 8858 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 8859 8860 Emit(0x45005400 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8861} 8862 8863#if 0 8864// This prototype maps to 2 instruction encodings: 8865// stnt1b_z_p_ar_d_64_unscaled 8866// stnt1b_z_p_ar_s_x32_unscaled 8867void Assembler::stnt1b(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) { 8868 // STNT1B { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}] 8869 // 1110 0100 000. .... 001. .... .... .... 8870 // msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0> 8871 8872 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8873 8874 Emit(0xe4002000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); 8875} 8876 8877void Assembler::stnt1d(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) { 8878 // STNT1D { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}] 8879 // 1110 0101 100. .... 001. .... .... .... 8880 // msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0> 8881 8882 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8883 8884 Emit(0xe5802000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); 8885} 8886 8887// This prototype maps to 2 instruction encodings: 8888// stnt1h_z_p_ar_d_64_unscaled 8889// stnt1h_z_p_ar_s_x32_unscaled 8890void Assembler::stnt1h(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) { 8891 // STNT1H { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}] 8892 // 1110 0100 100. .... 001. .... .... .... 8893 // msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0> 8894 8895 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8896 8897 Emit(0xe4802000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); 8898} 8899 8900// This prototype maps to 2 instruction encodings: 8901// stnt1w_z_p_ar_d_64_unscaled 8902// stnt1w_z_p_ar_s_x32_unscaled 8903void Assembler::stnt1w(const ZRegister& zt, const PRegister& pg, const ZRegister& zn, const Register& rm) { 8904 // STNT1W { <Zt>.D }, <Pg>, [<Zn>.D{, <Xm>}] 8905 // 1110 0101 000. .... 001. .... .... .... 8906 // msz<24:23> | Rm<20:16> | Pg<12:10> | Zn<9:5> | Zt<4:0> 8907 8908 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8909 8910 Emit(0xe5002000 | Rt(zt) | PgLow8(pg) | Rn(zn) | Rm(rm)); 8911} 8912#endif 8913 8914void Assembler::subhnb(const ZRegister& zd, 8915 const ZRegister& zn, 8916 const ZRegister& zm) { 8917 // SUBHNB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8918 // 0100 0101 ..1. .... 0111 00.. .... .... 8919 // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> 8920 8921 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8922 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8923 VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); 8924 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 8925 8926 Emit(0x45207000 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); 8927} 8928 8929void Assembler::subhnt(const ZRegister& zd, 8930 const ZRegister& zn, 8931 const ZRegister& zm) { 8932 // SUBHNT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 8933 // 0100 0101 ..1. .... 0111 01.. .... .... 8934 // size<23:22> | Zm<20:16> | S<12> | R<11> | T<10> | Zn<9:5> | Zd<4:0> 8935 8936 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8937 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 8938 VIXL_ASSERT(zn.GetLaneSizeInBytes() == (zd.GetLaneSizeInBytes() * 2)); 8939 VIXL_ASSERT(zn.GetLaneSizeInBytes() != kBRegSizeInBytes); 8940 8941 Emit(0x45207400 | SVESize(zn) | Rd(zd) | Rn(zn) | Rm(zm)); 8942} 8943 8944void Assembler::suqadd(const ZRegister& zd, 8945 const PRegisterM& pg, 8946 const ZRegister& zn, 8947 const ZRegister& zm) { 8948 // SUQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 8949 // 0100 0100 ..01 1100 100. .... .... .... 8950 // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 8951 8952 USE(zn); 8953 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8954 VIXL_ASSERT(zd.Is(zn)); 8955 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8956 8957 Emit(0x441c8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 8958} 8959 8960void Assembler::tbl(const ZRegister& zd, 8961 const ZRegister& zn1, 8962 const ZRegister& zn2, 8963 const ZRegister& zm) { 8964 // TBL <Zd>.<T>, { <Zn1>.<T>, <Zn2>.<T> }, <Zm>.<T> 8965 // 0000 0101 ..1. .... 0010 10.. .... .... 8966 // size<23:22> | Zm<20:16> | op<10> | Zn<9:5> | Zd<4:0> 8967 8968 USE(zn2); 8969 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8970 VIXL_ASSERT(AreConsecutive(zn1, zn2)); 8971 VIXL_ASSERT(AreSameLaneSize(zd, zn1, zn2, zm)); 8972 8973 Emit(0x05202800 | SVESize(zd) | Rd(zd) | Rn(zn1) | Rn(zn2) | Rm(zm)); 8974} 8975 8976void Assembler::tbx(const ZRegister& zd, 8977 const ZRegister& zn, 8978 const ZRegister& zm) { 8979 // TBX <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 8980 // 0000 0101 ..1. .... 0010 11.. .... .... 8981 // size<23:22> | Zm<20:16> | op<10> | Zn<9:5> | Zd<4:0> 8982 8983 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8984 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 8985 8986 Emit(0x05202c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 8987} 8988 8989void Assembler::uaba(const ZRegister& zda, 8990 const ZRegister& zn, 8991 const ZRegister& zm) { 8992 // UABA <Zda>.<T>, <Zn>.<T>, <Zm>.<T> 8993 // 0100 0101 ..0. .... 1111 11.. .... .... 8994 // size<23:22> | Zm<20:16> | U<10> | Zn<9:5> | Zda<4:0> 8995 8996 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 8997 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 8998 8999 Emit(0x4500fc00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 9000} 9001 9002void Assembler::uabalb(const ZRegister& zda, 9003 const ZRegister& zn, 9004 const ZRegister& zm) { 9005 // UABALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9006 // 0100 0101 ..0. .... 1100 10.. .... .... 9007 // size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 9008 9009 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9010 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9011 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9012 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 9013 9014 Emit(0x4500c800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 9015} 9016 9017void Assembler::uabalt(const ZRegister& zda, 9018 const ZRegister& zn, 9019 const ZRegister& zm) { 9020 // UABALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9021 // 0100 0101 ..0. .... 1100 11.. .... .... 9022 // size<23:22> | Zm<20:16> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 9023 9024 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9025 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9026 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9027 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 9028 9029 Emit(0x4500cc00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 9030} 9031 9032void Assembler::uabdlb(const ZRegister& zd, 9033 const ZRegister& zn, 9034 const ZRegister& zm) { 9035 // UABDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9036 // 0100 0101 ..0. .... 0011 10.. .... .... 9037 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 9038 // Zd<4:0> 9039 9040 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9041 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9042 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9043 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9044 9045 Emit(0x45003800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9046} 9047 9048void Assembler::uabdlt(const ZRegister& zd, 9049 const ZRegister& zn, 9050 const ZRegister& zm) { 9051 // UABDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9052 // 0100 0101 ..0. .... 0011 11.. .... .... 9053 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 9054 // Zd<4:0> 9055 9056 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9057 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9058 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9059 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9060 9061 Emit(0x45003c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9062} 9063 9064void Assembler::uadalp(const ZRegister& zda, 9065 const PRegisterM& pg, 9066 const ZRegister& zn) { 9067 // UADALP <Zda>.<T>, <Pg>/M, <Zn>.<Tb> 9068 // 0100 0100 ..00 0101 101. .... .... .... 9069 // size<23:22> | U<16> | Pg<12:10> | Zn<9:5> | Zda<4:0> 9070 9071 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9072 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9073 VIXL_ASSERT(zda.GetLaneSizeInBytes() != kBRegSizeInBytes); 9074 9075 Emit(0x4405a000 | SVESize(zda) | Rd(zda) | PgLow8(pg) | Rn(zn)); 9076} 9077 9078void Assembler::uaddlb(const ZRegister& zd, 9079 const ZRegister& zn, 9080 const ZRegister& zm) { 9081 // UADDLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9082 // 0100 0101 ..0. .... 0000 10.. .... .... 9083 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 9084 // Zd<4:0> 9085 9086 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9087 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9088 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9089 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9090 9091 Emit(0x45000800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9092} 9093 9094void Assembler::uaddlt(const ZRegister& zd, 9095 const ZRegister& zn, 9096 const ZRegister& zm) { 9097 // UADDLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9098 // 0100 0101 ..0. .... 0000 11.. .... .... 9099 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 9100 // Zd<4:0> 9101 9102 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9103 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9104 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9105 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9106 9107 Emit(0x45000c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9108} 9109 9110void Assembler::uaddwb(const ZRegister& zd, 9111 const ZRegister& zn, 9112 const ZRegister& zm) { 9113 // UADDWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> 9114 // 0100 0101 ..0. .... 0100 10.. .... .... 9115 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 9116 9117 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9118 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 9119 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); 9120 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9121 9122 Emit(0x45004800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9123} 9124 9125void Assembler::uaddwt(const ZRegister& zd, 9126 const ZRegister& zn, 9127 const ZRegister& zm) { 9128 // UADDWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> 9129 // 0100 0101 ..0. .... 0100 11.. .... .... 9130 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 9131 9132 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9133 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 9134 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); 9135 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9136 9137 Emit(0x45004c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9138} 9139 9140void Assembler::uhadd(const ZRegister& zd, 9141 const PRegisterM& pg, 9142 const ZRegister& zn, 9143 const ZRegister& zm) { 9144 // UHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9145 // 0100 0100 ..01 0001 100. .... .... .... 9146 // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9147 9148 USE(zn); 9149 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9150 VIXL_ASSERT(zd.Is(zn)); 9151 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9152 9153 Emit(0x44118000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9154} 9155 9156void Assembler::uhsub(const ZRegister& zd, 9157 const PRegisterM& pg, 9158 const ZRegister& zn, 9159 const ZRegister& zm) { 9160 // UHSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9161 // 0100 0100 ..01 0011 100. .... .... .... 9162 // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9163 9164 USE(zn); 9165 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9166 VIXL_ASSERT(zd.Is(zn)); 9167 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9168 9169 Emit(0x44138000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9170} 9171 9172void Assembler::uhsubr(const ZRegister& zd, 9173 const PRegisterM& pg, 9174 const ZRegister& zn, 9175 const ZRegister& zm) { 9176 // UHSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9177 // 0100 0100 ..01 0111 100. .... .... .... 9178 // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9179 9180 USE(zn); 9181 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9182 VIXL_ASSERT(zd.Is(zn)); 9183 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9184 9185 Emit(0x44178000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9186} 9187 9188void Assembler::umaxp(const ZRegister& zd, 9189 const PRegisterM& pg, 9190 const ZRegister& zn, 9191 const ZRegister& zm) { 9192 // UMAXP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9193 // 0100 0100 ..01 0101 101. .... .... .... 9194 // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9195 9196 USE(zn); 9197 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9198 VIXL_ASSERT(zd.Is(zn)); 9199 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9200 9201 Emit(0x4415a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9202} 9203 9204void Assembler::uminp(const ZRegister& zd, 9205 const PRegisterM& pg, 9206 const ZRegister& zn, 9207 const ZRegister& zm) { 9208 // UMINP <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9209 // 0100 0100 ..01 0111 101. .... .... .... 9210 // size<23:22> | opc<18:17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9211 9212 USE(zn); 9213 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9214 VIXL_ASSERT(zd.Is(zn)); 9215 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9216 9217 Emit(0x4417a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9218} 9219 9220void Assembler::umlalb(const ZRegister& zda, 9221 const ZRegister& zn, 9222 const ZRegister& zm) { 9223 // UMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9224 // 0100 0100 ..0. .... 0100 10.. .... .... 9225 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 9226 9227 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9228 VIXL_ASSERT(!zda.IsLaneSizeB()); 9229 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9230 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9231 9232 Emit(0x44004800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 9233} 9234 9235void Assembler::umlalt(const ZRegister& zda, 9236 const ZRegister& zn, 9237 const ZRegister& zm) { 9238 // UMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9239 // 0100 0100 ..0. .... 0100 11.. .... .... 9240 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 9241 9242 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9243 VIXL_ASSERT(!zda.IsLaneSizeB()); 9244 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9245 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9246 9247 Emit(0x44004c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 9248} 9249 9250void Assembler::umlslb(const ZRegister& zda, 9251 const ZRegister& zn, 9252 const ZRegister& zm) { 9253 // UMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9254 // 0100 0100 ..0. .... 0101 10.. .... .... 9255 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 9256 9257 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9258 VIXL_ASSERT(!zda.IsLaneSizeB()); 9259 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9260 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9261 9262 Emit(0x44005800 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 9263} 9264 9265void Assembler::umlslt(const ZRegister& zda, 9266 const ZRegister& zn, 9267 const ZRegister& zm) { 9268 // UMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9269 // 0100 0100 ..0. .... 0101 11.. .... .... 9270 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zda<4:0> 9271 9272 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9273 VIXL_ASSERT(!zda.IsLaneSizeB()); 9274 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9275 VIXL_ASSERT(zda.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9276 9277 Emit(0x44005c00 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 9278} 9279 9280void Assembler::umulh(const ZRegister& zd, 9281 const ZRegister& zn, 9282 const ZRegister& zm) { 9283 // UMULH <Zd>.<T>, <Zn>.<T>, <Zm>.<T> 9284 // 0000 0100 ..1. .... 0110 11.. .... .... 9285 // size<23:22> | Zm<20:16> | opc<11:10> | Zn<9:5> | Zd<4:0> 9286 9287 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9288 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9289 9290 Emit(0x04206c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9291} 9292 9293void Assembler::umullb(const ZRegister& zd, 9294 const ZRegister& zn, 9295 const ZRegister& zm) { 9296 // UMULLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9297 // 0100 0101 ..0. .... 0111 10.. .... .... 9298 // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 9299 9300 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9301 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9302 VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); 9303 VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); 9304 9305 Emit(0x45007800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9306} 9307 9308void Assembler::umullt(const ZRegister& zd, 9309 const ZRegister& zn, 9310 const ZRegister& zm) { 9311 // UMULLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9312 // 0100 0101 ..0. .... 0111 11.. .... .... 9313 // size<23:22> | Zm<20:16> | op<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 9314 9315 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9316 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9317 VIXL_ASSERT(!zd.IsLaneSizeB() && !zd.IsLaneSizeQ()); 9318 VIXL_ASSERT(zd.GetLaneSizeInBytes() == zn.GetLaneSizeInBytes() * 2); 9319 9320 Emit(0x45007c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9321} 9322 9323void Assembler::uqadd(const ZRegister& zd, 9324 const PRegisterM& pg, 9325 const ZRegister& zn, 9326 const ZRegister& zm) { 9327 // UQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9328 // 0100 0100 ..01 1001 100. .... .... .... 9329 // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9330 9331 USE(zn); 9332 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9333 VIXL_ASSERT(zd.Is(zn)); 9334 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9335 9336 Emit(0x44198000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9337} 9338 9339void Assembler::uqrshl(const ZRegister& zd, 9340 const PRegisterM& pg, 9341 const ZRegister& zn, 9342 const ZRegister& zm) { 9343 // UQRSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9344 // 0100 0100 ..00 1011 100. .... .... .... 9345 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 9346 // Zdn<4:0> 9347 9348 USE(zn); 9349 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9350 VIXL_ASSERT(zd.Is(zn)); 9351 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9352 9353 Emit(0x440b8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9354} 9355 9356void Assembler::uqrshlr(const ZRegister& zd, 9357 const PRegisterM& pg, 9358 const ZRegister& zn, 9359 const ZRegister& zm) { 9360 // UQRSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9361 // 0100 0100 ..00 1111 100. .... .... .... 9362 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 9363 // Zdn<4:0> 9364 9365 USE(zn); 9366 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9367 VIXL_ASSERT(zd.Is(zn)); 9368 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9369 9370 Emit(0x440f8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9371} 9372 9373void Assembler::uqshl(const ZRegister& zd, 9374 const PRegisterM& pg, 9375 const ZRegister& zn, 9376 int shift) { 9377 // UQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> 9378 // 0000 0100 ..00 0111 100. .... .... .... 9379 // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | 9380 // imm3<7:5> | Zdn<4:0> 9381 9382 USE(zn); 9383 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9384 VIXL_ASSERT(zd.Is(zn)); 9385 Instr encoded_imm = 9386 EncodeSVEShiftLeftImmediate(shift, zd.GetLaneSizeInBits()); 9387 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x04078000); 9388} 9389 9390void Assembler::uqshl(const ZRegister& zd, 9391 const PRegisterM& pg, 9392 const ZRegister& zn, 9393 const ZRegister& zm) { 9394 // UQSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9395 // 0100 0100 ..00 1001 100. .... .... .... 9396 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 9397 // Zdn<4:0> 9398 9399 USE(zn); 9400 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9401 VIXL_ASSERT(zd.Is(zn)); 9402 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9403 9404 Emit(0x44098000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9405} 9406 9407void Assembler::uqshlr(const ZRegister& zd, 9408 const PRegisterM& pg, 9409 const ZRegister& zn, 9410 const ZRegister& zm) { 9411 // UQSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9412 // 0100 0100 ..00 1101 100. .... .... .... 9413 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 9414 // Zdn<4:0> 9415 9416 USE(zn); 9417 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9418 VIXL_ASSERT(zd.Is(zn)); 9419 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9420 9421 Emit(0x440d8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9422} 9423 9424void Assembler::uqsub(const ZRegister& zd, 9425 const PRegisterM& pg, 9426 const ZRegister& zn, 9427 const ZRegister& zm) { 9428 // UQSUB <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9429 // 0100 0100 ..01 1011 100. .... .... .... 9430 // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9431 9432 USE(zn); 9433 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9434 VIXL_ASSERT(zd.Is(zn)); 9435 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9436 9437 Emit(0x441b8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9438} 9439 9440void Assembler::uqsubr(const ZRegister& zd, 9441 const PRegisterM& pg, 9442 const ZRegister& zn, 9443 const ZRegister& zm) { 9444 // UQSUBR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9445 // 0100 0100 ..01 1111 100. .... .... .... 9446 // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9447 9448 USE(zn); 9449 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9450 VIXL_ASSERT(zd.Is(zn)); 9451 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9452 9453 Emit(0x441f8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9454} 9455 9456void Assembler::uqxtnb(const ZRegister& zd, const ZRegister& zn) { 9457 // UQXTNB <Zd>.<T>, <Zn>.<Tb> 9458 // 0100 0101 0.1. .000 0100 10.. .... .... 9459 // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> 9460 9461 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9462 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); 9463 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); 9464 9465 // XTN instructions look like immediate shifts with zero shift distance. 9466 Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); 9467 SVEBitwiseShiftImmediate(zd, zn, size, 0x45204800); 9468} 9469 9470void Assembler::uqxtnt(const ZRegister& zd, const ZRegister& zn) { 9471 // UQXTNT <Zd>.<T>, <Zn>.<Tb> 9472 // 0100 0101 0.1. .000 0100 11.. .... .... 9473 // tszh<22> | tszl<20:19> | opc<12:11> | T<10> | Zn<9:5> | Zd<4:0> 9474 9475 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9476 VIXL_ASSERT(zd.IsLaneSizeS() || zd.IsLaneSizeH() || zd.IsLaneSizeB()); 9477 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() / 2)); 9478 9479 // XTN instructions look like immediate shifts with zero shift distance. 9480 Instr size = EncodeSVEShiftLeftImmediate(0, zd.GetLaneSizeInBits()); 9481 SVEBitwiseShiftImmediate(zd, zn, size, 0x45204c00); 9482} 9483 9484void Assembler::urecpe(const ZRegister& zd, 9485 const PRegisterM& pg, 9486 const ZRegister& zn) { 9487 // URECPE <Zd>.S, <Pg>/M, <Zn>.S 9488 // 0100 0100 ..00 0000 101. .... .... .... 9489 // size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 9490 9491 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9492 VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeS()); 9493 9494 Emit(0x4400a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 9495} 9496 9497void Assembler::urhadd(const ZRegister& zd, 9498 const PRegisterM& pg, 9499 const ZRegister& zn, 9500 const ZRegister& zm) { 9501 // URHADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9502 // 0100 0100 ..01 0101 100. .... .... .... 9503 // size<23:22> | R<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9504 9505 USE(zn); 9506 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9507 VIXL_ASSERT(zd.Is(zn)); 9508 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9509 9510 Emit(0x44158000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9511} 9512 9513void Assembler::urshl(const ZRegister& zd, 9514 const PRegisterM& pg, 9515 const ZRegister& zn, 9516 const ZRegister& zm) { 9517 // URSHL <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9518 // 0100 0100 ..00 0011 100. .... .... .... 9519 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 9520 // Zdn<4:0> 9521 9522 USE(zn); 9523 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9524 VIXL_ASSERT(zd.Is(zn)); 9525 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9526 9527 Emit(0x44038000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9528} 9529 9530void Assembler::urshlr(const ZRegister& zd, 9531 const PRegisterM& pg, 9532 const ZRegister& zn, 9533 const ZRegister& zm) { 9534 // URSHLR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9535 // 0100 0100 ..00 0111 100. .... .... .... 9536 // size<23:22> | Q<19> | R<18> | N<17> | U<16> | Pg<12:10> | Zm<9:5> | 9537 // Zdn<4:0> 9538 9539 USE(zn); 9540 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9541 VIXL_ASSERT(zd.Is(zn)); 9542 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9543 9544 Emit(0x44078000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9545} 9546 9547void Assembler::urshr(const ZRegister& zd, 9548 const PRegisterM& pg, 9549 const ZRegister& zn, 9550 int shift) { 9551 // URSHR <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, #<const> 9552 // 0000 0100 ..00 1101 100. .... .... .... 9553 // tszh<23:22> | opc<19:18> | L<17> | U<16> | Pg<12:10> | tszl<9:8> | 9554 // imm3<7:5> | Zdn<4:0> 9555 9556 USE(zn); 9557 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9558 VIXL_ASSERT(zd.Is(zn)); 9559 Instr encoded_imm = 9560 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); 9561 SVEBitwiseShiftImmediatePred(zd, pg, encoded_imm, 0x040d8000); 9562} 9563 9564void Assembler::ursqrte(const ZRegister& zd, 9565 const PRegisterM& pg, 9566 const ZRegister& zn) { 9567 // URSQRTE <Zd>.S, <Pg>/M, <Zn>.S 9568 // 0100 0100 ..00 0001 101. .... .... .... 9569 // size<23:22> | Q<19> | opc<17:16> | Pg<12:10> | Zn<9:5> | Zd<4:0> 9570 9571 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9572 VIXL_ASSERT(zd.IsLaneSizeS() && zn.IsLaneSizeS()); 9573 9574 Emit(0x4401a000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zn)); 9575} 9576 9577void Assembler::ursra(const ZRegister& zda, const ZRegister& zn, int shift) { 9578 // URSRA <Zda>.<T>, <Zn>.<T>, #<const> 9579 // 0100 0101 ..0. .... 1110 11.. .... .... 9580 // tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> | 9581 // Zda<4:0> 9582 9583 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9584 VIXL_ASSERT(AreSameLaneSize(zda, zn)); 9585 Instr encoded_imm = 9586 EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits()); 9587 9588 SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500ec00); 9589} 9590 9591void Assembler::ushllb(const ZRegister& zd, const ZRegister& zn, int shift) { 9592 // USHLLB <Zd>.<T>, <Zn>.<Tb>, #<const> 9593 // 0100 0101 0.0. .... 1010 10.. .... .... 9594 // tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 9595 9596 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9597 VIXL_ASSERT(!zd.IsLaneSizeB()); 9598 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9599 9600 Instr encoded_imm = 9601 EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits()); 9602 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500a800); 9603} 9604 9605void Assembler::ushllt(const ZRegister& zd, const ZRegister& zn, int shift) { 9606 // USHLLT <Zd>.<T>, <Zn>.<Tb>, #<const> 9607 // 0100 0101 0.0. .... 1010 11.. .... .... 9608 // tszh<22> | tszl<20:19> | imm3<18:16> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 9609 9610 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9611 VIXL_ASSERT(!zd.IsLaneSizeB()); 9612 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9613 9614 Instr encoded_imm = 9615 EncodeSVEShiftLeftImmediate(shift, zn.GetLaneSizeInBits()); 9616 SVEBitwiseShiftImmediate(zd, zn, encoded_imm, 0x4500ac00); 9617} 9618 9619void Assembler::usqadd(const ZRegister& zd, 9620 const PRegisterM& pg, 9621 const ZRegister& zn, 9622 const ZRegister& zm) { 9623 // USQADD <Zdn>.<T>, <Pg>/M, <Zdn>.<T>, <Zm>.<T> 9624 // 0100 0100 ..01 1101 100. .... .... .... 9625 // size<23:22> | op<18> | S<17> | U<16> | Pg<12:10> | Zm<9:5> | Zdn<4:0> 9626 9627 USE(zn); 9628 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9629 VIXL_ASSERT(zd.Is(zn)); 9630 VIXL_ASSERT(AreSameLaneSize(zd, zn, zm)); 9631 9632 Emit(0x441d8000 | SVESize(zd) | Rd(zd) | PgLow8(pg) | Rn(zm)); 9633} 9634 9635void Assembler::usra(const ZRegister& zda, const ZRegister& zn, int shift) { 9636 // USRA <Zda>.<T>, <Zn>.<T>, #<const> 9637 // 0100 0101 ..0. .... 1110 01.. .... .... 9638 // tszh<23:22> | tszl<20:19> | imm3<18:16> | R<11> | U<10> | Zn<9:5> | 9639 // Zda<4:0> 9640 9641 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9642 VIXL_ASSERT(AreSameLaneSize(zda, zn)); 9643 Instr encoded_imm = 9644 EncodeSVEShiftRightImmediate(shift, zda.GetLaneSizeInBits()); 9645 9646 SVEBitwiseShiftImmediate(zda, zn, encoded_imm, 0x4500e400); 9647} 9648 9649void Assembler::usublb(const ZRegister& zd, 9650 const ZRegister& zn, 9651 const ZRegister& zm) { 9652 // USUBLB <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9653 // 0100 0101 ..0. .... 0001 10.. .... .... 9654 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 9655 // Zd<4:0> 9656 9657 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9658 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9659 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9660 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9661 9662 Emit(0x45001800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9663} 9664 9665void Assembler::usublt(const ZRegister& zd, 9666 const ZRegister& zn, 9667 const ZRegister& zm) { 9668 // USUBLT <Zd>.<T>, <Zn>.<Tb>, <Zm>.<Tb> 9669 // 0100 0101 ..0. .... 0001 11.. .... .... 9670 // size<23:22> | Zm<20:16> | op<13> | S<12> | U<11> | T<10> | Zn<9:5> | 9671 // Zd<4:0> 9672 9673 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9674 VIXL_ASSERT(AreSameLaneSize(zn, zm)); 9675 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zn.GetLaneSizeInBytes() * 2)); 9676 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9677 9678 Emit(0x45001c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9679} 9680 9681void Assembler::usubwb(const ZRegister& zd, 9682 const ZRegister& zn, 9683 const ZRegister& zm) { 9684 // USUBWB <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> 9685 // 0100 0101 ..0. .... 0101 10.. .... .... 9686 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 9687 9688 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9689 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 9690 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); 9691 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9692 9693 Emit(0x45005800 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9694} 9695 9696void Assembler::usubwt(const ZRegister& zd, 9697 const ZRegister& zn, 9698 const ZRegister& zm) { 9699 // USUBWT <Zd>.<T>, <Zn>.<T>, <Zm>.<Tb> 9700 // 0100 0101 ..0. .... 0101 11.. .... .... 9701 // size<23:22> | Zm<20:16> | S<12> | U<11> | T<10> | Zn<9:5> | Zd<4:0> 9702 9703 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9704 VIXL_ASSERT(AreSameLaneSize(zd, zn)); 9705 VIXL_ASSERT(zd.GetLaneSizeInBytes() == (zm.GetLaneSizeInBytes() * 2)); 9706 VIXL_ASSERT(zd.GetLaneSizeInBytes() != kBRegSizeInBytes); 9707 9708 Emit(0x45005c00 | SVESize(zd) | Rd(zd) | Rn(zn) | Rm(zm)); 9709} 9710 9711void Assembler::whilege(const PRegisterWithLaneSize& pd, 9712 const Register& rn, 9713 const Register& rm) { 9714 // WHILEGE <Pd>.<T>, <R><n>, <R><m> 9715 // 0010 0101 ..1. .... 000. 00.. ...0 .... 9716 // size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> | 9717 // Pd<3:0> 9718 9719 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9720 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 9721 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; 9722 9723 Emit(0x25200000 | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); 9724} 9725 9726void Assembler::whilegt(const PRegisterWithLaneSize& pd, 9727 const Register& rn, 9728 const Register& rm) { 9729 // WHILEGT <Pd>.<T>, <R><n>, <R><m> 9730 // 0010 0101 ..1. .... 000. 00.. ...1 .... 9731 // size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> | 9732 // Pd<3:0> 9733 9734 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9735 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 9736 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; 9737 9738 Emit(0x25200010 | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); 9739} 9740 9741void Assembler::whilehi(const PRegisterWithLaneSize& pd, 9742 const Register& rn, 9743 const Register& rm) { 9744 // WHILEHI <Pd>.<T>, <R><n>, <R><m> 9745 // 0010 0101 ..1. .... 000. 10.. ...1 .... 9746 // size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> | 9747 // Pd<3:0> 9748 9749 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9750 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 9751 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; 9752 9753 Emit(0x25200810 | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); 9754} 9755 9756void Assembler::whilehs(const PRegisterWithLaneSize& pd, 9757 const Register& rn, 9758 const Register& rm) { 9759 // WHILEHS <Pd>.<T>, <R><n>, <R><m> 9760 // 0010 0101 ..1. .... 000. 10.. ...0 .... 9761 // size<23:22> | Rm<20:16> | sf<12> | U<11> | lt<10> | Rn<9:5> | eq<4> | 9762 // Pd<3:0> 9763 9764 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9765 VIXL_ASSERT(AreSameSizeAndType(rn, rm)); 9766 const Instr sf = rn.Is64Bits() ? 0x00001000 : 0x00000000; 9767 9768 Emit(0x25200800 | SVESize(pd) | sf | Pd(pd) | Rn(rn) | Rm(rm)); 9769} 9770 9771void Assembler::whilerw(const PRegisterWithLaneSize& pd, 9772 const Register& rn, 9773 const Register& rm) { 9774 // WHILERW <Pd>.<T>, <Xn>, <Xm> 9775 // 0010 0101 ..1. .... 0011 00.. ...1 .... 9776 // size<23:22> | Rm<20:16> | Rn<9:5> | rw<4> | Pd<3:0> 9777 9778 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9779 VIXL_ASSERT(rn.IsX() && rm.IsX()); 9780 9781 Emit(0x25203010 | SVESize(pd) | Pd(pd) | Rn(rn) | Rm(rm)); 9782} 9783 9784void Assembler::whilewr(const PRegisterWithLaneSize& pd, 9785 const Register& rn, 9786 const Register& rm) { 9787 // WHILEWR <Pd>.<T>, <Xn>, <Xm> 9788 // 0010 0101 ..1. .... 0011 00.. ...0 .... 9789 // size<23:22> | Rm<20:16> | Rn<9:5> | rw<4> | Pd<3:0> 9790 9791 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9792 VIXL_ASSERT(rn.IsX() && rm.IsX()); 9793 9794 Emit(0x25203000 | SVESize(pd) | Pd(pd) | Rn(rn) | Rm(rm)); 9795} 9796 9797void Assembler::xar(const ZRegister& zd, 9798 const ZRegister& zn, 9799 const ZRegister& zm, 9800 int shift) { 9801 // XAR <Zdn>.<T>, <Zdn>.<T>, <Zm>.<T>, #<const> 9802 // 0000 0100 ..1. .... 0011 01.. .... .... 9803 // tszh<23:22> | tszl<20:19> | imm3<18:16> | Zm<9:5> | Zdn<4:0> 9804 9805 USE(zn); 9806 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE2)); 9807 VIXL_ASSERT(zd.Is(zn)); 9808 VIXL_ASSERT(AreSameLaneSize(zd, zm)); 9809 9810 Instr encoded_imm = 9811 EncodeSVEShiftRightImmediate(shift, zd.GetLaneSizeInBits()); 9812 SVEBitwiseShiftImmediate(zd, zm, encoded_imm, 0x04203400); 9813} 9814 9815void Assembler::fmmla(const ZRegister& zda, 9816 const ZRegister& zn, 9817 const ZRegister& zm) { 9818 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 9819 VIXL_ASSERT((CPUHas(CPUFeatures::kSVEF32MM) && zda.IsLaneSizeS()) || 9820 (CPUHas(CPUFeatures::kSVEF64MM) && zda.IsLaneSizeD())); 9821 VIXL_ASSERT(AreSameLaneSize(zda, zn, zm)); 9822 9823 Emit(0x6420e400 | SVESize(zda) | Rd(zda) | Rn(zn) | Rm(zm)); 9824} 9825 9826void Assembler::smmla(const ZRegister& zda, 9827 const ZRegister& zn, 9828 const ZRegister& zm) { 9829 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 9830 VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM)); 9831 VIXL_ASSERT(zda.IsLaneSizeS()); 9832 VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB()); 9833 9834 Emit(0x45009800 | Rd(zda) | Rn(zn) | Rm(zm)); 9835} 9836 9837void Assembler::usmmla(const ZRegister& zda, 9838 const ZRegister& zn, 9839 const ZRegister& zm) { 9840 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 9841 VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM)); 9842 VIXL_ASSERT(zda.IsLaneSizeS()); 9843 VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB()); 9844 9845 Emit(0x45809800 | Rd(zda) | Rn(zn) | Rm(zm)); 9846} 9847 9848void Assembler::ummla(const ZRegister& zda, 9849 const ZRegister& zn, 9850 const ZRegister& zm) { 9851 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 9852 VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM)); 9853 VIXL_ASSERT(zda.IsLaneSizeS()); 9854 VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB()); 9855 9856 Emit(0x45c09800 | Rd(zda) | Rn(zn) | Rm(zm)); 9857} 9858 9859void Assembler::usdot(const ZRegister& zda, 9860 const ZRegister& zn, 9861 const ZRegister& zm) { 9862 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 9863 VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM)); 9864 VIXL_ASSERT(zda.IsLaneSizeS()); 9865 VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB()); 9866 9867 Emit(0x44807800 | Rd(zda) | Rn(zn) | Rm(zm)); 9868} 9869 9870void Assembler::usdot(const ZRegister& zda, 9871 const ZRegister& zn, 9872 const ZRegister& zm, 9873 int index) { 9874 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 9875 VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM)); 9876 VIXL_ASSERT(zda.IsLaneSizeS()); 9877 VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB()); 9878 VIXL_ASSERT(zm.GetCode() <= 7); 9879 VIXL_ASSERT(IsUint2(index)); 9880 9881 Emit(0x44a01800 | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn)); 9882} 9883 9884void Assembler::sudot(const ZRegister& zda, 9885 const ZRegister& zn, 9886 const ZRegister& zm, 9887 int index) { 9888 VIXL_ASSERT(CPUHas(CPUFeatures::kSVE)); 9889 VIXL_ASSERT(CPUHas(CPUFeatures::kSVEI8MM)); 9890 VIXL_ASSERT(zda.IsLaneSizeS()); 9891 VIXL_ASSERT(zn.IsLaneSizeB() && zm.IsLaneSizeB()); 9892 VIXL_ASSERT(zm.GetCode() <= 7); 9893 VIXL_ASSERT(IsUint2(index)); 9894 9895 Emit(0x44a01c00 | Rx<18, 16>(zm) | (index << 19) | Rd(zda) | Rn(zn)); 9896} 9897 9898} // namespace aarch64 9899} // namespace vixl 9900