Lines Matching refs:vd
315 void Assembler::NEONTable(const VRegister& vd,
320 VIXL_ASSERT(vd.Is16B() || vd.Is8B());
322 VIXL_ASSERT(AreSameFormat(vd, vm));
323 Emit(op | (vd.IsQ() ? NEON_Q : 0) | Rm(vm) | Rn(vn) | Rd(vd));
327 void Assembler::tbl(const VRegister& vd,
331 NEONTable(vd, vn, vm, NEON_TBL_1v);
335 void Assembler::tbl(const VRegister& vd,
343 NEONTable(vd, vn, vm, NEON_TBL_2v);
347 void Assembler::tbl(const VRegister& vd,
356 NEONTable(vd, vn, vm, NEON_TBL_3v);
360 void Assembler::tbl(const VRegister& vd,
370 NEONTable(vd, vn, vm, NEON_TBL_4v);
374 void Assembler::tbx(const VRegister& vd,
378 NEONTable(vd, vn, vm, NEON_TBX_1v);
382 void Assembler::tbx(const VRegister& vd,
390 NEONTable(vd, vn, vm, NEON_TBX_2v);
394 void Assembler::tbx(const VRegister& vd,
403 NEONTable(vd, vn, vm, NEON_TBX_3v);
407 void Assembler::tbx(const VRegister& vd,
417 NEONTable(vd, vn, vm, NEON_TBX_4v);
2917 void Assembler::NEON3DifferentL(const VRegister& vd,
2922 VIXL_ASSERT((vn.Is1H() && vd.Is1S()) || (vn.Is1S() && vd.Is1D()) ||
2923 (vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
2924 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
2925 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
2927 if (vd.IsScalar()) {
2933 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
2937 void Assembler::NEON3DifferentW(const VRegister& vd,
2941 VIXL_ASSERT(AreSameFormat(vd, vn));
2942 VIXL_ASSERT((vm.Is8B() && vd.Is8H()) || (vm.Is4H() && vd.Is4S()) ||
2943 (vm.Is2S() && vd.Is2D()) || (vm.Is16B() && vd.Is8H()) ||
2944 (vm.Is8H() && vd.Is4S()) || (vm.Is4S() && vd.Is2D()));
2945 Emit(VFormat(vm) | vop | Rm(vm) | Rn(vn) | Rd(vd));
2949 void Assembler::NEON3DifferentHN(const VRegister& vd,
2954 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
2955 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
2956 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
2957 Emit(VFormat(vd) | vop | Rm(vm) | Rn(vn) | Rd(vd));
3003 void Assembler::FN(const VRegister& vd, \
3008 NEON3DifferentL(vd, vn, vm, OP); \
3015 V(addhn, NEON_ADDHN, vd.IsD()) \
3016 V(addhn2, NEON_ADDHN2, vd.IsQ()) \
3017 V(raddhn, NEON_RADDHN, vd.IsD()) \
3018 V(raddhn2, NEON_RADDHN2, vd.IsQ()) \
3019 V(subhn, NEON_SUBHN, vd.IsD()) \
3020 V(subhn2, NEON_SUBHN2, vd.IsQ()) \
3021 V(rsubhn, NEON_RSUBHN, vd.IsD()) \
3022 V(rsubhn2, NEON_RSUBHN2, vd.IsQ())
3026 void Assembler::FN(const VRegister& vd, \
3031 NEON3DifferentHN(vd, vn, vm, OP); \
3036 void Assembler::uaddw(const VRegister& vd,
3041 NEON3DifferentW(vd, vn, vm, NEON_UADDW);
3045 void Assembler::uaddw2(const VRegister& vd,
3050 NEON3DifferentW(vd, vn, vm, NEON_UADDW2);
3054 void Assembler::saddw(const VRegister& vd,
3059 NEON3DifferentW(vd, vn, vm, NEON_SADDW);
3063 void Assembler::saddw2(const VRegister& vd,
3068 NEON3DifferentW(vd, vn, vm, NEON_SADDW2);
3072 void Assembler::usubw(const VRegister& vd,
3077 NEON3DifferentW(vd, vn, vm, NEON_USUBW);
3081 void Assembler::usubw2(const VRegister& vd,
3086 NEON3DifferentW(vd, vn, vm, NEON_USUBW2);
3090 void Assembler::ssubw(const VRegister& vd,
3095 NEON3DifferentW(vd, vn, vm, NEON_SSUBW);
3099 void Assembler::ssubw2(const VRegister& vd,
3104 NEON3DifferentW(vd, vn, vm, NEON_SSUBW2);
3252 void Assembler::fmov(const VRegister& vd, double imm) {
3254 if (vd.IsScalar()) {
3255 VIXL_ASSERT(vd.Is1D());
3256 Emit(FMOV_d_imm | Rd(vd) | ImmFP64(imm));
3259 VIXL_ASSERT(vd.Is2D());
3263 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
3268 void Assembler::fmov(const VRegister& vd, float imm) {
3270 if (vd.IsScalar()) {
3271 VIXL_ASSERT(vd.Is1S());
3272 Emit(FMOV_s_imm | Rd(vd) | ImmFP32(imm));
3275 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
3277 Instr q = vd.Is4S() ? NEON_Q : 0;
3279 Emit(q | op | ImmNEONabcdefgh(encoded_imm) | NEONCmode(0xf) | Rd(vd));
3284 void Assembler::fmov(const VRegister& vd, Float16 imm) {
3286 if (vd.IsScalar()) {
3288 VIXL_ASSERT(vd.Is1H());
3289 Emit(FMOV_h_imm | Rd(vd) | ImmFP16(imm));
3292 VIXL_ASSERT(vd.Is4H() || vd.Is8H());
3293 Instr q = vd.Is8H() ? NEON_Q : 0;
3296 NEONCmode(0xf) | Rd(vd));
3321 void Assembler::fmov(const VRegister& vd, const Register& rn) {
3323 (vd.Is1D() && CPUHas(CPUFeatures::kNEON)));
3324 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3325 VIXL_ASSERT((vd.GetSizeInBits() == rn.GetSizeInBits()) || vd.Is1H());
3327 switch (vd.GetSizeInBits()) {
3338 Emit(op | Rd(vd) | Rn(rn));
3342 void Assembler::fmov(const VRegister& vd, const VRegister& vn) {
3344 if (vd.Is1H()) {
3347 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3348 VIXL_ASSERT(vd.IsSameFormat(vn));
3349 Emit(FPType(vd) | FMOV | Rd(vd) | Rn(vn));
3353 void Assembler::fmov(const VRegister& vd, int index, const Register& rn) {
3355 VIXL_ASSERT((index == 1) && vd.Is1D() && rn.IsX());
3357 Emit(FMOV_d1_x | Rd(vd) | Rn(rn));
3369 void Assembler::fmadd(const VRegister& vd,
3375 if (vd.Is1H()) {
3378 } else if (vd.Is1S()) {
3381 VIXL_ASSERT(vd.Is1D());
3384 FPDataProcessing3Source(vd, vn, vm, va, op);
3388 void Assembler::fmsub(const VRegister& vd,
3394 if (vd.Is1H()) {
3397 } else if (vd.Is1S()) {
3400 VIXL_ASSERT(vd.Is1D());
3403 FPDataProcessing3Source(vd, vn, vm, va, op);
3407 void Assembler::fnmadd(const VRegister& vd,
3413 if (vd.Is1H()) {
3416 } else if (vd.Is1S()) {
3419 VIXL_ASSERT(vd.Is1D());
3422 FPDataProcessing3Source(vd, vn, vm, va, op);
3426 void Assembler::fnmsub(const VRegister& vd,
3432 if (vd.Is1H()) {
3435 } else if (vd.Is1S()) {
3438 VIXL_ASSERT(vd.Is1D());
3441 FPDataProcessing3Source(vd, vn, vm, va, op);
3445 void Assembler::fnmul(const VRegister& vd,
3449 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm));
3451 if (vd.Is1H()) {
3454 } else if (vd.Is1S()) {
3457 VIXL_ASSERT(vd.Is1D());
3460 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3547 void Assembler::fcsel(const VRegister& vd,
3552 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3553 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3554 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3555 Emit(FPType(vd) | FCSEL | Rm(vm) | Cond(cond) | Rn(vn) | Rd(vd));
3559 void Assembler::fcvt(const VRegister& vd, const VRegister& vn) {
3563 if (vd.Is1D()) {
3566 } else if (vd.Is1S()) {
3570 VIXL_ASSERT(vd.Is1H());
3574 FPDataProcessing1Source(vd, vn, op);
3578 void Assembler::fcvtl(const VRegister& vd, const VRegister& vn) {
3580 VIXL_ASSERT((vd.Is4S() && vn.Is4H()) || (vd.Is2D() && vn.Is2S()));
3582 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3583 Emit(format | NEON_FCVTL | Rn(vn) | Rd(vd));
3587 void Assembler::fcvtl2(const VRegister& vd, const VRegister& vn) {
3589 VIXL_ASSERT((vd.Is4S() && vn.Is8H()) || (vd.Is2D() && vn.Is4S()));
3591 Instr format = vd.Is2D() ? (1 << NEONSize_offset) : 0;
3592 Emit(NEON_Q | format | NEON_FCVTL | Rn(vn) | Rd(vd));
3596 void Assembler::fcvtn(const VRegister& vd, const VRegister& vn) {
3598 VIXL_ASSERT((vn.Is4S() && vd.Is4H()) || (vn.Is2D() && vd.Is2S()));
3601 Emit(format | NEON_FCVTN | Rn(vn) | Rd(vd));
3605 void Assembler::fcvtn2(const VRegister& vd, const VRegister& vn) {
3607 VIXL_ASSERT((vn.Is4S() && vd.Is8H()) || (vn.Is2D() && vd.Is4S()));
3610 Emit(NEON_Q | format | NEON_FCVTN | Rn(vn) | Rd(vd));
3614 void Assembler::fcvtxn(const VRegister& vd, const VRegister& vn) {
3617 if (vd.IsScalar()) {
3618 VIXL_ASSERT(vd.Is1S() && vn.Is1D());
3619 Emit(format | NEON_FCVTXN_scalar | Rn(vn) | Rd(vd));
3621 VIXL_ASSERT(vd.Is2S() && vn.Is2D());
3622 Emit(format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3627 void Assembler::fcvtxn2(const VRegister& vd, const VRegister& vn) {
3629 VIXL_ASSERT(vd.Is4S() && vn.Is2D());
3631 Emit(NEON_Q | format | NEON_FCVTXN | Rn(vn) | Rd(vd));
3648 void Assembler::NEONFPConvertToInt(const VRegister& vd,
3652 VIXL_ASSERT((vd.Is1S() && vn.Is1S()) || (vd.Is1D() && vn.Is1D()));
3655 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
3659 void Assembler::NEONFP16ConvertToInt(const VRegister& vd,
3662 VIXL_ASSERT(AreSameFormat(vd, vn));
3669 Emit(op | Rn(vn) | Rd(vd));
3689 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3691 if (vd.IsLaneSizeH()) { \
3693 NEONFP16ConvertToInt(vd, vn, VEC_OP##_H); \
3695 NEONFPConvertToInt(vd, vn, VEC_OP); \
3716 void Assembler::fcvtzs(const VRegister& vd, const VRegister& vn, int fbits) {
3722 if (vd.IsLaneSizeH()) {
3723 NEONFP2RegMiscFP16(vd, vn, NEON_FCVTZS_H);
3725 NEONFP2RegMisc(vd, vn, NEON_FCVTZS);
3728 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3729 vd.Is1H() || vd.Is4H() || vd.Is8H());
3730 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZS_imm);
3749 void Assembler::fcvtzu(const VRegister& vd, const VRegister& vn, int fbits) {
3755 if (vd.IsLaneSizeH()) {
3756 NEONFP2RegMiscFP16(vd, vn, NEON_FCVTZU_H);
3758 NEONFP2RegMisc(vd, vn, NEON_FCVTZU);
3761 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3762 vd.Is1H() || vd.Is4H() || vd.Is8H());
3763 NEONShiftRightImmediate(vd, vn, fbits, NEON_FCVTZU_imm);
3767 void Assembler::ucvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3773 if (vd.IsLaneSizeH()) {
3774 NEONFP2RegMiscFP16(vd, vn, NEON_UCVTF_H);
3776 NEONFP2RegMisc(vd, vn, NEON_UCVTF);
3779 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3780 vd.Is1H() || vd.Is4H() || vd.Is8H());
3781 NEONShiftRightImmediate(vd, vn, fbits, NEON_UCVTF_imm);
3785 void Assembler::scvtf(const VRegister& vd, const VRegister& vn, int fbits) {
3791 if (vd.IsLaneSizeH()) {
3792 NEONFP2RegMiscFP16(vd, vn, NEON_SCVTF_H);
3794 NEONFP2RegMisc(vd, vn, NEON_SCVTF);
3797 VIXL_ASSERT(vd.Is1D() || vd.Is1S() || vd.Is2D() || vd.Is2S() || vd.Is4S() ||
3798 vd.Is1H() || vd.Is4H() || vd.Is8H());
3799 NEONShiftRightImmediate(vd, vn, fbits, NEON_SCVTF_imm);
3804 void Assembler::scvtf(const VRegister& vd, const Register& rn, int fbits) {
3806 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3807 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3810 Emit(SF(rn) | FPType(vd) | SCVTF | Rn(rn) | Rd(vd));
3812 Emit(SF(rn) | FPType(vd) | SCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3813 Rd(vd));
3818 void Assembler::ucvtf(const VRegister& vd, const Register& rn, int fbits) {
3820 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kFPHalf));
3821 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
3824 Emit(SF(rn) | FPType(vd) | UCVTF | Rn(rn) | Rd(vd));
3826 Emit(SF(rn) | FPType(vd) | UCVTF_fixed | FPScale(64 - fbits) | Rn(rn) |
3827 Rd(vd));
3832 void Assembler::NEON3Same(const VRegister& vd,
3836 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3837 VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
3840 if (vd.IsScalar()) {
3842 format = SFormat(vd);
3844 format = VFormat(vd);
3847 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
3851 void Assembler::NEONFP3Same(const VRegister& vd,
3855 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3856 Emit(FPFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
3860 void Assembler::NEON3SameFP16(const VRegister& vd,
3864 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
3865 VIXL_ASSERT(vd.GetLaneSizeInBytes() == kHRegSizeInBytes);
3866 if (vd.Is8H()) op |= NEON_Q;
3867 Emit(op | Rm(vm) | Rn(vn) | Rd(vd));
3888 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3891 if (vd.IsScalar()) { \
3892 if (vd.Is1H()) { \
3904 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
3909 VIXL_ASSERT(vd.Is4H() || vd.Is8H() || vd.Is2S() || vd.Is2D() || \
3910 vd.Is4S()); \
3911 if (vd.IsLaneSizeH()) { \
3914 if (vd.Is8H()) { \
3921 if (vd.IsLaneSizeH()) { \
3922 NEONFP2RegMiscFP16(vd, vn, op); \
3924 NEONFP2RegMisc(vd, vn, op); \
3939 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
3942 if (vd.IsScalar()) { \
3943 VIXL_ASSERT(vd.Is1S() || vd.Is1D()); \
3947 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
3950 NEONFP2RegMisc(vd, vn, op); \
3955 void Assembler::NEONFP2RegMiscFP16(const VRegister& vd,
3958 VIXL_ASSERT(AreSameFormat(vd, vn));
3959 Emit(op | Rn(vn) | Rd(vd));
3963 void Assembler::NEONFP2RegMisc(const VRegister& vd,
3966 VIXL_ASSERT(AreSameFormat(vd, vn));
3967 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
3971 void Assembler::NEON2RegMisc(const VRegister& vd,
3975 VIXL_ASSERT(AreSameFormat(vd, vn));
3980 if (vd.IsScalar()) {
3982 format = SFormat(vd);
3984 format = VFormat(vd);
3987 Emit(format | op | Rn(vn) | Rd(vd));
3991 void Assembler::cmeq(const VRegister& vd, const VRegister& vn, int value) {
3993 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
3994 NEON2RegMisc(vd, vn, NEON_CMEQ_zero, value);
3998 void Assembler::cmge(const VRegister& vd, const VRegister& vn, int value) {
4000 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4001 NEON2RegMisc(vd, vn, NEON_CMGE_zero, value);
4005 void Assembler::cmgt(const VRegister& vd, const VRegister& vn, int value) {
4007 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4008 NEON2RegMisc(vd, vn, NEON_CMGT_zero, value);
4012 void Assembler::cmle(const VRegister& vd, const VRegister& vn, int value) {
4013 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4015 NEON2RegMisc(vd, vn, NEON_CMLE_zero, value);
4019 void Assembler::cmlt(const VRegister& vd, const VRegister& vn, int value) {
4021 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4022 NEON2RegMisc(vd, vn, NEON_CMLT_zero, value);
4026 void Assembler::shll(const VRegister& vd, const VRegister& vn, int shift) {
4029 VIXL_ASSERT((vd.Is8H() && vn.Is8B() && shift == 8) ||
4030 (vd.Is4S() && vn.Is4H() && shift == 16) ||
4031 (vd.Is2D() && vn.Is2S() && shift == 32));
4032 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
4036 void Assembler::shll2(const VRegister& vd, const VRegister& vn, int shift) {
4039 VIXL_ASSERT((vd.Is8H() && vn.Is16B() && shift == 8) ||
4040 (vd.Is4S() && vn.Is8H() && shift == 16) ||
4041 (vd.Is2D() && vn.Is4S() && shift == 32));
4042 Emit(VFormat(vn) | NEON_SHLL | Rn(vn) | Rd(vd));
4046 void Assembler::NEONFP2RegMisc(const VRegister& vd,
4050 VIXL_ASSERT(AreSameFormat(vd, vn));
4055 if (vd.IsScalar()) {
4056 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4059 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S());
4062 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
4066 void Assembler::NEONFP2RegMiscFP16(const VRegister& vd,
4070 VIXL_ASSERT(AreSameFormat(vd, vn));
4075 if (vd.IsScalar()) {
4076 VIXL_ASSERT(vd.Is1H());
4079 VIXL_ASSERT(vd.Is4H() || vd.Is8H());
4080 if (vd.Is8H()) {
4085 Emit(op | Rn(vn) | Rd(vd));
4089 void Assembler::fcmeq(const VRegister& vd, const VRegister& vn, double value) {
4091 if (vd.IsLaneSizeH()) {
4093 NEONFP2RegMiscFP16(vd, vn, NEON_FCMEQ_H_zero, value);
4095 NEONFP2RegMisc(vd, vn, NEON_FCMEQ_zero, value);
4100 void Assembler::fcmge(const VRegister& vd, const VRegister& vn, double value) {
4102 if (vd.IsLaneSizeH()) {
4104 NEONFP2RegMiscFP16(vd, vn, NEON_FCMGE_H_zero, value);
4106 NEONFP2RegMisc(vd, vn, NEON_FCMGE_zero, value);
4111 void Assembler::fcmgt(const VRegister& vd, const VRegister& vn, double value) {
4113 if (vd.IsLaneSizeH()) {
4115 NEONFP2RegMiscFP16(vd, vn, NEON_FCMGT_H_zero, value);
4117 NEONFP2RegMisc(vd, vn, NEON_FCMGT_zero, value);
4122 void Assembler::fcmle(const VRegister& vd, const VRegister& vn, double value) {
4124 if (vd.IsLaneSizeH()) {
4126 NEONFP2RegMiscFP16(vd, vn, NEON_FCMLE_H_zero, value);
4128 NEONFP2RegMisc(vd, vn, NEON_FCMLE_zero, value);
4133 void Assembler::fcmlt(const VRegister& vd, const VRegister& vn, double value) {
4135 if (vd.IsLaneSizeH()) {
4137 NEONFP2RegMiscFP16(vd, vn, NEON_FCMLT_H_zero, value);
4139 NEONFP2RegMisc(vd, vn, NEON_FCMLT_zero, value);
4144 void Assembler::frecpx(const VRegister& vd, const VRegister& vn) {
4146 VIXL_ASSERT(vd.IsScalar());
4147 VIXL_ASSERT(AreSameFormat(vd, vn));
4149 if (vd.Is1H()) {
4153 VIXL_ASSERT(vd.Is1S() || vd.Is1D());
4156 Emit(FPFormat(vd) | op | Rn(vn) | Rd(vd));
4162 V(add, NEON_ADD, vd.IsVector() || vd.Is1D()) \
4163 V(addp, NEON_ADDP, vd.IsVector() || vd.Is1D()) \
4164 V(sub, NEON_SUB, vd.IsVector() || vd.Is1D()) \
4165 V(cmeq, NEON_CMEQ, vd.IsVector() || vd.Is1D()) \
4166 V(cmge, NEON_CMGE, vd.IsVector() || vd.Is1D()) \
4167 V(cmgt, NEON_CMGT, vd.IsVector() || vd.Is1D()) \
4168 V(cmhi, NEON_CMHI, vd.IsVector() || vd.Is1D()) \
4169 V(cmhs, NEON_CMHS, vd.IsVector() || vd.Is1D()) \
4170 V(cmtst, NEON_CMTST, vd.IsVector() || vd.Is1D()) \
4171 V(sshl, NEON_SSHL, vd.IsVector() || vd.Is1D()) \
4172 V(ushl, NEON_USHL, vd.IsVector() || vd.Is1D()) \
4173 V(srshl, NEON_SRSHL, vd.IsVector() || vd.Is1D()) \
4174 V(urshl, NEON_URSHL, vd.IsVector() || vd.Is1D()) \
4175 V(sqdmulh, NEON_SQDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
4176 V(sqrdmulh, NEON_SQRDMULH, vd.IsLaneSizeH() || vd.IsLaneSizeS()) \
4177 V(shadd, NEON_SHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4178 V(uhadd, NEON_UHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4179 V(srhadd, NEON_SRHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4180 V(urhadd, NEON_URHADD, vd.IsVector() && !vd.IsLaneSizeD()) \
4181 V(shsub, NEON_SHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
4182 V(uhsub, NEON_UHSUB, vd.IsVector() && !vd.IsLaneSizeD()) \
4183 V(smax, NEON_SMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
4184 V(smaxp, NEON_SMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
4185 V(smin, NEON_SMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
4186 V(sminp, NEON_SMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
4187 V(umax, NEON_UMAX, vd.IsVector() && !vd.IsLaneSizeD()) \
4188 V(umaxp, NEON_UMAXP, vd.IsVector() && !vd.IsLaneSizeD()) \
4189 V(umin, NEON_UMIN, vd.IsVector() && !vd.IsLaneSizeD()) \
4190 V(uminp, NEON_UMINP, vd.IsVector() && !vd.IsLaneSizeD()) \
4191 V(saba, NEON_SABA, vd.IsVector() && !vd.IsLaneSizeD()) \
4192 V(sabd, NEON_SABD, vd.IsVector() && !vd.IsLaneSizeD()) \
4193 V(uaba, NEON_UABA, vd.IsVector() && !vd.IsLaneSizeD()) \
4194 V(uabd, NEON_UABD, vd.IsVector() && !vd.IsLaneSizeD()) \
4195 V(mla, NEON_MLA, vd.IsVector() && !vd.IsLaneSizeD()) \
4196 V(mls, NEON_MLS, vd.IsVector() && !vd.IsLaneSizeD()) \
4197 V(mul, NEON_MUL, vd.IsVector() && !vd.IsLaneSizeD()) \
4198 V(and_, NEON_AND, vd.Is8B() || vd.Is16B()) \
4199 V(orr, NEON_ORR, vd.Is8B() || vd.Is16B()) \
4200 V(orn, NEON_ORN, vd.Is8B() || vd.Is16B()) \
4201 V(eor, NEON_EOR, vd.Is8B() || vd.Is16B()) \
4202 V(bic, NEON_BIC, vd.Is8B() || vd.Is16B()) \
4203 V(bit, NEON_BIT, vd.Is8B() || vd.Is16B()) \
4204 V(bif, NEON_BIF, vd.Is8B() || vd.Is16B()) \
4205 V(bsl, NEON_BSL, vd.Is8B() || vd.Is16B()) \
4206 V(pmul, NEON_PMUL, vd.Is8B() || vd.Is16B()) \
4218 void Assembler::FN(const VRegister& vd, \
4223 NEON3Same(vd, vn, vm, OP); \
4260 void Assembler::FN(const VRegister& vd, \
4266 if ((SCA_OP != 0) && vd.IsScalar()) { \
4267 if ((SCA_OP_H != 0) && vd.Is1H()) { \
4272 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D()); \
4275 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
4276 } else if (vd.Is1H()) { \
4283 VIXL_ASSERT(vd.IsVector()); \
4284 if (vd.Is4H() || vd.Is8H()) { \
4289 VIXL_ASSERT(vd.Is2S() || vd.Is2D() || vd.Is4S()); \
4294 NEON3SameFP16(vd, vn, vm, op); \
4296 NEONFP3Same(vd, vn, vm, op); \
4312 void Assembler::FN(const VRegister& vd, \
4319 VIXL_ASSERT((vd.Is2S() && vn.Is2H() && vm.Is2H()) || \
4320 (vd.Is4S() && vn.Is4H() && vm.Is4H())); \
4321 Emit(FPFormat(vd) | VEC_OP | Rm(vm) | Rn(vn) | Rd(vd)); \
4327 void Assembler::addp(const VRegister& vd, const VRegister& vn) {
4329 VIXL_ASSERT((vd.Is1D() && vn.Is2D()));
4330 Emit(SFormat(vd) | NEON_ADDP_scalar | Rn(vn) | Rd(vd));
4334 void Assembler::sqrdmlah(const VRegister& vd,
4338 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4339 VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
4342 if (vd.IsScalar()) {
4344 format = SFormat(vd);
4346 format = VFormat(vd);
4349 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
4353 void Assembler::sqrdmlsh(const VRegister& vd,
4357 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4358 VIXL_ASSERT(vd.IsVector() || !vd.IsQ());
4361 if (vd.IsScalar()) {
4363 format = SFormat(vd);
4365 format = VFormat(vd);
4368 Emit(format | op | Rm(vm) | Rn(vn) | Rd(vd));
4372 void Assembler::sdot(const VRegister& vd,
4377 VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
4379 Emit(VFormat(vd) | NEON_SDOT | Rm(vm) | Rn(vn) | Rd(vd));
4383 void Assembler::udot(const VRegister& vd,
4388 VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
4390 Emit(VFormat(vd) | NEON_UDOT | Rm(vm) | Rn(vn) | Rd(vd));
4393 void Assembler::usdot(const VRegister& vd,
4398 VIXL_ASSERT((vd.Is2S() && vn.Is8B()) || (vd.Is4S() && vn.Is16B()));
4400 Emit(VFormat(vd) | 0x0e809c00 | Rm(vm) | Rn(vn) | Rd(vd));
4403 void Assembler::faddp(const VRegister& vd, const VRegister& vn) {
4405 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4406 (vd.Is1H() && vn.Is2H()));
4407 if (vd.Is1H()) {
4409 Emit(NEON_FADDP_h_scalar | Rn(vn) | Rd(vd));
4411 Emit(FPFormat(vd) | NEON_FADDP_scalar | Rn(vn) | Rd(vd));
4416 void Assembler::fmaxp(const VRegister& vd, const VRegister& vn) {
4418 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4419 (vd.Is1H() && vn.Is2H()));
4420 if (vd.Is1H()) {
4422 Emit(NEON_FMAXP_h_scalar | Rn(vn) | Rd(vd));
4424 Emit(FPFormat(vd) | NEON_FMAXP_scalar | Rn(vn) | Rd(vd));
4429 void Assembler::fminp(const VRegister& vd, const VRegister& vn) {
4431 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4432 (vd.Is1H() && vn.Is2H()));
4433 if (vd.Is1H()) {
4435 Emit(NEON_FMINP_h_scalar | Rn(vn) | Rd(vd));
4437 Emit(FPFormat(vd) | NEON_FMINP_scalar | Rn(vn) | Rd(vd));
4442 void Assembler::fmaxnmp(const VRegister& vd, const VRegister& vn) {
4444 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4445 (vd.Is1H() && vn.Is2H()));
4446 if (vd.Is1H()) {
4448 Emit(NEON_FMAXNMP_h_scalar | Rn(vn) | Rd(vd));
4450 Emit(FPFormat(vd) | NEON_FMAXNMP_scalar | Rn(vn) | Rd(vd));
4455 void Assembler::fminnmp(const VRegister& vd, const VRegister& vn) {
4457 VIXL_ASSERT((vd.Is1S() && vn.Is2S()) || (vd.Is1D() && vn.Is2D()) ||
4458 (vd.Is1H() && vn.Is2H()));
4459 if (vd.Is1H()) {
4461 Emit(NEON_FMINNMP_h_scalar | Rn(vn) | Rd(vd));
4463 Emit(FPFormat(vd) | NEON_FMINNMP_scalar | Rn(vn) | Rd(vd));
4469 void Assembler::fcmla(const VRegister& vd,
4475 VIXL_ASSERT(vd.IsVector() && AreSameFormat(vd, vn));
4476 VIXL_ASSERT((vm.IsH() && (vd.Is8H() || vd.Is4H())) ||
4477 (vm.IsS() && vd.Is4S()));
4478 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4479 int index_num_bits = vd.Is4S() ? 1 : 2;
4480 Emit(VFormat(vd) | Rm(vm) | NEON_FCMLA_byelement |
4482 Rd(vd));
4486 void Assembler::fcmla(const VRegister& vd,
4491 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4492 VIXL_ASSERT(vd.IsVector() && !vd.IsLaneSizeB());
4493 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4494 Emit(VFormat(vd) | Rm(vm) | NEON_FCMLA | ImmRotFcmlaVec(rot) | Rn(vn) |
4495 Rd(vd));
4500 void Assembler::fcadd(const VRegister& vd,
4505 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
4506 VIXL_ASSERT(vd.IsVector() && !vd.IsLaneSizeB());
4507 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf));
4508 Emit(VFormat(vd) | Rm(vm) | NEON_FCADD | ImmRotFcadd(rot) | Rn(vn) | Rd(vd));
4512 void Assembler::orr(const VRegister& vd, const int imm8, const int left_shift) {
4514 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_ORR);
4518 void Assembler::mov(const VRegister& vd, const VRegister& vn) {
4520 VIXL_ASSERT(AreSameFormat(vd, vn));
4521 if (vd.IsD()) {
4522 orr(vd.V8B(), vn.V8B(), vn.V8B());
4524 VIXL_ASSERT(vd.IsQ());
4525 orr(vd.V16B(), vn.V16B(), vn.V16B());
4530 void Assembler::bic(const VRegister& vd, const int imm8, const int left_shift) {
4532 NEONModifiedImmShiftLsl(vd, imm8, left_shift, NEONModifiedImmediate_BIC);
4536 void Assembler::movi(const VRegister& vd,
4542 if (vd.Is2D() || vd.Is1D()) {
4552 int q = vd.Is2D() ? NEON_Q : 0;
4554 ImmNEONabcdefgh(imm8) | NEONCmode(0xe) | Rd(vd));
4557 NEONModifiedImmShiftLsl(vd,
4563 NEONModifiedImmShiftMsl(vd,
4571 void Assembler::mvn(const VRegister& vd, const VRegister& vn) {
4573 VIXL_ASSERT(AreSameFormat(vd, vn));
4574 if (vd.IsD()) {
4575 not_(vd.V8B(), vn.V8B());
4577 VIXL_ASSERT(vd.IsQ());
4578 not_(vd.V16B(), vn.V16B());
4583 void Assembler::mvni(const VRegister& vd,
4590 NEONModifiedImmShiftLsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
4592 NEONModifiedImmShiftMsl(vd, imm8, shift_amount, NEONModifiedImmediate_MVNI);
4597 void Assembler::NEONFPByElement(const VRegister& vd,
4603 VIXL_ASSERT(AreSameFormat(vd, vn));
4604 VIXL_ASSERT((vd.Is2S() && vm.Is1S()) || (vd.Is4S() && vm.Is1S()) ||
4605 (vd.Is1S() && vm.Is1S()) || (vd.Is2D() && vm.Is1D()) ||
4606 (vd.Is1D() && vm.Is1D()) || (vd.Is4H() && vm.Is1H()) ||
4607 (vd.Is8H() && vm.Is1H()) || (vd.Is1H() && vm.Is1H()));
4622 if (vd.IsScalar()) {
4627 op |= FPFormat(vd);
4628 } else if (vd.Is8H()) {
4632 Emit(op | ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4636 void Assembler::NEONByElement(const VRegister& vd,
4641 VIXL_ASSERT(AreSameFormat(vd, vn));
4642 VIXL_ASSERT((vd.Is4H() && vm.Is1H()) || (vd.Is8H() && vm.Is1H()) ||
4643 (vd.Is1H() && vm.Is1H()) || (vd.Is2S() && vm.Is1S()) ||
4644 (vd.Is4S() && vm.Is1S()) || (vd.Is1S() && vm.Is1S()));
4650 if (vd.IsScalar()) {
4657 Rd(vd));
4661 void Assembler::NEONByElementL(const VRegister& vd,
4666 VIXL_ASSERT((vd.Is4S() && vn.Is4H() && vm.Is1H()) ||
4667 (vd.Is4S() && vn.Is8H() && vm.Is1H()) ||
4668 (vd.Is1S() && vn.Is1H() && vm.Is1H()) ||
4669 (vd.Is2D() && vn.Is2S() && vm.Is1S()) ||
4670 (vd.Is2D() && vn.Is4S() && vm.Is1S()) ||
4671 (vd.Is1D() && vn.Is1S() && vm.Is1S()));
4678 if (vd.IsScalar()) {
4685 Rd(vd));
4689 void Assembler::sdot(const VRegister& vd,
4694 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4695 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4698 Emit(VFormat(vd) | NEON_SDOT_byelement |
4699 ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4703 void Assembler::udot(const VRegister& vd,
4708 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4709 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4712 Emit(VFormat(vd) | NEON_UDOT_byelement |
4713 ImmNEONHLM(vm_index, index_num_bits) | Rm(vm) | Rn(vn) | Rd(vd));
4716 void Assembler::sudot(const VRegister& vd,
4721 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4722 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4723 int q = vd.Is4S() ? (1U << NEONQ_offset) : 0;
4726 Rd(vd));
4730 void Assembler::usdot(const VRegister& vd,
4735 VIXL_ASSERT((vd.Is2S() && vn.Is8B() && vm.Is1S4B()) ||
4736 (vd.Is4S() && vn.Is16B() && vm.Is1S4B()));
4737 int q = vd.Is4S() ? (1U << NEONQ_offset) : 0;
4740 Rd(vd));
4753 void Assembler::FN(const VRegister& vd, \
4759 NEONByElement(vd, vn, vm, vm_index, OP); \
4772 void Assembler::FN(const VRegister& vd, \
4777 NEONByElement(vd, vn, vm, vm_index, OP); \
4792 void Assembler::FN(const VRegister& vd, \
4797 if (vd.IsLaneSizeH()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
4798 NEONFPByElement(vd, vn, vm, vm_index, OP, OP_H); \
4828 void Assembler::FN(const VRegister& vd, \
4834 NEONByElementL(vd, vn, vm, vm_index, OP); \
4850 void Assembler::FN(const VRegister& vd, \
4858 VIXL_ASSERT((vd.Is2S() && vn.Is2H()) || (vd.Is4S() && vn.Is4H())); \
4863 Emit(FPFormat(vd) | OP | Rd(vd) | Rn(vn) | Rm(vm) | \
4869 void Assembler::suqadd(const VRegister& vd, const VRegister& vn) {
4871 NEON2RegMisc(vd, vn, NEON_SUQADD);
4875 void Assembler::usqadd(const VRegister& vd, const VRegister& vn) {
4877 NEON2RegMisc(vd, vn, NEON_USQADD);
4881 void Assembler::abs(const VRegister& vd, const VRegister& vn) {
4883 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4884 NEON2RegMisc(vd, vn, NEON_ABS);
4888 void Assembler::sqabs(const VRegister& vd, const VRegister& vn) {
4890 NEON2RegMisc(vd, vn, NEON_SQABS);
4894 void Assembler::neg(const VRegister& vd, const VRegister& vn) {
4896 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
4897 NEON2RegMisc(vd, vn, NEON_NEG);
4901 void Assembler::sqneg(const VRegister& vd, const VRegister& vn) {
4903 NEON2RegMisc(vd, vn, NEON_SQNEG);
4907 void Assembler::NEONXtn(const VRegister& vd,
4912 if (vd.IsScalar()) {
4913 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
4914 (vd.Is1S() && vn.Is1D()));
4916 format = SFormat(vd);
4918 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
4919 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
4920 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
4921 format = VFormat(vd);
4923 Emit(format | op | Rn(vn) | Rd(vd));
4927 void Assembler::xtn(const VRegister& vd, const VRegister& vn) {
4929 VIXL_ASSERT(vd.IsVector() && vd.IsD());
4930 NEONXtn(vd, vn, NEON_XTN);
4934 void Assembler::xtn2(const VRegister& vd, const VRegister& vn) {
4936 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4937 NEONXtn(vd, vn, NEON_XTN);
4941 void Assembler::sqxtn(const VRegister& vd, const VRegister& vn) {
4943 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4944 NEONXtn(vd, vn, NEON_SQXTN);
4948 void Assembler::sqxtn2(const VRegister& vd, const VRegister& vn) {
4950 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4951 NEONXtn(vd, vn, NEON_SQXTN);
4955 void Assembler::sqxtun(const VRegister& vd, const VRegister& vn) {
4957 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4958 NEONXtn(vd, vn, NEON_SQXTUN);
4962 void Assembler::sqxtun2(const VRegister& vd, const VRegister& vn) {
4964 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4965 NEONXtn(vd, vn, NEON_SQXTUN);
4969 void Assembler::uqxtn(const VRegister& vd, const VRegister& vn) {
4971 VIXL_ASSERT(vd.IsScalar() || vd.IsD());
4972 NEONXtn(vd, vn, NEON_UQXTN);
4976 void Assembler::uqxtn2(const VRegister& vd, const VRegister& vn) {
4978 VIXL_ASSERT(vd.IsVector() && vd.IsQ());
4979 NEONXtn(vd, vn, NEON_UQXTN);
4984 void Assembler::not_(const VRegister& vd, const VRegister& vn) {
4986 VIXL_ASSERT(AreSameFormat(vd, vn));
4987 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
4988 Emit(VFormat(vd) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
4992 void Assembler::rbit(const VRegister& vd, const VRegister& vn) {
4994 VIXL_ASSERT(AreSameFormat(vd, vn));
4995 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
4996 Emit(VFormat(vn) | (1 << NEONSize_offset) | NEON_RBIT_NOT | Rn(vn) | Rd(vd));
5000 void Assembler::ext(const VRegister& vd,
5005 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
5006 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5007 VIXL_ASSERT((0 <= index) && (index < vd.GetLanes()));
5008 Emit(VFormat(vd) | NEON_EXT | Rm(vm) | ImmNEONExt(index) | Rn(vn) | Rd(vd));
5012 void Assembler::dup(const VRegister& vd, const VRegister& vn, int vn_index) {
5036 if (vd.IsScalar()) {
5040 VIXL_ASSERT(!vd.Is1D());
5041 q = vd.IsD() ? 0 : NEON_Q;
5045 Rd(vd));
5049 void Assembler::mov(const VRegister& vd, const VRegister& vn, int vn_index) {
5051 VIXL_ASSERT(vd.IsScalar());
5052 dup(vd, vn, vn_index);
5056 void Assembler::dup(const VRegister& vd, const Register& rn) {
5058 VIXL_ASSERT(!vd.Is1D());
5059 VIXL_ASSERT(vd.Is2D() == rn.IsX());
5060 int q = vd.IsD() ? 0 : NEON_Q;
5061 Emit(q | NEON_DUP_GENERAL | ImmNEON5(VFormat(vd), 0) | Rn(rn) | Rd(vd));
5065 void Assembler::ins(const VRegister& vd,
5070 VIXL_ASSERT(AreSameFormat(vd, vn));
5071 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
5073 int lane_size = vd.GetLaneSizeInBytes();
5098 ImmNEON4(format, vn_index) | Rn(vn) | Rd(vd));
5102 void Assembler::mov(const VRegister& vd,
5107 ins(vd, vd_index, vn, vn_index);
5111 void Assembler::ins(const VRegister& vd, int vd_index, const Register& rn) {
5113 // We support vd arguments of the form vd.VxT() or vd.T(), where x is the
5115 int lane_size = vd.GetLaneSizeInBytes();
5140 Emit(NEON_INS_GENERAL | ImmNEON5(format, vd_index) | Rn(rn) | Rd(vd));
5144 void Assembler::mov(const VRegister& vd, int vd_index, const Register& rn) {
5146 ins(vd, vd_index, rn);
5221 void Assembler::cls(const VRegister& vd, const VRegister& vn) {
5223 VIXL_ASSERT(AreSameFormat(vd, vn));
5224 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
5225 Emit(VFormat(vn) | NEON_CLS | Rn(vn) | Rd(vd));
5229 void Assembler::clz(const VRegister& vd, const VRegister& vn) {
5231 VIXL_ASSERT(AreSameFormat(vd, vn));
5232 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
5233 Emit(VFormat(vn) | NEON_CLZ | Rn(vn) | Rd(vd));
5237 void Assembler::cnt(const VRegister& vd, const VRegister& vn) {
5239 VIXL_ASSERT(AreSameFormat(vd, vn));
5240 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5241 Emit(VFormat(vn) | NEON_CNT | Rn(vn) | Rd(vd));
5245 void Assembler::rev16(const VRegister& vd, const VRegister& vn) {
5247 VIXL_ASSERT(AreSameFormat(vd, vn));
5248 VIXL_ASSERT(vd.Is8B() || vd.Is16B());
5249 Emit(VFormat(vn) | NEON_REV16 | Rn(vn) | Rd(vd));
5253 void Assembler::rev32(const VRegister& vd, const VRegister& vn) {
5255 VIXL_ASSERT(AreSameFormat(vd, vn));
5256 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H());
5257 Emit(VFormat(vn) | NEON_REV32 | Rn(vn) | Rd(vd));
5261 void Assembler::rev64(const VRegister& vd, const VRegister& vn) {
5263 VIXL_ASSERT(AreSameFormat(vd, vn));
5264 VIXL_ASSERT(!vd.Is1D() && !vd.Is2D());
5265 Emit(VFormat(vn) | NEON_REV64 | Rn(vn) | Rd(vd));
5269 void Assembler::ursqrte(const VRegister& vd, const VRegister& vn) {
5271 VIXL_ASSERT(AreSameFormat(vd, vn));
5272 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
5273 Emit(VFormat(vn) | NEON_URSQRTE | Rn(vn) | Rd(vd));
5277 void Assembler::urecpe(const VRegister& vd, const VRegister& vn) {
5279 VIXL_ASSERT(AreSameFormat(vd, vn));
5280 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
5281 Emit(VFormat(vn) | NEON_URECPE | Rn(vn) | Rd(vd));
5285 void Assembler::NEONAddlp(const VRegister& vd,
5292 VIXL_ASSERT((vn.Is8B() && vd.Is4H()) || (vn.Is4H() && vd.Is2S()) ||
5293 (vn.Is2S() && vd.Is1D()) || (vn.Is16B() && vd.Is8H()) ||
5294 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
5295 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
5299 void Assembler::saddlp(const VRegister& vd, const VRegister& vn) {
5301 NEONAddlp(vd, vn, NEON_SADDLP);
5305 void Assembler::uaddlp(const VRegister& vd, const VRegister& vn) {
5307 NEONAddlp(vd, vn, NEON_UADDLP);
5311 void Assembler::sadalp(const VRegister& vd, const VRegister& vn) {
5313 NEONAddlp(vd, vn, NEON_SADALP);
5317 void Assembler::uadalp(const VRegister& vd, const VRegister& vn) {
5319 NEONAddlp(vd, vn, NEON_UADALP);
5323 void Assembler::NEONAcrossLanesL(const VRegister& vd,
5326 VIXL_ASSERT((vn.Is8B() && vd.Is1H()) || (vn.Is16B() && vd.Is1H()) ||
5327 (vn.Is4H() && vd.Is1S()) || (vn.Is8H() && vd.Is1S()) ||
5328 (vn.Is4S() && vd.Is1D()));
5329 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
5333 void Assembler::saddlv(const VRegister& vd, const VRegister& vn) {
5335 NEONAcrossLanesL(vd, vn, NEON_SADDLV);
5339 void Assembler::uaddlv(const VRegister& vd, const VRegister& vn) {
5341 NEONAcrossLanesL(vd, vn, NEON_UADDLV);
5345 void Assembler::NEONAcrossLanes(const VRegister& vd,
5349 VIXL_ASSERT((vn.Is8B() && vd.Is1B()) || (vn.Is16B() && vd.Is1B()) ||
5350 (vn.Is4H() && vd.Is1H()) || (vn.Is8H() && vd.Is1H()) ||
5351 (vn.Is4S() && vd.Is1S()));
5353 if (vd.Is1H()) {
5359 Emit(vop | Rn(vn) | Rd(vd));
5361 Emit(FPFormat(vn) | op | Rn(vn) | Rd(vd));
5364 Emit(VFormat(vn) | op | Rn(vn) | Rd(vd));
5378 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
5380 NEONAcrossLanes(vd, vn, OP, 0); \
5395 void Assembler::FN(const VRegister& vd, const VRegister& vn) { \
5397 if (vd.Is1H()) VIXL_ASSERT(CPUHas(CPUFeatures::kNEONHalf)); \
5398 VIXL_ASSERT(vd.Is1S() || vd.Is1H()); \
5399 NEONAcrossLanes(vd, vn, OP, OP_H); \
5405 void Assembler::NEONPerm(const VRegister& vd,
5409 VIXL_ASSERT(AreSameFormat(vd, vn, vm));
5410 VIXL_ASSERT(!vd.Is1D());
5411 Emit(VFormat(vd) | op | Rm(vm) | Rn(vn) | Rd(vd));
5415 void Assembler::trn1(const VRegister& vd,
5419 NEONPerm(vd, vn, vm, NEON_TRN1);
5423 void Assembler::trn2(const VRegister& vd,
5427 NEONPerm(vd, vn, vm, NEON_TRN2);
5431 void Assembler::uzp1(const VRegister& vd,
5435 NEONPerm(vd, vn, vm, NEON_UZP1);
5439 void Assembler::uzp2(const VRegister& vd,
5443 NEONPerm(vd, vn, vm, NEON_UZP2);
5447 void Assembler::zip1(const VRegister& vd,
5451 NEONPerm(vd, vn, vm, NEON_ZIP1);
5455 void Assembler::zip2(const VRegister& vd,
5459 NEONPerm(vd, vn, vm, NEON_ZIP2);
5463 void Assembler::NEONShiftImmediate(const VRegister& vd,
5467 VIXL_ASSERT(AreSameFormat(vd, vn));
5473 q = vd.IsD() ? 0 : NEON_Q;
5476 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
5480 void Assembler::NEONShiftLeftImmediate(const VRegister& vd,
5486 NEONShiftImmediate(vd, vn, op, (lane_size_in_bits + shift) << 16);
5490 void Assembler::NEONShiftRightImmediate(const VRegister& vd,
5496 NEONShiftImmediate(vd, vn, op, ((2 * lane_size_in_bits) - shift) << 16);
5500 void Assembler::NEONShiftImmediateL(const VRegister& vd,
5508 VIXL_ASSERT((vn.Is8B() && vd.Is8H()) || (vn.Is4H() && vd.Is4S()) ||
5509 (vn.Is2S() && vd.Is2D()) || (vn.Is16B() && vd.Is8H()) ||
5510 (vn.Is8H() && vd.Is4S()) || (vn.Is4S() && vd.Is2D()));
5513 Emit(q | op | immh_immb | Rn(vn) | Rd(vd));
5517 void Assembler::NEONShiftImmediateN(const VRegister& vd,
5522 int lane_size_in_bits = vd.GetLaneSizeInBits();
5527 VIXL_ASSERT((vd.Is1B() && vn.Is1H()) || (vd.Is1H() && vn.Is1S()) ||
5528 (vd.Is1S() && vn.Is1D()));
5532 VIXL_ASSERT((vd.Is8B() && vn.Is8H()) || (vd.Is4H() && vn.Is4S()) ||
5533 (vd.Is2S() && vn.Is2D()) || (vd.Is16B() && vn.Is8H()) ||
5534 (vd.Is8H() && vn.Is4S()) || (vd.Is4S() && vn.Is2D()));
5536 q = vd.IsD() ? 0 : NEON_Q;
5538 Emit(q | op | scalar | immh_immb | Rn(vn) | Rd(vd));
5542 void Assembler::shl(const VRegister& vd, const VRegister& vn, int shift) {
5544 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5545 NEONShiftLeftImmediate(vd, vn, shift, NEON_SHL);
5549 void Assembler::sli(const VRegister& vd, const VRegister& vn, int shift) {
5551 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5552 NEONShiftLeftImmediate(vd, vn, shift, NEON_SLI);
5556 void Assembler::sqshl(const VRegister& vd, const VRegister& vn, int shift) {
5558 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHL_imm);
5562 void Assembler::sqshlu(const VRegister& vd, const VRegister& vn, int shift) {
5564 NEONShiftLeftImmediate(vd, vn, shift, NEON_SQSHLU);
5568 void Assembler::uqshl(const VRegister& vd, const VRegister& vn, int shift) {
5570 NEONShiftLeftImmediate(vd, vn, shift, NEON_UQSHL_imm);
5574 void Assembler::sshll(const VRegister& vd, const VRegister& vn, int shift) {
5577 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
5581 void Assembler::sshll2(const VRegister& vd, const VRegister& vn, int shift) {
5584 NEONShiftImmediateL(vd, vn, shift, NEON_SSHLL);
5588 void Assembler::sxtl(const VRegister& vd, const VRegister& vn) {
5590 sshll(vd, vn, 0);
5594 void Assembler::sxtl2(const VRegister& vd, const VRegister& vn) {
5596 sshll2(vd, vn, 0);
5600 void Assembler::ushll(const VRegister& vd, const VRegister& vn, int shift) {
5603 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
5607 void Assembler::ushll2(const VRegister& vd, const VRegister& vn, int shift) {
5610 NEONShiftImmediateL(vd, vn, shift, NEON_USHLL);
5614 void Assembler::uxtl(const VRegister& vd, const VRegister& vn) {
5616 ushll(vd, vn, 0);
5620 void Assembler::uxtl2(const VRegister& vd, const VRegister& vn) {
5622 ushll2(vd, vn, 0);
5626 void Assembler::sri(const VRegister& vd, const VRegister& vn, int shift) {
5628 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5629 NEONShiftRightImmediate(vd, vn, shift, NEON_SRI);
5633 void Assembler::sshr(const VRegister& vd, const VRegister& vn, int shift) {
5635 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5636 NEONShiftRightImmediate(vd, vn, shift, NEON_SSHR);
5640 void Assembler::ushr(const VRegister& vd, const VRegister& vn, int shift) {
5642 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5643 NEONShiftRightImmediate(vd, vn, shift, NEON_USHR);
5647 void Assembler::srshr(const VRegister& vd, const VRegister& vn, int shift) {
5649 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5650 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSHR);
5654 void Assembler::urshr(const VRegister& vd, const VRegister& vn, int shift) {
5656 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5657 NEONShiftRightImmediate(vd, vn, shift, NEON_URSHR);
5661 void Assembler::ssra(const VRegister& vd, const VRegister& vn, int shift) {
5663 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5664 NEONShiftRightImmediate(vd, vn, shift, NEON_SSRA);
5668 void Assembler::usra(const VRegister& vd, const VRegister& vn, int shift) {
5670 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5671 NEONShiftRightImmediate(vd, vn, shift, NEON_USRA);
5675 void Assembler::srsra(const VRegister& vd, const VRegister& vn, int shift) {
5677 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5678 NEONShiftRightImmediate(vd, vn, shift, NEON_SRSRA);
5682 void Assembler::ursra(const VRegister& vd, const VRegister& vn, int shift) {
5684 VIXL_ASSERT(vd.IsVector() || vd.Is1D());
5685 NEONShiftRightImmediate(vd, vn, shift, NEON_URSRA);
5689 void Assembler::shrn(const VRegister& vd, const VRegister& vn, int shift) {
5691 VIXL_ASSERT(vn.IsVector() && vd.IsD());
5692 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
5696 void Assembler::shrn2(const VRegister& vd, const VRegister& vn, int shift) {
5698 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5699 NEONShiftImmediateN(vd, vn, shift, NEON_SHRN);
5703 void Assembler::rshrn(const VRegister& vd, const VRegister& vn, int shift) {
5705 VIXL_ASSERT(vn.IsVector() && vd.IsD());
5706 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
5710 void Assembler::rshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5712 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5713 NEONShiftImmediateN(vd, vn, shift, NEON_RSHRN);
5717 void Assembler::sqshrn(const VRegister& vd, const VRegister& vn, int shift) {
5719 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5720 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
5724 void Assembler::sqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5726 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5727 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRN);
5731 void Assembler::sqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
5733 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5734 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
5738 void Assembler::sqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5740 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5741 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRN);
5745 void Assembler::sqshrun(const VRegister& vd, const VRegister& vn, int shift) {
5747 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5748 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
5752 void Assembler::sqshrun2(const VRegister& vd, const VRegister& vn, int shift) {
5754 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5755 NEONShiftImmediateN(vd, vn, shift, NEON_SQSHRUN);
5759 void Assembler::sqrshrun(const VRegister& vd, const VRegister& vn, int shift) {
5761 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5762 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
5766 void Assembler::sqrshrun2(const VRegister& vd, const VRegister& vn, int shift) {
5768 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5769 NEONShiftImmediateN(vd, vn, shift, NEON_SQRSHRUN);
5773 void Assembler::uqshrn(const VRegister& vd, const VRegister& vn, int shift) {
5775 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5776 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
5780 void Assembler::uqshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5782 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5783 NEONShiftImmediateN(vd, vn, shift, NEON_UQSHRN);
5787 void Assembler::uqrshrn(const VRegister& vd, const VRegister& vn, int shift) {
5789 VIXL_ASSERT(vd.IsD() || (vn.IsScalar() && vd.IsScalar()));
5790 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
5794 void Assembler::uqrshrn2(const VRegister& vd, const VRegister& vn, int shift) {
5796 VIXL_ASSERT(vn.IsVector() && vd.IsQ());
5797 NEONShiftImmediateN(vd, vn, shift, NEON_UQRSHRN);
5800 void Assembler::smmla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5803 VIXL_ASSERT(vd.IsLaneSizeS());
5806 Emit(0x4e80a400 | Rd(vd) | Rn(vn) | Rm(vm));
5809 void Assembler::usmmla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5812 VIXL_ASSERT(vd.IsLaneSizeS());
5815 Emit(0x4e80ac00 | Rd(vd) | Rn(vn) | Rm(vm));
5818 void Assembler::ummla(const VRegister& vd, const VRegister& vn, const VRegister& vm) {
5821 VIXL_ASSERT(vd.IsLaneSizeS());
5824 Emit(0x6e80a400 | Rd(vd) | Rn(vn) | Rm(vm));
6122 void Assembler::FPDataProcessing1Source(const VRegister& vd,
6125 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
6126 Emit(FPType(vn) | op | Rn(vn) | Rd(vd));
6130 void Assembler::FPDataProcessing3Source(const VRegister& vd,
6135 VIXL_ASSERT(vd.Is1H() || vd.Is1S() || vd.Is1D());
6136 VIXL_ASSERT(AreSameSizeAndType(vd, vn, vm, va));
6137 Emit(FPType(vd) | op | Rm(vm) | Rn(vn) | Rd(vd) | Ra(va));
6141 void Assembler::NEONModifiedImmShiftLsl(const VRegister& vd,
6145 VIXL_ASSERT(vd.Is8B() || vd.Is16B() || vd.Is4H() || vd.Is8H() || vd.Is2S() ||
6146 vd.Is4S());
6152 if (vd.Is8B() || vd.Is16B()) {
6161 if (vd.Is4H() || vd.Is8H()) {
6168 int q = vd.IsQ() ? NEON_Q : 0;
6170 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));
6174 void Assembler::NEONModifiedImmShiftMsl(const VRegister& vd,
6178 VIXL_ASSERT(vd.Is2S() || vd.Is4S());
6185 int q = vd.IsQ() ? NEON_Q : 0;
6187 Emit(q | op | ImmNEONabcdefgh(imm8) | NEONCmode(cmode) | Rd(vd));