[AArch64] Enable Rounding Double Multiply Add/Subtract instructions on Falkor.

[mcrosier]

Summary

Falkor only partially implements the ARMv8.1a extensions, so this patch refactors the support for the SQRDML[A|S]H instruction into a separate feature.

Chad

[ab]

LGTM

test/CodeGen/AArch64/arm64-neon-v8.1a.ll

2

Maybe add --check-prefix=CHECK-RDM to the v8.1a line, and remove the rdm CHECK-V81a lines?

[mcrosier]

test/CodeGen/AArch64/arm64-neon-v8.1a.ll

2

I'm not sure I follow, Ahmed. Are you suggesting that I not add an additional RUN line and replace

; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+v8.1a -aarch64-neon-syntax=generic | FileCheck %s --check-prefix=CHECK-V81a

with

; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+rdm -aarch64-neon-syntax=generic | FileCheck %s --check-prefix=CHECK-RDM

and update the CHECK-V81a directives to CHECK-RDM?

I'm happy to do that, I just want to confirm that is what you were suggesting.

[ab]

test/CodeGen/AArch64/arm64-neon-v8.1a.ll

2

Oh sorry: I was thinking you'd add the RUN line:

; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+rdm -aarch64-neon-syntax=generic | FileCheck %s --check-prefix=CHECK-RDM
; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+v8.1a -aarch64-neon-syntax=generic | FileCheck %s --check-prefix=CHECK-V81a --check-prefix=CHECK-RDM

Which lets you avoid duplicating the CHECK-RDM lines; e.g., instead of:

    %prod = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %mhs,  <4 x i16> %rhs)
    %retval =  call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> %acc,  <4 x i16> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.4h, v1.4h, v2.4h
+; CHECK-RDM:        sqrdmlah    v0.4h, v1.4h, v2.4h
 ; CHECK-V81a:       sqrdmlah    v0.4h, v1.4h, v2.4h
 ; CHECK-V81a-apple: sqrdmlah.4h v0,    v1,    v2
    ret <4 x i16> %retval

you'd do:

    %prod = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %mhs,  <4 x i16> %rhs)
    %retval =  call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> %acc,  <4 x i16> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.4h, v1.4h, v2.4h
-; CHECK-V81a:       sqrdmlah    v0.4h, v1.4h, v2.4h
+; CHECK-RDM:        sqrdmlah    v0.4h, v1.4h, v2.4h
 ; CHECK-V81a-apple: sqrdmlah.4h v0,    v1,    v2
    ret <4 x i16> %retval

Not a big deal either way, really.

[mcrosier]

I settled on adding a new RUN line, but used the CHECK-V81a prefix. I believe this was in the same vein as your original suggestion, Ahmed, but minimized code changes while still providing the necessary testing coverage.

lib/Target/AArch64/AArch64.td

def FeatureCRC : SubtargetFeature<"crc", "HasCRC", "true",
   "Enable ARMv8 CRC-32 checksum instructions">;
 
 def FeatureRAS : SubtargetFeature<"ras", "HasRAS", "true",
   "Enable ARMv8 Reliability, Availability and Serviceability Extensions">;
 
 def FeatureLSE : SubtargetFeature<"lse", "HasLSE", "true",
   "Enable ARMv8.1 Large System Extension (LSE) atomic instructions">;
 
+def FeatureRDM : SubtargetFeature<"rdm", "HasRDM", "true",
+  "Enable ARMv8.1 Rounding Double Multiply Add/Subtract instructions">;
+
 def FeaturePerfMon : SubtargetFeature<"perfmon", "HasPerfMon", "true",
   "Enable ARMv8 PMUv3 Performance Monitors extension">;
 
 def FeatureFullFP16 : SubtargetFeature<"fullfp16", "HasFullFP16", "true",
   "Full FP16", [FeatureFPARMv8]>;
 
 def FeatureSPE : SubtargetFeature<"spe", "HasSPE", "true",
   "Enable Statistical Profiling extension">;
def FeatureUseRSqrt : SubtargetFeature<
     "use-reciprocal-square-root", "UseRSqrt", "true",
     "Use the reciprocal square root approximation">;
 
 //===----------------------------------------------------------------------===//
 // Architectures.
 //
 
 def HasV8_1aOps : SubtargetFeature<"v8.1a", "HasV8_1aOps", "true",
-  "Support ARM v8.1a instructions", [FeatureCRC, FeatureLSE]>;
+  "Support ARM v8.1a instructions", [FeatureCRC, FeatureLSE, FeatureRDM]>;
 
 def HasV8_2aOps : SubtargetFeature<"v8.2a", "HasV8_2aOps", "true",
   "Support ARM v8.2a instructions", [HasV8_1aOps, FeatureRAS]>;
 
 //===----------------------------------------------------------------------===//
 // Register File Description
 //===----------------------------------------------------------------------===//
 
def ProcFalkor  : SubtargetFeature<"falkor", "ARMProcFamily", "Falkor",
                                    FeatureCRC,
                                    FeatureCrypto,
                                    FeatureCustomCheapAsMoveHandling,
                                    FeatureFPARMv8,
                                    FeatureNEON,
                                    FeaturePerfMon,
                                    FeaturePostRAScheduler,
                                    FeaturePredictableSelectIsExpensive,
+                                   FeatureRDM,
                                    FeatureZCZeroing
                                    ]>;
 
 def ProcVulcan  : SubtargetFeature<"vulcan", "ARMProcFamily", "Vulcan",
                                    "Broadcom Vulcan processors", [
                                    FeatureCRC,
                                    FeatureCrypto,
                                    FeatureFPARMv8,

lib/Target/AArch64/AArch64InstrFormats.td

multiclass SIMDLdSt4SingleAliases<string asm> {
   defm : SIMDLdStSingleAliases<asm, "d", "i64", "Four", 32, VectorIndexD>;
 }
 } // end of 'let Predicates = [HasNEON]'
 
 //----------------------------------------------------------------------------
 // AdvSIMD v8.1 Rounding Double Multiply Add/Subtract
 //----------------------------------------------------------------------------
 
-let Predicates = [HasNEON, HasV8_1a] in {
+let Predicates = [HasNEON, HasRDM] in {
 
 class BaseSIMDThreeSameVectorTiedR0<bit Q, bit U, bits<2> size, bits<5> opcode,
                                     RegisterOperand regtype, string asm, 
                                     string kind, list<dag> pattern>
   : BaseSIMDThreeSameVectorTied<Q, U, {size,0}, opcode, regtype, asm, kind, 
                                 pattern> {
 }
 multiclass SIMDThreeSameVectorSQRDMLxHTiedHS<bit U, bits<5> opc, string asm,
[(set (i32 FPR32Op:$dst),
                         (i32 FPR32Op:$Rn),
                         (i32 (vector_extract (v4i32 V128:$Rm),
                                              VectorIndexS:$idx))))))]> {
     bits<2> idx;
     let Inst{11} = idx{1};
     let Inst{21} = idx{0};
   }
 }
-} // let Predicates = [HasNeon, HasV8_1a]
+} // let Predicates = [HasNeon, HasRDM]
 
 //----------------------------------------------------------------------------
 // Crypto extensions
 //----------------------------------------------------------------------------
 
 let Predicates = [HasCrypto] in {
 let mayLoad = 0, mayStore = 0, hasSideEffects = 0 in
 class AESBase<bits<4> opc, string asm, dag outs, dag ins, string cstr,

lib/Target/AArch64/AArch64InstrInfo.td

 def HasCrypto        : Predicate<"Subtarget->hasCrypto()">,
                                  AssemblerPredicate<"FeatureCrypto", "crypto">;
 def HasCRC           : Predicate<"Subtarget->hasCRC()">,
                                  AssemblerPredicate<"FeatureCRC", "crc">;
 def HasLSE           : Predicate<"Subtarget->hasLSE()">,
                                  AssemblerPredicate<"FeatureLSE", "lse">;
 def HasRAS           : Predicate<"Subtarget->hasRAS()">,
                                  AssemblerPredicate<"FeatureRAS", "ras">;
+def HasRDM           : Predicate<"Subtarget->hasRDM()">,
+                                 AssemblerPredicate<"FeatureRDM", "rdm">;
 def HasPerfMon       : Predicate<"Subtarget->hasPerfMon()">;
 def HasFullFP16      : Predicate<"Subtarget->hasFullFP16()">,
                                  AssemblerPredicate<"FeatureFullFP16", "fullfp16">;
 def HasSPE           : Predicate<"Subtarget->hasSPE()">,
                                  AssemblerPredicate<"FeatureSPE", "spe">;
 
 def IsLE             : Predicate<"Subtarget->isLittleEndian()">;
 def IsBE             : Predicate<"!Subtarget->isLittleEndian()">;

 defm SSHL     : SIMDThreeScalarD<   0, 0b01000, "sshl", int_aarch64_neon_sshl>;
 defm SUB      : SIMDThreeScalarD<   1, 0b10000, "sub", sub>;
 defm UQADD    : SIMDThreeScalarBHSD<1, 0b00001, "uqadd", int_aarch64_neon_uqadd>;
 defm UQRSHL   : SIMDThreeScalarBHSD<1, 0b01011, "uqrshl",int_aarch64_neon_uqrshl>;
 defm UQSHL    : SIMDThreeScalarBHSD<1, 0b01001, "uqshl", int_aarch64_neon_uqshl>;
 defm UQSUB    : SIMDThreeScalarBHSD<1, 0b00101, "uqsub", int_aarch64_neon_uqsub>;
 defm URSHL    : SIMDThreeScalarD<   1, 0b01010, "urshl", int_aarch64_neon_urshl>;
 defm USHL     : SIMDThreeScalarD<   1, 0b01000, "ushl", int_aarch64_neon_ushl>;
-let Predicates = [HasV8_1a] in {
+let Predicates = [HasRDM] in {
   defm SQRDMLAH : SIMDThreeScalarHSTied<1, 0, 0b10000, "sqrdmlah">;
   defm SQRDMLSH : SIMDThreeScalarHSTied<1, 0, 0b10001, "sqrdmlsh">;
   def : Pat<(i32 (int_aarch64_neon_sqadd
                    (i32 FPR32:$Rd),
                    (i32 (int_aarch64_neon_sqrdmulh (i32 FPR32:$Rn),
                                                    (i32 FPR32:$Rm))))),
             (SQRDMLAHv1i32 FPR32:$Rd, FPR32:$Rn, FPR32:$Rm)>;
   def : Pat<(i32 (int_aarch64_neon_sqsub

lib/Target/AArch64/AArch64Subtarget.h

protected:
   bool HasV8_2aOps = false;
 
   bool HasFPARMv8 = false;
   bool HasNEON = false;
   bool HasCrypto = false;
   bool HasCRC = false;
   bool HasLSE = false;
   bool HasRAS = false;
+  bool HasRDM = false;
   bool HasPerfMon = false;
   bool HasFullFP16 = false;
   bool HasSPE = false;
 
   // HasZeroCycleRegMove - Has zero-cycle register mov instructions.
   bool HasZeroCycleRegMove = false;
 
   // HasZeroCycleZeroing - Has zero-cycle zeroing instructions.
public:
 
   bool isX18Reserved() const { return ReserveX18; }
   bool hasFPARMv8() const { return HasFPARMv8; }
   bool hasNEON() const { return HasNEON; }
   bool hasCrypto() const { return HasCrypto; }
   bool hasCRC() const { return HasCRC; }
   bool hasLSE() const { return HasLSE; }
   bool hasRAS() const { return HasRAS; }
+  bool hasRDM() const { return HasRDM; }
   bool balanceFPOps() const { return BalanceFPOps; }
   bool predictableSelectIsExpensive() const {
     return PredictableSelectIsExpensive;
   }
   bool hasCustomCheapAsMoveHandling() const { return CustomAsCheapAsMove; }
   bool isMisaligned128StoreSlow() const { return Misaligned128StoreIsSlow; }
   bool avoidQuadLdStPairs() const { return AvoidQuadLdStPairs; }
   bool useAlternateSExtLoadCVTF32Pattern() const {

test/CodeGen/AArch64/arm64-neon-v8.1a.ll

 ; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -aarch64-neon-syntax=generic | FileCheck %s --check-prefix=CHECK-V8a
+; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+rdm -aarch64-neon-syntax=generic | FileCheck %s --check-prefix=CHECK-RDM
 ; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+v8.1a -aarch64-neon-syntax=generic | FileCheck %s --check-prefix=CHECK-V81a
 ; RUN: llc < %s -verify-machineinstrs -mtriple=arm64-eabi -mattr=+v8.1a -aarch64-neon-syntax=apple | FileCheck %s --check-prefix=CHECK-V81a-apple
 
 declare <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16>, <4 x i16>)
 declare <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16>, <8 x i16>)
 declare <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32>, <2 x i32>)
 declare <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32>, <4 x i32>)
 declare i32 @llvm.aarch64.neon.sqrdmulh.i32(i32, i32)

 ; RDMA Vector
 ; test for SIMDThreeSameVectorSQRDMLxHTiedHS
 
 define <4 x i16> @test_sqrdmlah_v4i16(<4 x i16> %acc, <4 x i16> %mhs, <4 x i16> %rhs) {
 ; CHECK-LABEL: test_sqrdmlah_v4i16:
    %prod = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %mhs,  <4 x i16> %rhs)
    %retval =  call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> %acc,  <4 x i16> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.4h, v1.4h, v2.4h
+; CHECK-RDM:        sqrdmlah    v0.4h, v1.4h, v2.4h
 ; CHECK-V81a:       sqrdmlah    v0.4h, v1.4h, v2.4h
 ; CHECK-V81a-apple: sqrdmlah.4h v0,    v1,    v2
    ret <4 x i16> %retval
 }
 
 define <8 x i16> @test_sqrdmlah_v8i16(<8 x i16> %acc, <8 x i16> %mhs, <8 x i16> %rhs) {
 ; CHECK-LABEL: test_sqrdmlah_v8i16:
    %prod = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %mhs, <8 x i16> %rhs)
    %retval =  call <8 x i16> @llvm.aarch64.neon.sqadd.v8i16(<8 x i16> %acc, <8 x i16> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.8h, v1.8h, v2.8h
+; CHECK-RDM:        sqrdmlah    v0.8h, v1.8h, v2.8h
 ; CHECK-V81a:       sqrdmlah    v0.8h, v1.8h, v2.8h
 ; CHECK-V81a-apple: sqrdmlah.8h v0, v1, v2
    ret <8 x i16> %retval
 }
 
 define <2 x i32> @test_sqrdmlah_v2i32(<2 x i32> %acc, <2 x i32> %mhs, <2 x i32> %rhs) {
 ; CHECK-LABEL: test_sqrdmlah_v2i32:
    %prod = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %mhs, <2 x i32> %rhs)
    %retval =  call <2 x i32> @llvm.aarch64.neon.sqadd.v2i32(<2 x i32> %acc, <2 x i32> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.2s, v1.2s, v2.2s
+; CHECK-RDM:        sqrdmlah    v0.2s, v1.2s, v2.2s
 ; CHECK-V81a:       sqrdmlah    v0.2s, v1.2s, v2.2s
 ; CHECK-V81a-apple: sqrdmlah.2s v0,    v1,    v2
    ret <2 x i32> %retval
 }
 
 define <4 x i32> @test_sqrdmlah_v4i32(<4 x i32> %acc, <4 x i32> %mhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: test_sqrdmlah_v4i32:
    %prod = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %mhs, <4 x i32> %rhs)
    %retval =  call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %acc, <4 x i32> %prod)
-; CHECK-V81:        sqrdmulh    v1.4s, v1.4s, v2.4s
+; CHECK-V8:         sqrdmulh    v1.4s, v1.4s, v2.4s
+; CHECK-RDM:        sqrdmlah    v0.4s, v1.4s, v2.4s
 ; CHECK-V81a:       sqrdmlah    v0.4s, v1.4s, v2.4s
 ; CHECK-V81a-apple: sqrdmlah.4s v0,    v1,    v2
    ret <4 x i32> %retval
 }
 
 define <4 x i16> @test_sqrdmlsh_v4i16(<4 x i16> %acc, <4 x i16> %mhs, <4 x i16> %rhs) {
 ; CHECK-LABEL: test_sqrdmlsh_v4i16:
    %prod = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %mhs,  <4 x i16> %rhs)
    %retval =  call <4 x i16> @llvm.aarch64.neon.sqsub.v4i16(<4 x i16> %acc, <4 x i16> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.4h, v1.4h, v2.4h
+; CHECK-RDM:        sqrdmlsh    v0.4h, v1.4h, v2.4h
 ; CHECK-V81a:       sqrdmlsh    v0.4h, v1.4h, v2.4h
 ; CHECK-V81a-apple: sqrdmlsh.4h v0,    v1,    v2
    ret <4 x i16> %retval
 }
 
 define <8 x i16> @test_sqrdmlsh_v8i16(<8 x i16> %acc, <8 x i16> %mhs, <8 x i16> %rhs) {
 ; CHECK-LABEL: test_sqrdmlsh_v8i16:
    %prod = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %mhs, <8 x i16> %rhs)
    %retval =  call <8 x i16> @llvm.aarch64.neon.sqsub.v8i16(<8 x i16> %acc, <8 x i16> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.8h, v1.8h, v2.8h
+; CHECK-RDM:        sqrdmlsh    v0.8h, v1.8h, v2.8h
 ; CHECK-V81a:       sqrdmlsh    v0.8h, v1.8h, v2.8h
 ; CHECK-V81a-apple: sqrdmlsh.8h v0,    v1,    v2
    ret <8 x i16> %retval
 }
 
 define <2 x i32> @test_sqrdmlsh_v2i32(<2 x i32> %acc, <2 x i32> %mhs, <2 x i32> %rhs) {
 ; CHECK-LABEL: test_sqrdmlsh_v2i32:
    %prod = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %mhs, <2 x i32> %rhs)
    %retval =  call <2 x i32> @llvm.aarch64.neon.sqsub.v2i32(<2 x i32> %acc, <2 x i32> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.2s, v1.2s, v2.2s
+; CHECK-RDM:        sqrdmlsh    v0.2s, v1.2s, v2.2s
 ; CHECK-V81a:       sqrdmlsh    v0.2s, v1.2s, v2.2s
 ; CHECK-V81a-apple: sqrdmlsh.2s v0,    v1,    v2
    ret <2 x i32> %retval
 }
 
 define <4 x i32> @test_sqrdmlsh_v4i32(<4 x i32> %acc, <4 x i32> %mhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: test_sqrdmlsh_v4i32:
    %prod = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %mhs, <4 x i32> %rhs)
    %retval =  call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %acc, <4 x i32> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.4s, v1.4s, v2.4s
+; CHECK-RDM:        sqrdmlsh    v0.4s, v1.4s, v2.4s
 ; CHECK-V81a:       sqrdmlsh    v0.4s, v1.4s, v2.4s
 ; CHECK-V81a-apple: sqrdmlsh.4s v0,    v1,    v2
    ret <4 x i32> %retval
 }
 
 ;-----------------------------------------------------------------------------
 ; RDMA Vector, by element
 ; tests for vXiYY_indexed in SIMDIndexedSQRDMLxHSDTied
 
 define <4 x i16> @test_sqrdmlah_lane_s16(<4 x i16> %acc, <4 x i16> %x, <4 x i16> %v) {
 ; CHECK-LABEL: test_sqrdmlah_lane_s16:
 entry:
   %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %prod = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %x, <4 x i16> %shuffle)
   %retval =  call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> %acc, <4 x i16> %prod)
 ; CHECK-V8a :       sqrdmulh    v1.4h, v1.4h, v2.h[3]
+; CHECK-RDM:        sqrdmlah    v0.4h, v1.4h, v2.h[3]
 ; CHECK-V81a:       sqrdmlah    v0.4h, v1.4h, v2.h[3]
 ; CHECK-V81a-apple: sqrdmlah.4h v0,    v1,    v2[3]
   ret <4 x i16> %retval
 }
 
 define <8 x i16> @test_sqrdmlahq_lane_s16(<8 x i16> %acc, <8 x i16> %x, <8 x i16> %v) {
 ; CHECK-LABEL: test_sqrdmlahq_lane_s16:
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
   %prod = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %x, <8 x i16> %shuffle)
   %retval =  call <8 x i16> @llvm.aarch64.neon.sqadd.v8i16(<8 x i16> %acc, <8 x i16> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.8h, v1.8h, v2.h[2]
+; CHECK-RDM:        sqrdmlah    v0.8h, v1.8h, v2.h[2]
 ; CHECK-V81a:       sqrdmlah    v0.8h, v1.8h, v2.h[2]
 ; CHECK-V81a-apple: sqrdmlah.8h v0,    v1,    v2[2]
   ret <8 x i16> %retval
 }
 
 define <2 x i32> @test_sqrdmlah_lane_s32(<2 x i32> %acc, <2 x i32> %x, <2 x i32> %v) {
 ; CHECK-LABEL: test_sqrdmlah_lane_s32:
 entry:
   %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
   %prod = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %x, <2 x i32> %shuffle)
   %retval =  call <2 x i32> @llvm.aarch64.neon.sqadd.v2i32(<2 x i32> %acc, <2 x i32> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.2s, v1.2s, v2.s[1]
+; CHECK-RDM:        sqrdmlah    v0.2s, v1.2s, v2.s[1]
 ; CHECK-V81a:       sqrdmlah    v0.2s, v1.2s, v2.s[1]
 ; CHECK-V81a-apple: sqrdmlah.2s v0,    v1,    v2[1]
   ret <2 x i32> %retval
 }
 
 define <4 x i32> @test_sqrdmlahq_lane_s32(<4 x i32> %acc,<4 x i32> %x, <4 x i32> %v) {
 ; CHECK-LABEL: test_sqrdmlahq_lane_s32:
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> zeroinitializer
   %prod = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %x, <4 x i32> %shuffle)
   %retval =  call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %acc, <4 x i32> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.4s, v1.4s, v2.s[0]
+; CHECK-RDM:        sqrdmlah    v0.4s, v1.4s, v2.s[0]
 ; CHECK-V81a:       sqrdmlah    v0.4s, v1.4s, v2.s[0]
 ; CHECK-V81a-apple: sqrdmlah.4s v0,    v1,    v2[0]
   ret <4 x i32> %retval
 }
 
 define <4 x i16> @test_sqrdmlsh_lane_s16(<4 x i16> %acc, <4 x i16> %x, <4 x i16> %v) {
 ; CHECK-LABEL: test_sqrdmlsh_lane_s16:
 entry:
   %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 3, i32 3, i32 3, i32 3>
   %prod = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %x, <4 x i16> %shuffle)
   %retval =  call <4 x i16> @llvm.aarch64.neon.sqsub.v4i16(<4 x i16> %acc, <4 x i16> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.4h, v1.4h, v2.h[3]
+; CHECK-RDM:        sqrdmlsh    v0.4h, v1.4h, v2.h[3]
 ; CHECK-V81a:       sqrdmlsh    v0.4h, v1.4h, v2.h[3]
 ; CHECK-V81a-apple: sqrdmlsh.4h v0,    v1,    v2[3]
   ret <4 x i16> %retval
 }
 
 define <8 x i16> @test_sqrdmlshq_lane_s16(<8 x i16> %acc, <8 x i16> %x, <8 x i16> %v) {
 ; CHECK-LABEL: test_sqrdmlshq_lane_s16:
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> <i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2, i32 2>
   %prod = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %x, <8 x i16> %shuffle)
   %retval =  call <8 x i16> @llvm.aarch64.neon.sqsub.v8i16(<8 x i16> %acc, <8 x i16> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.8h, v1.8h, v2.h[2]
+; CHECK-RDM:        sqrdmlsh    v0.8h, v1.8h, v2.h[2]
 ; CHECK-V81a:       sqrdmlsh    v0.8h, v1.8h, v2.h[2]
 ; CHECK-V81a-apple: sqrdmlsh.8h v0,    v1,    v2[2]
   ret <8 x i16> %retval
 }
 
 define <2 x i32> @test_sqrdmlsh_lane_s32(<2 x i32> %acc, <2 x i32> %x, <2 x i32> %v) {
 ; CHECK-LABEL: test_sqrdmlsh_lane_s32:
 entry:
   %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> <i32 1, i32 1>
   %prod = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %x, <2 x i32> %shuffle)
   %retval =  call <2 x i32> @llvm.aarch64.neon.sqsub.v2i32(<2 x i32> %acc, <2 x i32> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.2s, v1.2s, v2.s[1]
+; CHECK-RDM:        sqrdmlsh    v0.2s, v1.2s, v2.s[1]
 ; CHECK-V81a:       sqrdmlsh    v0.2s, v1.2s, v2.s[1]
 ; CHECK-V81a-apple: sqrdmlsh.2s v0,    v1,    v2[1]
   ret <2 x i32> %retval
 }
 
 define <4 x i32> @test_sqrdmlshq_lane_s32(<4 x i32> %acc,<4 x i32> %x, <4 x i32> %v) {
 ; CHECK-LABEL: test_sqrdmlshq_lane_s32:
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> zeroinitializer
   %prod = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %x, <4 x i32> %shuffle)
   %retval =  call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %acc, <4 x i32> %prod)
 ; CHECK-V8a:        sqrdmulh    v1.4s, v1.4s, v2.s[0]
+; CHECK-RDM:        sqrdmlsh    v0.4s, v1.4s, v2.s[0]
 ; CHECK-V81a:       sqrdmlsh    v0.4s, v1.4s, v2.s[0]
 ; CHECK-V81a-apple: sqrdmlsh.4s v0,    v1,    v2[0]
   ret <4 x i32> %retval
 }
 
 ;-----------------------------------------------------------------------------
 ; RDMA Vector, by element, extracted
 ; i16 tests are for vXi16_indexed in SIMDIndexedSQRDMLxHSDTied, with IR in ACLE style
 ; i32 tests are for   "def : Pat" in SIMDIndexedSQRDMLxHSDTied
 
 define i16 @test_sqrdmlah_extracted_lane_s16(i16 %acc,<4 x i16> %x, <4 x i16> %v) {
 ; CHECK-LABEL: test_sqrdmlah_extracted_lane_s16:
 entry:
   %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 1,i32 1,i32 1,i32 1>
   %prod = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %x, <4 x i16> %shuffle)
   %acc_vec = insertelement <4 x i16> undef, i16 %acc, i64 0
   %retval_vec =  call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> %acc_vec, <4 x i16> %prod)
   %retval = extractelement <4 x i16> %retval_vec, i64 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.4h, v0.4h, v1.h[1]
+; CHECK-RDM:        sqrdmlah    {{v[2-9]+}}.4h, v0.4h, v1.h[1]
 ; CHECK-V81a:       sqrdmlah    {{v[2-9]+}}.4h, v0.4h, v1.h[1]
 ; CHECK-V81a-apple: sqrdmlah.4h {{v[2-9]+}},    v0,    v1[1]
   ret i16 %retval
 }
 
 define i16 @test_sqrdmlahq_extracted_lane_s16(i16 %acc,<8 x i16> %x, <8 x i16> %v) {
 ; CHECK-LABEL: test_sqrdmlahq_extracted_lane_s16:
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> <i32 1,i32 1,i32 1,i32 1, i32 1,i32 1,i32 1,i32 1>
   %prod = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %x, <8 x i16> %shuffle)
   %acc_vec = insertelement <8 x i16> undef, i16 %acc, i64 0
   %retval_vec =  call <8 x i16> @llvm.aarch64.neon.sqadd.v8i16(<8 x i16> %acc_vec, <8 x i16> %prod)
   %retval = extractelement <8 x i16> %retval_vec, i64 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.8h, v0.8h, v1.h[1]
+; CHECK-RDM:        sqrdmlah    {{v[2-9]+}}.8h, v0.8h, v1.h[1]
 ; CHECK-V81a:       sqrdmlah    {{v[2-9]+}}.8h, v0.8h, v1.h[1]
 ; CHECK-V81a-apple: sqrdmlah.8h {{v[2-9]+}},    v0,    v1[1]
   ret i16 %retval
 }
 
 define i32 @test_sqrdmlah_extracted_lane_s32(i32 %acc,<2 x i32> %x, <2 x i32> %v) {
 ; CHECK-LABEL: test_sqrdmlah_extracted_lane_s32:
 entry:
   %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer
   %prod = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %x, <2 x i32> %shuffle)
   %extract = extractelement <2 x i32> %prod, i64 0
   %retval =  call i32 @llvm.aarch64.neon.sqadd.i32(i32 %acc, i32 %extract)
 ; CHECK-V8a:        sqrdmulh    v0.2s, v0.2s, v1.s[0]
+; CHECK-RDM:        sqrdmlah    v2.2s, v0.2s, v1.s[0]
 ; CHECK-V81a:       sqrdmlah    v2.2s, v0.2s, v1.s[0]
 ; CHECK-V81a-apple: sqrdmlah.2s v2,    v0,    v1[0]
   ret i32 %retval
 }
 
 define i32 @test_sqrdmlahq_extracted_lane_s32(i32 %acc,<4 x i32> %x, <4 x i32> %v) {
 ; CHECK-LABEL: test_sqrdmlahq_extracted_lane_s32:
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> zeroinitializer
   %prod = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %x, <4 x i32> %shuffle)
   %extract = extractelement <4 x i32> %prod, i64 0
   %retval =  call i32 @llvm.aarch64.neon.sqadd.i32(i32 %acc, i32 %extract)
 ; CHECK-V8a:        sqrdmulh    v0.4s, v0.4s, v1.s[0]
+; CHECK-RDM:        sqrdmlah    v2.4s, v0.4s, v1.s[0]
 ; CHECK-V81a:       sqrdmlah    v2.4s, v0.4s, v1.s[0]
 ; CHECK-V81a-apple: sqrdmlah.4s v2,    v0,    v1[0]
   ret i32 %retval
 }
 
 define i16 @test_sqrdmlsh_extracted_lane_s16(i16 %acc,<4 x i16> %x, <4 x i16> %v) {
 ; CHECK-LABEL: test_sqrdmlsh_extracted_lane_s16:
 entry:
   %shuffle = shufflevector <4 x i16> %v, <4 x i16> undef, <4 x i32> <i32 1,i32 1,i32 1,i32 1>
   %prod = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %x, <4 x i16> %shuffle)
   %acc_vec = insertelement <4 x i16> undef, i16 %acc, i64 0
   %retval_vec =  call <4 x i16> @llvm.aarch64.neon.sqsub.v4i16(<4 x i16> %acc_vec, <4 x i16> %prod)
   %retval = extractelement <4 x i16> %retval_vec, i64 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.4h, v0.4h, v1.h[1]
+; CHECK-RDM:        sqrdmlsh    {{v[2-9]+}}.4h, v0.4h, v1.h[1]
 ; CHECK-V81a:       sqrdmlsh    {{v[2-9]+}}.4h, v0.4h, v1.h[1]
 ; CHECK-V81a-apple: sqrdmlsh.4h {{v[2-9]+}},    v0,    v1[1]
   ret i16 %retval
 }
 
 define i16 @test_sqrdmlshq_extracted_lane_s16(i16 %acc,<8 x i16> %x, <8 x i16> %v) {
 ; CHECK-LABEL: test_sqrdmlshq_extracted_lane_s16:
 entry:
   %shuffle = shufflevector <8 x i16> %v, <8 x i16> undef, <8 x i32> <i32 1,i32 1,i32 1,i32 1, i32 1,i32 1,i32 1,i32 1>
   %prod = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %x, <8 x i16> %shuffle)
   %acc_vec = insertelement <8 x i16> undef, i16 %acc, i64 0
   %retval_vec =  call <8 x i16> @llvm.aarch64.neon.sqsub.v8i16(<8 x i16> %acc_vec, <8 x i16> %prod)
   %retval = extractelement <8 x i16> %retval_vec, i64 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.8h, v0.8h, v1.h[1]
+; CHECK-RDM:        sqrdmlsh    {{v[2-9]+}}.8h, v0.8h, v1.h[1]
 ; CHECK-V81a:       sqrdmlsh    {{v[2-9]+}}.8h, v0.8h, v1.h[1]
 ; CHECK-V81a-apple: sqrdmlsh.8h {{v[2-9]+}},    v0,    v1[1]
   ret i16 %retval
 }
 
 define i32 @test_sqrdmlsh_extracted_lane_s32(i32 %acc,<2 x i32> %x, <2 x i32> %v) {
 ; CHECK-LABEL: test_sqrdmlsh_extracted_lane_s32:
 entry:
   %shuffle = shufflevector <2 x i32> %v, <2 x i32> undef, <2 x i32> zeroinitializer
   %prod = call <2 x i32> @llvm.aarch64.neon.sqrdmulh.v2i32(<2 x i32> %x, <2 x i32> %shuffle)
   %extract = extractelement <2 x i32> %prod, i64 0
   %retval =  call i32 @llvm.aarch64.neon.sqsub.i32(i32 %acc, i32 %extract)
 ; CHECK-V8a:        sqrdmulh    v0.2s, v0.2s, v1.s[0]
+; CHECK-RDM:        sqrdmlsh    v2.2s, v0.2s, v1.s[0]
 ; CHECK-V81a:       sqrdmlsh    v2.2s, v0.2s, v1.s[0]
 ; CHECK-V81a-apple: sqrdmlsh.2s v2,    v0,    v1[0]
   ret i32 %retval
 }
 
 define i32 @test_sqrdmlshq_extracted_lane_s32(i32 %acc,<4 x i32> %x, <4 x i32> %v) {
 ; CHECK-LABEL: test_sqrdmlshq_extracted_lane_s32:
 entry:
   %shuffle = shufflevector <4 x i32> %v, <4 x i32> undef, <4 x i32> zeroinitializer
   %prod = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %x, <4 x i32> %shuffle)
   %extract = extractelement <4 x i32> %prod, i64 0
   %retval =  call i32 @llvm.aarch64.neon.sqsub.i32(i32 %acc, i32 %extract)
 ; CHECK-V8a:        sqrdmulh    v0.4s, v0.4s, v1.s[0]
+; CHECK-RDM:        sqrdmlsh    v2.4s, v0.4s, v1.s[0]
 ; CHECK-V81a:       sqrdmlsh    v2.4s, v0.4s, v1.s[0]
 ; CHECK-V81a-apple: sqrdmlsh.4s v2,    v0,    v1[0]
   ret i32 %retval
 }
 
 ;-----------------------------------------------------------------------------
 ; RDMA Scalar
 ; test for "def : Pat" near SIMDThreeScalarHSTied in AArch64InstInfo.td
 
 define i16 @test_sqrdmlah_v1i16(i16 %acc, i16 %x, i16 %y) {
 ; CHECK-LABEL: test_sqrdmlah_v1i16:
   %x_vec = insertelement <4 x i16> undef, i16 %x, i64 0
   %y_vec = insertelement <4 x i16> undef, i16 %y, i64 0
   %prod_vec = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %x_vec,  <4 x i16> %y_vec)
   %acc_vec = insertelement <4 x i16> undef, i16 %acc, i64 0
   %retval_vec =  call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> %acc_vec,  <4 x i16> %prod_vec)
   %retval = extractelement <4 x i16> %retval_vec, i64 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+; CHECK-RDM:        sqrdmlah    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
 ; CHECK-V81a:       sqrdmlah    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
 ; CHECK-V81a-apple: sqrdmlah.4h {{v[0-9]+}},    {{v[0-9]+}},    {{v[0-9]+}}
   ret i16 %retval
 }
 
 define i32 @test_sqrdmlah_v1i32(i32 %acc, i32 %x, i32 %y) {
 ; CHECK-LABEL: test_sqrdmlah_v1i32:
   %x_vec = insertelement <4 x i32> undef, i32 %x, i64 0
   %y_vec = insertelement <4 x i32> undef, i32 %y, i64 0
   %prod_vec = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %x_vec,  <4 x i32> %y_vec)
   %acc_vec = insertelement <4 x i32> undef, i32 %acc, i64 0
   %retval_vec =  call <4 x i32> @llvm.aarch64.neon.sqadd.v4i32(<4 x i32> %acc_vec,  <4 x i32> %prod_vec)
   %retval = extractelement <4 x i32> %retval_vec, i64 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-RDM:        sqrdmlah    {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
 ; CHECK-V81a:       sqrdmlah    {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
 ; CHECK-V81a-apple: sqrdmlah.4s {{v[0-9]+}},    {{v[0-9]+}},    {{v[0-9]+}}
   ret i32 %retval
 }
 
 
 define i16 @test_sqrdmlsh_v1i16(i16 %acc, i16 %x, i16 %y) {
 ; CHECK-LABEL: test_sqrdmlsh_v1i16:
   %x_vec = insertelement <4 x i16> undef, i16 %x, i64 0
   %y_vec = insertelement <4 x i16> undef, i16 %y, i64 0
   %prod_vec = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %x_vec,  <4 x i16> %y_vec)
   %acc_vec = insertelement <4 x i16> undef, i16 %acc, i64 0
   %retval_vec =  call <4 x i16> @llvm.aarch64.neon.sqsub.v4i16(<4 x i16> %acc_vec,  <4 x i16> %prod_vec)
   %retval = extractelement <4 x i16> %retval_vec, i64 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
+; CHECK-RDM:        sqrdmlsh    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
 ; CHECK-V81a:       sqrdmlsh    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, {{v[0-9]+}}.4h
 ; CHECK-V81a-apple: sqrdmlsh.4h {{v[0-9]+}},    {{v[0-9]+}},    {{v[0-9]+}}
   ret i16 %retval
 }
 
 define i32 @test_sqrdmlsh_v1i32(i32 %acc, i32 %x, i32 %y) {
 ; CHECK-LABEL: test_sqrdmlsh_v1i32:
   %x_vec = insertelement <4 x i32> undef, i32 %x, i64 0
   %y_vec = insertelement <4 x i32> undef, i32 %y, i64 0
   %prod_vec = call <4 x i32> @llvm.aarch64.neon.sqrdmulh.v4i32(<4 x i32> %x_vec,  <4 x i32> %y_vec)
   %acc_vec = insertelement <4 x i32> undef, i32 %acc, i64 0
   %retval_vec =  call <4 x i32> @llvm.aarch64.neon.sqsub.v4i32(<4 x i32> %acc_vec,  <4 x i32> %prod_vec)
   %retval = extractelement <4 x i32> %retval_vec, i64 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
+; CHECK-RDM:        sqrdmlsh    {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
 ; CHECK-V81a:       sqrdmlsh    {{v[0-9]+}}.4s, {{v[0-9]+}}.4s, {{v[0-9]+}}.4s
 ; CHECK-V81a-apple: sqrdmlsh.4s {{v[0-9]+}},    {{v[0-9]+}},    {{v[0-9]+}}
   ret i32 %retval
 }
 define i32 @test_sqrdmlah_i32(i32 %acc, i32 %mhs, i32 %rhs) {
 ; CHECK-LABEL: test_sqrdmlah_i32:
   %prod = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %mhs,  i32 %rhs)
   %retval =  call i32 @llvm.aarch64.neon.sqadd.i32(i32 %acc,  i32 %prod)
 ; CHECK-V8a:        sqrdmulh {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
+; CHECK-RDM:        sqrdmlah {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 ; CHECK-V81a:       sqrdmlah {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 ; CHECK-V81a-apple: sqrdmlah {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
   ret i32 %retval
 }
 
 define i32 @test_sqrdmlsh_i32(i32 %acc, i32 %mhs, i32 %rhs) {
 ; CHECK-LABEL: test_sqrdmlsh_i32:
   %prod = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %mhs,  i32 %rhs)
   %retval =  call i32 @llvm.aarch64.neon.sqsub.i32(i32 %acc,  i32 %prod)
 ; CHECK-V8a:        sqrdmulh {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
+; CHECK-RDM:        sqrdmlsh {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 ; CHECK-V81a:       sqrdmlsh {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
 ; CHECK-V81a-apple: sqrdmlsh {{s[0-9]+}}, {{s[0-9]+}}, {{s[0-9]+}}
   ret i32 %retval
 }
 
 ;-----------------------------------------------------------------------------
 ; RDMA Scalar, by element
 ; i16 tests are performed via tests in above chapter, with IR in ACLE style
 ; i32 tests are for i32_indexed in SIMDIndexedSQRDMLxHSDTied
 
 define i16 @test_sqrdmlah_extract_i16(i16 %acc, i16 %x, <4 x i16> %y_vec) {
 ; CHECK-LABEL: test_sqrdmlah_extract_i16:
   %shuffle = shufflevector <4 x i16> %y_vec, <4 x i16> undef, <4 x i32> <i32 1,i32 1,i32 1,i32 1>
   %x_vec = insertelement <4 x i16> undef, i16 %x, i64 0
   %prod = call <4 x i16> @llvm.aarch64.neon.sqrdmulh.v4i16(<4 x i16> %x_vec, <4 x i16> %shuffle)
   %acc_vec = insertelement <4 x i16> undef, i16 %acc, i64 0
   %retval_vec =  call <4 x i16> @llvm.aarch64.neon.sqadd.v4i16(<4 x i16> %acc_vec, <4 x i16> %prod)
   %retval = extractelement <4 x i16> %retval_vec, i32 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, v0.h[1]
+; CHECK-RDM:        sqrdmlah    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, v0.h[1]
 ; CHECK-V81a:       sqrdmlah    {{v[0-9]+}}.4h, {{v[0-9]+}}.4h, v0.h[1]
 ; CHECK-V81a-apple: sqrdmlah.4h {{v[0-9]+}},    {{v[0-9]+}}, v0[1]
   ret i16 %retval
 }
 
 define i32 @test_sqrdmlah_extract_i32(i32 %acc, i32 %mhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: test_sqrdmlah_extract_i32:
   %extract = extractelement <4 x i32> %rhs, i32 3
   %prod = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %mhs,  i32 %extract)
   %retval =  call i32 @llvm.aarch64.neon.sqadd.i32(i32 %acc,  i32 %prod)
 ; CHECK-V8a:        sqrdmulh   {{s[0-9]+}}, {{s[0-9]+}}, v0.s[3]
+; CHECK-RDM:        sqrdmlah   {{s[0-9]+}}, {{s[0-9]+}}, v0.s[3]
 ; CHECK-V81a:       sqrdmlah   {{s[0-9]+}}, {{s[0-9]+}}, v0.s[3]
 ; CHECK-V81a-apple: sqrdmlah.s {{s[0-9]+}}, {{s[0-9]+}}, v0[3]
   ret i32 %retval
 }
 
 define i16 @test_sqrdmlshq_extract_i16(i16 %acc, i16 %x, <8 x i16> %y_vec) {
 ; CHECK-LABEL: test_sqrdmlshq_extract_i16:
   %shuffle = shufflevector <8 x i16> %y_vec, <8 x i16> undef, <8 x i32> <i32 1,i32 1,i32 1,i32 1,i32 1,i32 1,i32 1,i32 1>
   %x_vec = insertelement <8 x i16> undef, i16 %x, i64 0
   %prod = call <8 x i16> @llvm.aarch64.neon.sqrdmulh.v8i16(<8 x i16> %x_vec, <8 x i16> %shuffle)
   %acc_vec = insertelement <8 x i16> undef, i16 %acc, i64 0
   %retval_vec =  call <8 x i16> @llvm.aarch64.neon.sqsub.v8i16(<8 x i16> %acc_vec, <8 x i16> %prod)
   %retval = extractelement <8 x i16> %retval_vec, i32 0
 ; CHECK-V8a:        sqrdmulh    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, v0.h[1]
+; CHECK-RDM:        sqrdmlsh    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, v0.h[1]
 ; CHECK-V81a:       sqrdmlsh    {{v[0-9]+}}.8h, {{v[0-9]+}}.8h, v0.h[1]
 ; CHECK-V81a-apple: sqrdmlsh.8h {{v[0-9]+}},    {{v[0-9]+}}, v0[1]
   ret i16 %retval
 }
 
 define i32 @test_sqrdmlsh_extract_i32(i32 %acc, i32 %mhs, <4 x i32> %rhs) {
 ; CHECK-LABEL: test_sqrdmlsh_extract_i32:
   %extract = extractelement <4 x i32> %rhs, i32 3
   %prod = call i32 @llvm.aarch64.neon.sqrdmulh.i32(i32 %mhs,  i32 %extract)
   %retval =  call i32 @llvm.aarch64.neon.sqsub.i32(i32 %acc,  i32 %prod)
 ; CHECK-V8a:        sqrdmulh   {{s[0-9]+}}, {{s[0-9]+}}, v0.s[3]
+; CHECK-RDM:        sqrdmlsh   {{s[0-9]+}}, {{s[0-9]+}}, v0.s[3]
 ; CHECK-V81a:       sqrdmlsh   {{s[0-9]+}}, {{s[0-9]+}}, v0.s[3]
 ; CHECK-V81a-apple: sqrdmlsh.s {{s[0-9]+}}, {{s[0-9]+}}, v0[3]
   ret i32 %retval
 }