[AArch64] Merge [US]MULL with half adds and subs into [US]ML[AS]L

[avieira]

Summary

Hi,

This patch adds patterns to teach the AArch64 backend to merge [US]MULL
instructions and adds/subs of half the size into [US]ML[AS]L where we don't use
the top half of the result.

Is this OK for main?

Kind regards,
Andre

[dmgreen]

Sounds good to me. Can you clean up the tests a little?

llvm/lib/Target/AArch64/AArch64InstrInfo.td

4795

Maybe explain the reason for these patterns? That they are for when only the bottom half of the add is used.

4803

Can you line these up? I'm pretty sure it's off by a space.

llvm/test/CodeGen/AArch64/mla_mls_merge.ll

3

You can remove these.

4

Remove dso_local and local_unnamed_addr.

18

Hmm. These tests could probably be simplified, or simpler versions added. I don't think that the urshl/store is needed if the value is returned directly, and the loaded values could be arguments?

Can you also run update_llc_test_checks on it.

273

You can remove these too.

[avieira]

Updated the testcase. Ok to commit?

[avieira]

llvm/test/CodeGen/AArch64/mla_mls_merge.ll

273

I assume you meant the function attrs and not the intrinsic declarations.

[dmgreen]

LGTM

llvm/test/CodeGen/AArch64/mla_mls_merge.ll

273

Yep. That's great. Thanks!

[avieira]

Thanks. I'll wait till Monday to commit. To avoid Friday hit-and-run's ;)

llvm/lib/Target/AArch64/AArch64InstrInfo.td

 defm UMLSL   : SIMDLongThreeVectorTiedBHS<1, 0b1010, "umlsl",
     TriOpFrag<(sub node:$LHS, (int_aarch64_neon_umull node:$MHS, node:$RHS))>>;
 defm UMULL   : SIMDLongThreeVectorBHS<1, 0b1100, "umull", int_aarch64_neon_umull>;
 defm USUBL   : SIMDLongThreeVectorBHS<1, 0b0010, "usubl",
                  BinOpFrag<(sub (zanyext node:$LHS), (zanyext node:$RHS))>>;
 defm USUBW   : SIMDWideThreeVectorBHS<   1, 0b0011, "usubw",
                  BinOpFrag<(sub node:$LHS, (zanyext node:$RHS))>>;
 
+// Additional patterns for [SU]ML[AS]L
+multiclass Neon_mul_acc_widen_patterns<SDPatternOperator opnode, SDPatternOperator vecopnode,
+  Instruction INST8B, Instruction INST4H, Instruction INST2S> {
+  def : Pat<(v4i16 (opnode
+                    V64:$Ra,
+                    (v4i16 (extract_subvector
+                            (vecopnode (v8i8 V64:$Rn),(v8i8 V64:$Rm)),
+                            (i64 0))))),
+             (EXTRACT_SUBREG (v8i16 (INST8B
+                                     (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)), V64:$Ra, dsub),
+                                     V64:$Rn, V64:$Rm)), dsub)>;
+  def : Pat<(v2i32 (opnode
+                    V64:$Ra,
+                    (v2i32 (extract_subvector
+                            (vecopnode (v4i16 V64:$Rn),(v4i16 V64:$Rm)),
+                            (i64 0))))),
+             (EXTRACT_SUBREG (v4i32 (INST4H
+                                     (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), V64:$Ra, dsub),
+                                     V64:$Rn, V64:$Rm)), dsub)>;
+  def : Pat<(v1i64 (opnode
+                    V64:$Ra,
+                    (v1i64 (extract_subvector
+                            (vecopnode (v2i32 V64:$Rn),(v2i32 V64:$Rm)),
+                            (i64 0))))),
+             (EXTRACT_SUBREG (v2i64 (INST2S
+                                     (INSERT_SUBREG (v2i64 (IMPLICIT_DEF)), V64:$Ra, dsub),
+                                     V64:$Rn, V64:$Rm)), dsub)>;
+}
+
+defm : Neon_mul_acc_widen_patterns<add, int_aarch64_neon_umull,
+     UMLALv8i8_v8i16, UMLALv4i16_v4i32, UMLALv2i32_v2i64>;
+defm : Neon_mul_acc_widen_patterns<add, int_aarch64_neon_smull,
+     SMLALv8i8_v8i16, SMLALv4i16_v4i32, SMLALv2i32_v2i64>;
+defm : Neon_mul_acc_widen_patterns<sub, int_aarch64_neon_umull,
+     UMLSLv8i8_v8i16, UMLSLv4i16_v4i32, UMLSLv2i32_v2i64>;
+defm : Neon_mul_acc_widen_patterns<sub, int_aarch64_neon_smull,
+     SMLSLv8i8_v8i16, SMLSLv4i16_v4i32, SMLSLv2i32_v2i64>;
+
 // Additional patterns for SMULL and UMULL
 multiclass Neon_mul_widen_patterns<SDPatternOperator opnode,
   Instruction INST8B, Instruction INST4H, Instruction INST2S> {
   def : Pat<(v8i16 (opnode (v8i8 V64:$Rn), (v8i8 V64:$Rm))),
             (INST8B V64:$Rn, V64:$Rm)>;
   def : Pat<(v4i32 (opnode (v4i16 V64:$Rn), (v4i16 V64:$Rm))),
             (INST4H V64:$Rn, V64:$Rm)>;
   def : Pat<(v2i64 (opnode (v2i32 V64:$Rn), (v2i32 V64:$Rm))),

llvm/test/CodeGen/AArch64/mla_mls_merge.ll

+; RUN: llc -verify-machineinstrs < %s -mtriple=aarch64-unknown-linux-gnu | FileCheck %s
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mla0(<8 x i8>* nocapture readonly %a, <8 x i8>* nocapture readonly %b, i16* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <8 x i8>, <8 x i8>* %a, align 8
+  %1 = load <8 x i8>, <8 x i8>* %b, align 8
+  %vmull.i = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %0, <8 x i8> %1) #2
+  %arrayidx2 = getelementptr inbounds <8 x i8>, <8 x i8>* %a, i64 1
+  %2 = load <8 x i8>, <8 x i8>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <8 x i8>, <8 x i8>* %b, i64 1
+  %3 = load <8 x i8>, <8 x i8>* %arrayidx3, align 8
+  %vmull.i.i = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %2, <8 x i8> %3) #2
+  %add.i = add <8 x i16> %vmull.i.i, %vmull.i
+  %shuffle.i = shufflevector <8 x i16> %add.i, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %vrshr_n6 = tail call <4 x i16> @llvm.aarch64.neon.urshl.v4i16(<4 x i16> %shuffle.i, <4 x i16> <i16 -2, i16 -2, i16 -2, i16 -2>)
+  %4 = bitcast i16* %dest to <4 x i16>*
+  store <4 x i16> %vrshr_n6, <4 x i16>* %4, align 2
+  ret void
+}
+; CHECK-LABEL: test_mla0
+; CHECK: umlal v{{[0-9]+}}.8h
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <4 x i16> @llvm.aarch64.neon.urshl.v4i16(<4 x i16>, <4 x i16>)
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mla1(<8 x i8>* nocapture readonly %a, <8 x i8>* nocapture readonly %b, i16* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <8 x i8>, <8 x i8>* %a, align 8
+  %1 = load <8 x i8>, <8 x i8>* %b, align 8
+  %vmull.i = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %0, <8 x i8> %1) #2
+  %arrayidx2 = getelementptr inbounds <8 x i8>, <8 x i8>* %a, i64 1
+  %2 = load <8 x i8>, <8 x i8>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <8 x i8>, <8 x i8>* %b, i64 1
+  %3 = load <8 x i8>, <8 x i8>* %arrayidx3, align 8
+  %vmull.i.i = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %2, <8 x i8> %3) #2
+  %add.i = add <8 x i16> %vmull.i.i, %vmull.i
+  %shuffle.i = shufflevector <8 x i16> %add.i, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %vrshr_n6 = tail call <4 x i16> @llvm.aarch64.neon.srshl.v4i16(<4 x i16> %shuffle.i, <4 x i16> <i16 -2, i16 -2, i16 -2, i16 -2>)
+  %4 = bitcast i16* %dest to <4 x i16>*
+  store <4 x i16> %vrshr_n6, <4 x i16>* %4, align 2
+  ret void
+}
+; CHECK-LABEL: test_mla1
+; CHECK: smlal v{{[0-9]+}}.8h
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <4 x i16> @llvm.aarch64.neon.srshl.v4i16(<4 x i16>, <4 x i16>)
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mla2(<4 x i16>* nocapture readonly %a, <4 x i16>* nocapture readonly %b, i32* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <4 x i16>, <4 x i16>* %a, align 8
+  %1 = load <4 x i16>, <4 x i16>* %b, align 8
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %0, <4 x i16> %1) #2
+  %arrayidx2 = getelementptr inbounds <4 x i16>, <4 x i16>* %a, i64 1
+  %2 = load <4 x i16>, <4 x i16>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <4 x i16>, <4 x i16>* %b, i64 1
+  %3 = load <4 x i16>, <4 x i16>* %arrayidx3, align 8
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %2, <4 x i16> %3) #2
+  %add.i = add <4 x i32> %vmull2.i.i, %vmull2.i
+  %shuffle.i = shufflevector <4 x i32> %add.i, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %vrshr_n6 = tail call <2 x i32> @llvm.aarch64.neon.urshl.v2i32(<2 x i32> %shuffle.i, <2 x i32> <i32 -2, i32 -2>)
+  %4 = bitcast i32* %dest to <2 x i32>*
+  store <2 x i32> %vrshr_n6, <2 x i32>* %4, align 4
+  ret void
+}
+; CHECK-LABEL: test_mla2
+; CHECK: umlal v{{[0-9]+}}.4s
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <2 x i32> @llvm.aarch64.neon.urshl.v2i32(<2 x i32>, <2 x i32>)
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mla3(<4 x i16>* nocapture readonly %a, <4 x i16>* nocapture readonly %b, i32* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <4 x i16>, <4 x i16>* %a, align 8
+  %1 = load <4 x i16>, <4 x i16>* %b, align 8
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %0, <4 x i16> %1) #2
+  %arrayidx2 = getelementptr inbounds <4 x i16>, <4 x i16>* %a, i64 1
+  %2 = load <4 x i16>, <4 x i16>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <4 x i16>, <4 x i16>* %b, i64 1
+  %3 = load <4 x i16>, <4 x i16>* %arrayidx3, align 8
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %2, <4 x i16> %3) #2
+  %add.i = add <4 x i32> %vmull2.i.i, %vmull2.i
+  %shuffle.i = shufflevector <4 x i32> %add.i, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %vrshr_n6 = tail call <2 x i32> @llvm.aarch64.neon.srshl.v2i32(<2 x i32> %shuffle.i, <2 x i32> <i32 -2, i32 -2>)
+  %4 = bitcast i32* %dest to <2 x i32>*
+  store <2 x i32> %vrshr_n6, <2 x i32>* %4, align 4
+  ret void
+}
+; CHECK-LABEL: test_mla3
+; CHECK: smlal v{{[0-9]+}}.4s
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <2 x i32> @llvm.aarch64.neon.srshl.v2i32(<2 x i32>, <2 x i32>)
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mla4(<2 x i32>* nocapture readonly %a, <2 x i32>* nocapture readonly %b, i64* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %a, align 8
+  %1 = load <2 x i32>, <2 x i32>* %b, align 8
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %0, <2 x i32> %1) #2
+  %arrayidx2 = getelementptr inbounds <2 x i32>, <2 x i32>* %a, i64 1
+  %2 = load <2 x i32>, <2 x i32>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <2 x i32>, <2 x i32>* %b, i64 1
+  %3 = load <2 x i32>, <2 x i32>* %arrayidx3, align 8
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %2, <2 x i32> %3) #2
+  %add.i = add <2 x i64> %vmull2.i.i, %vmull2.i
+  %shuffle.i = shufflevector <2 x i64> %add.i, <2 x i64> undef, <1 x i32> zeroinitializer
+  %vrshr_n6 = tail call <1 x i64> @llvm.aarch64.neon.urshl.v1i64(<1 x i64> %shuffle.i, <1 x i64> <i64 -2>)
+  %4 = bitcast i64* %dest to <1 x i64>*
+  store <1 x i64> %vrshr_n6, <1 x i64>* %4, align 8
+  ret void
+}
+; CHECK-LABEL: test_mla4
+; CHECK: umlal v{{[0-9]+}}.2d
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <1 x i64> @llvm.aarch64.neon.urshl.v1i64(<1 x i64>, <1 x i64>)
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mla5(<2 x i32>* nocapture readonly %a, <2 x i32>* nocapture readonly %b, i64* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %a, align 8
+  %1 = load <2 x i32>, <2 x i32>* %b, align 8
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %0, <2 x i32> %1) #2
+  %arrayidx2 = getelementptr inbounds <2 x i32>, <2 x i32>* %a, i64 1
+  %2 = load <2 x i32>, <2 x i32>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <2 x i32>, <2 x i32>* %b, i64 1
+  %3 = load <2 x i32>, <2 x i32>* %arrayidx3, align 8
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %2, <2 x i32> %3) #2
+  %add.i = add <2 x i64> %vmull2.i.i, %vmull2.i
+  %shuffle.i = shufflevector <2 x i64> %add.i, <2 x i64> undef, <1 x i32> zeroinitializer
+  %vrshr_n6 = tail call <1 x i64> @llvm.aarch64.neon.srshl.v1i64(<1 x i64> %shuffle.i, <1 x i64> <i64 -2>)
+  %4 = bitcast i64* %dest to <1 x i64>*
+  store <1 x i64> %vrshr_n6, <1 x i64>* %4, align 8
+  ret void
+}
+; CHECK-LABEL: test_mla5
+; CHECK: smlal v{{[0-9]+}}.2d
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <1 x i64> @llvm.aarch64.neon.srshl.v1i64(<1 x i64>, <1 x i64>)
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mls0(<8 x i8>* nocapture readonly %a, <8 x i8>* nocapture readonly %b, i16* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <8 x i8>, <8 x i8>* %a, align 8
+  %1 = load <8 x i8>, <8 x i8>* %b, align 8
+  %vmull.i = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %0, <8 x i8> %1) #2
+  %arrayidx2 = getelementptr inbounds <8 x i8>, <8 x i8>* %a, i64 1
+  %2 = load <8 x i8>, <8 x i8>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <8 x i8>, <8 x i8>* %b, i64 1
+  %3 = load <8 x i8>, <8 x i8>* %arrayidx3, align 8
+  %vmull.i.i = tail call <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8> %2, <8 x i8> %3) #2
+  %sub.i = sub <8 x i16> %vmull.i, %vmull.i.i
+  %shuffle.i = shufflevector <8 x i16> %sub.i, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %vrshr_n6 = tail call <4 x i16> @llvm.aarch64.neon.urshl.v4i16(<4 x i16> %shuffle.i, <4 x i16> <i16 -2, i16 -2, i16 -2, i16 -2>)
+  %4 = bitcast i16* %dest to <4 x i16>*
+  store <4 x i16> %vrshr_n6, <4 x i16>* %4, align 2
+  ret void
+}
+; CHECK-LABEL: test_mls0
+; CHECK: umlsl v{{[0-9]+}}.8h
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mls1(<8 x i8>* nocapture readonly %a, <8 x i8>* nocapture readonly %b, i16* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <8 x i8>, <8 x i8>* %a, align 8
+  %1 = load <8 x i8>, <8 x i8>* %b, align 8
+  %vmull.i = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %0, <8 x i8> %1) #2
+  %arrayidx2 = getelementptr inbounds <8 x i8>, <8 x i8>* %a, i64 1
+  %2 = load <8 x i8>, <8 x i8>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <8 x i8>, <8 x i8>* %b, i64 1
+  %3 = load <8 x i8>, <8 x i8>* %arrayidx3, align 8
+  %vmull.i.i = tail call <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8> %2, <8 x i8> %3) #2
+  %sub.i = sub <8 x i16> %vmull.i, %vmull.i.i
+  %shuffle.i = shufflevector <8 x i16> %sub.i, <8 x i16> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+  %vrshr_n6 = tail call <4 x i16> @llvm.aarch64.neon.srshl.v4i16(<4 x i16> %shuffle.i, <4 x i16> <i16 -2, i16 -2, i16 -2, i16 -2>)
+  %4 = bitcast i16* %dest to <4 x i16>*
+  store <4 x i16> %vrshr_n6, <4 x i16>* %4, align 2
+  ret void
+}
+; CHECK-LABEL: test_mls1
+; CHECK: smlsl v{{[0-9]+}}.8h
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mls2(<4 x i16>* nocapture readonly %a, <4 x i16>* nocapture readonly %b, i32* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <4 x i16>, <4 x i16>* %a, align 8
+  %1 = load <4 x i16>, <4 x i16>* %b, align 8
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %0, <4 x i16> %1) #2
+  %arrayidx2 = getelementptr inbounds <4 x i16>, <4 x i16>* %a, i64 1
+  %2 = load <4 x i16>, <4 x i16>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <4 x i16>, <4 x i16>* %b, i64 1
+  %3 = load <4 x i16>, <4 x i16>* %arrayidx3, align 8
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16> %2, <4 x i16> %3) #2
+  %sub.i = sub <4 x i32> %vmull2.i, %vmull2.i.i
+  %shuffle.i = shufflevector <4 x i32> %sub.i, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %vrshr_n6 = tail call <2 x i32> @llvm.aarch64.neon.urshl.v2i32(<2 x i32> %shuffle.i, <2 x i32> <i32 -2, i32 -2>)
+  %4 = bitcast i32* %dest to <2 x i32>*
+  store <2 x i32> %vrshr_n6, <2 x i32>* %4, align 4
+  ret void
+}
+; CHECK-LABEL: test_mls2
+; CHECK: umlsl v{{[0-9]+}}.4s
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mls3(<4 x i16>* nocapture readonly %a, <4 x i16>* nocapture readonly %b, i32* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <4 x i16>, <4 x i16>* %a, align 8
+  %1 = load <4 x i16>, <4 x i16>* %b, align 8
+  %vmull2.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %0, <4 x i16> %1) #2
+  %arrayidx2 = getelementptr inbounds <4 x i16>, <4 x i16>* %a, i64 1
+  %2 = load <4 x i16>, <4 x i16>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <4 x i16>, <4 x i16>* %b, i64 1
+  %3 = load <4 x i16>, <4 x i16>* %arrayidx3, align 8
+  %vmull2.i.i = tail call <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16> %2, <4 x i16> %3) #2
+  %sub.i = sub <4 x i32> %vmull2.i, %vmull2.i.i
+  %shuffle.i = shufflevector <4 x i32> %sub.i, <4 x i32> undef, <2 x i32> <i32 0, i32 1>
+  %vrshr_n6 = tail call <2 x i32> @llvm.aarch64.neon.srshl.v2i32(<2 x i32> %shuffle.i, <2 x i32> <i32 -2, i32 -2>)
+  %4 = bitcast i32* %dest to <2 x i32>*
+  store <2 x i32> %vrshr_n6, <2 x i32>* %4, align 4
+  ret void
+}
+; CHECK-LABEL: test_mls3
+; CHECK: smlsl v{{[0-9]+}}.4s
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mls4(<2 x i32>* nocapture readonly %a, <2 x i32>* nocapture readonly %b, i64* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %a, align 8
+  %1 = load <2 x i32>, <2 x i32>* %b, align 8
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %0, <2 x i32> %1) #2
+  %arrayidx2 = getelementptr inbounds <2 x i32>, <2 x i32>* %a, i64 1
+  %2 = load <2 x i32>, <2 x i32>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <2 x i32>, <2 x i32>* %b, i64 1
+  %3 = load <2 x i32>, <2 x i32>* %arrayidx3, align 8
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32> %2, <2 x i32> %3) #2
+  %sub.i = sub <2 x i64> %vmull2.i, %vmull2.i.i
+  %shuffle.i = shufflevector <2 x i64> %sub.i, <2 x i64> undef, <1 x i32> zeroinitializer
+  %vrshr_n6 = tail call <1 x i64> @llvm.aarch64.neon.urshl.v1i64(<1 x i64> %shuffle.i, <1 x i64> <i64 -2>)
+  %4 = bitcast i64* %dest to <1 x i64>*
+  store <1 x i64> %vrshr_n6, <1 x i64>* %4, align 8
+  ret void
+}
+; CHECK-LABEL: test_mls4
+; CHECK: umlsl v{{[0-9]+}}.2d
+
+; Function Attrs: nofree nounwind uwtable
+define dso_local void @test_mls5(<2 x i32>* nocapture readonly %a, <2 x i32>* nocapture readonly %b, i64* nocapture %dest) local_unnamed_addr {
+entry:
+  %0 = load <2 x i32>, <2 x i32>* %a, align 8
+  %1 = load <2 x i32>, <2 x i32>* %b, align 8
+  %vmull2.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %0, <2 x i32> %1) #2
+  %arrayidx2 = getelementptr inbounds <2 x i32>, <2 x i32>* %a, i64 1
+  %2 = load <2 x i32>, <2 x i32>* %arrayidx2, align 8
+  %arrayidx3 = getelementptr inbounds <2 x i32>, <2 x i32>* %b, i64 1
+  %3 = load <2 x i32>, <2 x i32>* %arrayidx3, align 8
+  %vmull2.i.i = tail call <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32> %2, <2 x i32> %3) #2
+  %sub.i = sub <2 x i64> %vmull2.i, %vmull2.i.i
+  %shuffle.i = shufflevector <2 x i64> %sub.i, <2 x i64> undef, <1 x i32> zeroinitializer
+  %vrshr_n6 = tail call <1 x i64> @llvm.aarch64.neon.srshl.v1i64(<1 x i64> %shuffle.i, <1 x i64> <i64 -2>)
+  %4 = bitcast i64* %dest to <1 x i64>*
+  store <1 x i64> %vrshr_n6, <1 x i64>* %4, align 8
+  ret void
+}
+; CHECK-LABEL: test_mls5
+; CHECK: smlsl v{{[0-9]+}}.2d
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <8 x i16> @llvm.aarch64.neon.umull.v8i16(<8 x i8>, <8 x i8>)
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <8 x i16> @llvm.aarch64.neon.smull.v8i16(<8 x i8>, <8 x i8>)
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <4 x i32> @llvm.aarch64.neon.umull.v4i32(<4 x i16>, <4 x i16>)
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <4 x i32> @llvm.aarch64.neon.smull.v4i32(<4 x i16>, <4 x i16>)
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <2 x i64> @llvm.aarch64.neon.umull.v2i64(<2 x i32>, <2 x i32>)
+
+; Function Attrs: nofree nosync nounwind readnone willreturn
+declare <2 x i64> @llvm.aarch64.neon.smull.v2i64(<2 x i32>, <2 x i32>)
+