[ARM] Remove duplicates in ParallelDSP

[samparker]

Summary

While scanning a block for sequential loads, also check for duplicate instructions and remove them if possible. To do this we have to ensure that for any memory reads, there aren't any writes between the two duplicates and also that memory operations are simple. I've also moved all the tests into a ParallelDSP directory.

[efriedma]

When does this come up, realistically? EarlyCSE already has a stronger form of load CSE.

[samparker]

This happens because of unroll and jam and I had tried to run cse after that... but doing the more obvious option of running it before parallel dsp solves the problem! I'll do that instead. Thanks.

lib/Target/ARM/ARMParallelDSP.cpp

class ARMParallelDSP : public LoopPass {
     AliasAnalysis     *AA;
     TargetLibraryInfo *TLI;
     DominatorTree     *DT;
     LoopInfo          *LI;
     Loop              *L;
     const DataLayout  *DL;
     Module            *M;
     std::map<LoadInst*, LoadInst*> LoadPairs;
-    std::map<LoadInst*, SmallVector<LoadInst*, 4>> SequentialLoads;
 
-    bool RecordSequentialLoads(BasicBlock *Header);
+    bool PrepareForBlock(BasicBlock *BB);
     bool InsertParallelMACs(Reduction &Reduction);
     bool AreSequentialLoads(LoadInst *Ld0, LoadInst *Ld1, MemInstList &VecMem);
     void CreateParallelMACPairs(Reduction &R);
     Instruction *CreateSMLADCall(LoadInst *VecLd0, LoadInst *VecLd1,
                                  Instruction *Acc, bool Exchange,
                                  Instruction *InsertAfter);
 
     /// Try to match and generate: SMLAD, SMLADX - Signed Multiply Accumulate
bool runOnLoop(Loop *TheLoop, LPPassManager &) override {
       }
 
       LoopAccessInfo LAI(L, SE, TLI, AA, DT, LI);
       bool Changes = false;
 
       LLVM_DEBUG(dbgs() << "\n== Parallel DSP pass ==\n");
       LLVM_DEBUG(dbgs() << " - " << F.getName() << "\n\n");
 
-      if (!RecordSequentialLoads(Header)) {
+      Changes = PrepareForBlock(Header);
+      if (LoadPairs.size() < 1) {
         LLVM_DEBUG(dbgs() << " - No sequential loads found.\n");
-        return false;
+        return Changes;
       }
 
-      Changes = MatchSMLAD(F);
+      Changes |= MatchSMLAD(F);
       return Changes;
     }
   };
 }
 
 // MaxBitwidth: the maximum supported bitwidth of the elements in the DSP
 // instructions, which is set to 16. So here we should collect all i8 and i16
 // narrow operations.
static bool IsNarrowSequence(Value *V, ValueList &VL) {
   }
   LLVM_DEBUG(dbgs() << "No, unsupported Op:\t"; I->dump());
   return false;
 }
 
 template<typename MemInst>
 static bool AreSequentialAccesses(MemInst *MemOp0, MemInst *MemOp1,
                                   const DataLayout &DL, ScalarEvolution &SE) {
-  if (!MemOp0->isSimple() || !MemOp1->isSimple()) {
-    LLVM_DEBUG(dbgs() << "No, not touching volatile access\n");
-    return false;
-  }
-  if (isConsecutiveAccess(MemOp0, MemOp1, DL, SE)) {
-    LLVM_DEBUG(dbgs() << "OK: accesses are consecutive.\n");
+  if (isConsecutiveAccess(MemOp0, MemOp1, DL, SE))
     return true;
-  }
-  LLVM_DEBUG(dbgs() << "No, accesses aren't consecutive.\n");
   return false;
 }
 
 bool ARMParallelDSP::AreSequentialLoads(LoadInst *Ld0, LoadInst *Ld1,
                                         MemInstList &VecMem) {
   if (!Ld0 || !Ld1)
     return false;
 
-  LLVM_DEBUG(dbgs() << "Are consecutive loads:\n";
-    dbgs() << "Ld0:"; Ld0->dump();
-    dbgs() << "Ld1:"; Ld1->dump();
-  );
+  // Expect that the load is only used by one extending instruction.
+  if (!Ld0->hasOneUse() || !Ld1->hasOneUse())
+    return false;
 
-  if (!Ld0->hasOneUse() || !Ld1->hasOneUse()) {
-    LLVM_DEBUG(dbgs() << "No, load has more than one use.\n");
+  // The extending instruction should also only have a single user.
+  if (!Ld0->user_back()->hasOneUse() || !Ld1->user_back()->hasOneUse())
     return false;
-  }
 
   if (!LoadPairs.count(Ld0) || LoadPairs[Ld0] != Ld1)
     return false;
 
+  LLVM_DEBUG(dbgs() << "Loads are sequential and valid:\n";
+    dbgs() << "Ld0:"; Ld0->dump();
+    dbgs() << "Ld1:"; Ld1->dump();
+  );
+
   VecMem.clear();
   VecMem.push_back(Ld0);
   VecMem.push_back(Ld1);
   return true;
 }
 
 /// Iterate through the block and record base, offset pairs of loads as well as
 /// maximal sequences of sequential loads.
-bool ARMParallelDSP::RecordSequentialLoads(BasicBlock *Header) {
-  SmallVector<LoadInst*, 8> Loads;
-  for (auto &I : *Header) {
+bool ARMParallelDSP::PrepareForBlock(BasicBlock *BB) {
+  SetVector<LoadInst*> Loads;
+  SmallPtrSet<Instruction*, 8> Writes;
+  for (auto &I : *BB) {
+    if (I.mayWriteToMemory())
+      Writes.insert(&I);
     auto *Ld = dyn_cast<LoadInst>(&I);
-    if (!Ld)
+    if (!Ld || !Ld->isSimple())
       continue;
-    Loads.push_back(Ld);
+    Loads.insert(Ld);
   }
 
-  std::map<LoadInst*, LoadInst*> BaseLoads;
+  // Record any loads which have their address written before they read.
+  const auto Size = LocationSize::unknown();
+  SmallPtrSet<Instruction*, 8> UnsafeToRemove;
+  for (auto Write : Writes) {
+    for (auto Read : Loads) {
+      if (!DT->dominates(Write, Read))
+        continue;
+      MemoryLocation ReadLoc =
+        MemoryLocation(Read->getPointerOperand(), Size);
+      if (isModOrRefSet(intersectModRef(AA->getModRefInfo(Write, ReadLoc),
+                                        ModRefInfo::ModRef)))
+        UnsafeToRemove.insert(Read);
+    }
+  }
+
+  auto DontRemove = [&UnsafeToRemove](Instruction *I) {
+    if (UnsafeToRemove.count(I) || !I->isSafeToRemove())
+      return true;
+    if (auto *Ld = dyn_cast<LoadInst>(I))
+      return !Ld->isSimple();
+    if (auto *St = dyn_cast<StoreInst>(I))
+      return !St->isSimple();
+    return false;
+  };
+
+  // Scan the block for duplicate instructions that can be safely replaced with
+  // a dominating instruction.
+  SmallPtrSet<Instruction*, 4> ToRemove;
+  SmallPtrSet<Instruction*, 4> ToKeep;
+  for (auto &DomI : *BB) {
+    for (auto &I : *BB) {
+      if (&DomI == &I || ToRemove.count(&DomI) || ToRemove.count(&I) ||
+          ToKeep.count(&I))
+        continue;
+      if (DontRemove(&DomI) || DontRemove(&I))
+        continue;
+      if (I.isIdenticalTo(&DomI)) {
+        I.replaceAllUsesWith(&DomI);
+        ToRemove.insert(&I);
+        ToKeep.insert(&DomI);
+      }
+    }
+  }
+
+  bool Changed = !ToRemove.empty();
+  for (auto *I : ToRemove) {
+    if (auto *Ld = dyn_cast<LoadInst>(I))
+      Loads.remove(Ld);
+    LLVM_DEBUG(dbgs() << "Removing: " << *I << "\n");
+    I->dropAllReferences();
+    I->eraseFromParent();
+  }
 
+  // Record sequential load pairs.
   for (auto *Ld0 : Loads) {
     for (auto *Ld1 : Loads) {
       if (Ld0 == Ld1)
         continue;
 
       if (AreSequentialAccesses<LoadInst>(Ld0, Ld1, *DL, *SE)) {
         LoadPairs[Ld0] = Ld1;
-        if (BaseLoads.count(Ld0)) {
-          LoadInst *Base = BaseLoads[Ld0];
-          BaseLoads[Ld1] = Base;
-          SequentialLoads[Base].push_back(Ld1);
-        } else {
-          BaseLoads[Ld1] = Ld0;
-          SequentialLoads[Ld0].push_back(Ld1);
-        }
-      }
-    }
+        break;
+
+  LLVM_DEBUG(if (!LoadPairs.empty()) {
+               dbgs() << "Consecutive load pairs:\n";
+               for (auto &MapIt : LoadPairs) {
+                 LLVM_DEBUG(dbgs() << *MapIt.first << ", "
+                            << *MapIt.second << "\n");
                }
-  return LoadPairs.size() > 1;
+             });
+  return Changed;
 }
 
 void ARMParallelDSP::CreateParallelMACPairs(Reduction &R) {
   OpChainList &Candidates = R.MACCandidates;
   PMACPairList &PMACPairs = R.PMACPairs;
   const unsigned Elems = Candidates.size();
 
   if (Elems < 2)

test/CodeGen/ARM/ParallelDSP/remove-duplicate-loads.ll

+; RUN: opt -mtriple=thumbv7em -arm-parallel-dsp -verify %s -S -o - | FileCheck %s
+
+; CHECK-LABEL: remove_duplicate_load
+define void @remove_duplicate_load(i32* %res, i16* %A, i32 %N) {
+entry:
+  br label %for.body
+
+for.body:
+; CHECK: for.body
+; CHECK: load i16, i16* %ld.addr
+; CHECK: load i16, i16* %ld1.addr
+; CHECK-NOT: load
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %for.body ]
+  %acc = phi i32 [ 0, %entry ], [ %acc.next, %for.body ]
+  %ld.addr = getelementptr inbounds i16, i16* %A, i32 %idx
+  %ld = load i16, i16* %ld.addr
+  %sext = sext i16 %ld to i32
+  %idx.1 = add nuw nsw i32 %idx, 1
+  %ld1.addr = getelementptr inbounds i16, i16* %A, i32 %idx.1
+  %ld1 = load i16, i16* %ld1.addr
+  %sext.1 = sext i16 %ld1 to i32
+  %mul = mul nsw i32 %sext, %sext.1
+  %idx.1.again = add nuw nsw i32 %idx, 1
+  %ld1.addr.again = getelementptr inbounds i16, i16* %A, i32 %idx.1.again
+  %ld1.again = load i16, i16* %ld1.addr.again
+  %sext.again = sext i16 %ld1.again to i32
+  %add = add i32 %mul, %sext.again
+  %acc.next = add i32 %add, %acc
+  %idx.next = add nuw nsw i32 %idx, 2
+  %cmp = icmp ult i32 %idx.next, %N
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  store i32 %acc.next, i32* %res
+  ret void
+}
+
+; CHECK-LABEL: dont_remove_duplicate_load_store
+define void @dont_remove_duplicate_load_store(i32* %res, i16* %A, i32 %N) {
+entry:
+  br label %for.body
+
+for.body:
+; CHECK: for.body
+; CHECK: %sext.again = sext i16 %ld1.again to i32
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %for.body ]
+  %acc = phi i32 [ 0, %entry ], [ %acc.next, %for.body ]
+  %ld.addr = getelementptr inbounds i16, i16* %A, i32 %idx
+  %ld = load i16, i16* %ld.addr
+  %sext = sext i16 %ld to i32
+  %idx.1 = add nuw nsw i32 %idx, 1
+  %ld1.addr = getelementptr inbounds i16, i16* %A, i32 %idx.1
+  %ld1 = load i16, i16* %ld1.addr
+  %sext.1 = sext i16 %ld1 to i32
+  %mul = mul nsw i32 %sext, %sext.1
+  %idx.1.again = add nuw nsw i32 %idx, 1
+  %trunc = trunc i32 %mul to i16
+  store i16 %trunc, i16* %ld1.addr
+  %ld1.addr.again = getelementptr inbounds i16, i16* %A, i32 %idx.1.again
+  %ld1.again = load i16, i16* %ld1.addr.again
+  %sext.again = sext i16 %ld1.again to i32
+  %add = add i32 %mul, %sext.again
+  %acc.next = add i32 %add, %acc
+  %idx.next = add nuw nsw i32 %idx, 2
+  %cmp = icmp ult i32 %idx.next, %N
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  store i32 %acc.next, i32* %res
+  ret void
+}
+
+; CHECK-LABEL: dont_remove_duplicate_volatile_load
+define void @dont_remove_duplicate_volatile_load(i32* %res, i16* %A, i32 %N) {
+entry:
+  br label %for.body
+
+for.body:
+; CHECK: for.body
+; CHECK: %sext.again = sext i16 %ld1.again to i32
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %for.body ]
+  %acc = phi i32 [ 0, %entry ], [ %acc.next, %for.body ]
+  %ld.addr = getelementptr inbounds i16, i16* %A, i32 %idx
+  %ld = load i16, i16* %ld.addr
+  %sext = sext i16 %ld to i32
+  %idx.1 = add nuw nsw i32 %idx, 1
+  %ld1.addr = getelementptr inbounds i16, i16* %A, i32 %idx.1
+  %ld1 = load i16, i16* %ld1.addr
+  %sext.1 = sext i16 %ld1 to i32
+  %mul = mul nsw i32 %sext, %sext.1
+  %idx.1.again = add nuw nsw i32 %idx, 1
+  %ld1.addr.again = getelementptr inbounds i16, i16* %A, i32 %idx.1.again
+  %ld1.again = load volatile i16, i16* %ld1.addr.again
+  %sext.again = sext i16 %ld1.again to i32
+  %add = add i32 %mul, %sext.again
+  %acc.next = add i32 %add, %acc
+  %idx.next = add nuw nsw i32 %idx, 2
+  %cmp = icmp ult i32 %idx.next, %N
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  store i32 %acc.next, i32* %res
+  ret void
+}
+
+; CHECK-LABEL: dont_remove_duplicate_load_call
+define void @dont_remove_duplicate_load_call(i32* %res, i16* %A, i32 %N) {
+entry:
+  br label %for.body
+
+for.body:
+; CHECK: for.body
+; CHECK: %sext.again = sext i16 %ld1.again to i32
+  %idx = phi i32 [ 0, %entry ], [ %idx.next, %for.body ]
+  %acc = phi i32 [ 0, %entry ], [ %acc.next, %for.body ]
+  %ld.addr = getelementptr inbounds i16, i16* %A, i32 %idx
+  %ld = load i16, i16* %ld.addr
+  %sext = sext i16 %ld to i32
+  %idx.1 = add nuw nsw i32 %idx, 1
+  %ld1.addr = getelementptr inbounds i16, i16* %A, i32 %idx.1
+  %ld1 = load i16, i16* %ld1.addr
+  %sext.1 = sext i16 %ld1 to i32
+  %mul = mul nsw i32 %sext, %sext.1
+  %idx.1.again = add nuw nsw i32 %idx, 1
+  call void @capture_ptr(i16* %ld1.addr)
+  %ld1.addr.again = getelementptr inbounds i16, i16* %A, i32 %idx.1.again
+  %ld1.again = load i16, i16* %ld1.addr.again
+  %sext.again = sext i16 %ld1.again to i32
+  %add = add i32 %mul, %sext.again
+  %acc.next = add i32 %add, %acc
+  %idx.next = add nuw nsw i32 %idx, 2
+  %cmp = icmp ult i32 %idx.next, %N
+  br i1 %cmp, label %for.body, label %exit
+
+exit:
+  store i32 %acc.next, i32* %res
+  ret void
+}
+
+declare void @capture_ptr(i16*)
+
+; CHECK-LABEL: remove_duplicate_loads_smlad
+define void @remove_duplicate_loads_smlad(i32* %res, i16* %A, i16* %B, i32 %N, i32 %idx) {
+entry:
+  %xtraiter306.i = and i32 %N, 3
+  %unroll_iter310.i = sub i32 %N, %xtraiter306.i
+  %arrayidx.us.i117.i = getelementptr inbounds i32, i32* %res, i32 %idx
+  store i32 0, i32* %arrayidx.us.i117.i, align 4
+  %mul.us.i118.i = mul i32 %idx, %N
+  %inc11.us.i.i = or i32 %idx, 1
+  %arrayidx.us.i117.1.i = getelementptr inbounds i32, i32* %res, i32 %inc11.us.i.i
+  store i32 0, i32* %arrayidx.us.i117.1.i, align 4
+  %mul.us.i118.1.i = mul i32 %inc11.us.i.i, %N
+  %inc11.us.i.1.i = or i32 %idx, 2
+  %arrayidx.us.i117.2.i = getelementptr inbounds i32, i32* %res, i32 %inc11.us.i.1.i
+  store i32 0, i32* %arrayidx.us.i117.2.i, align 4
+  %mul.us.i118.2.i = mul i32 %inc11.us.i.1.i, %N
+  %inc11.us.i.2.i = or i32 %idx, 3
+  %arrayidx.us.i117.3.i = getelementptr inbounds i32, i32* %res, i32 %inc11.us.i.2.i
+  store i32 0, i32* %arrayidx.us.i117.3.i, align 4
+  %mul.us.i118.3.i = mul i32 %inc11.us.i.2.i, %N
+  %inc11.us.i.3.i = add i32 %idx, 4
+  br label %for.body
+
+; CHECK: %for.body
+; CHECK: %A7 = load i16, i16* %arrayidx4.us.i.i, align 2
+; CHECK: %A8 = load i16, i16* %arrayidx5.us.i.i, align 2
+; CHECK: %A9 = load i16, i16* %arrayidx4.us.i.1.i, align 2
+; CHECK: %B1 = load i16, i16* %arrayidx4.us.i.2.i, align 2
+; CHECK: %B3 = load i16, i16* %arrayidx4.us.i.3.i, align 2
+; CHECK: %B5 = load i16, i16* %arrayidx4.us.i.1338.i, align 2
+; CHECK: %B6 = load i16, i16* %arrayidx5.us.i.1340.i, align 2
+; CHECK: %B7 = load i16, i16* %arrayidx4.us.i.1.1.i, align 2
+; CHECK: %B9 = load i16, i16* %arrayidx4.us.i.2.1.i, align 2
+; CHECK: %B11 = load i16, i16* %arrayidx4.us.i.3.1.i, align 2
+; CHECK: %B13 = load i16, i16* %arrayidx4.us.i.2347.i, align 2
+; CHECK: %B14 = load i16, i16* %arrayidx5.us.i.2349.i, align 2
+; CHECK: %B15 = load i16, i16* %arrayidx4.us.i.1.2.i, align 2
+; CHECK: %B17 = load i16, i16* %arrayidx4.us.i.2.2.i, align 2
+; CHECK: %B19 = load i16, i16* %arrayidx4.us.i.3.2.i, align 2
+; CHECK: %B21 = load i16, i16* %arrayidx4.us.i.3356.i, align 2
+; CHECK: %B22 = load i16, i16* %arrayidx5.us.i.3358.i, align 2
+; CHECK: %B23 = load i16, i16* %arrayidx4.us.i.1.3.i, align 2
+; CHECK: %B25 = load i16, i16* %arrayidx4.us.i.2.3.i, align 2
+; CHECK: %B27 = load i16, i16* %arrayidx4.us.i.3.3.i, align 2
+; CHECK-NOT: load
+
+for.body:
+  %A3 = phi i32 [ %add9.us.i.3361.i, %for.body ], [ 0, %entry ]
+  %j.026.us.i.i = phi i32 [ %inc.us.i.3362.i, %for.body ], [ 0, %entry ]
+  %A4 = phi i32 [ %add9.us.i.1.3.i, %for.body ], [ 0, %entry ]
+  %A5 = phi i32 [ %add9.us.i.2.3.i, %for.body ], [ 0, %entry ]
+  %A6 = phi i32 [ %add9.us.i.3.3.i, %for.body ], [ 0, %entry ]
+  %niter335.i = phi i32 [ %niter335.nsub.3.i, %for.body ], [ %unroll_iter310.i, %entry ]
+  %add.us.i.i = add i32 %j.026.us.i.i, %mul.us.i118.i
+  %arrayidx4.us.i.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.i
+  %A7 = load i16, i16* %arrayidx4.us.i.i, align 2
+  %conv.us.i.i = sext i16 %A7 to i32
+  %arrayidx5.us.i.i = getelementptr inbounds i16, i16* %B, i32 %j.026.us.i.i
+  %A8 = load i16, i16* %arrayidx5.us.i.i, align 2
+  %conv6.us.i.i = sext i16 %A8 to i32
+  %mul7.us.i.i = mul nsw i32 %conv6.us.i.i, %conv.us.i.i
+  %add9.us.i.i = add nsw i32 %mul7.us.i.i, %A3
+  %inc.us.i.i = or i32 %j.026.us.i.i, 1
+  %add.us.i.1.i = add i32 %j.026.us.i.i, %mul.us.i118.1.i
+  %arrayidx4.us.i.1.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.1.i
+  %A9 = load i16, i16* %arrayidx4.us.i.1.i, align 2
+  %conv.us.i.1.i = sext i16 %A9 to i32
+  %arrayidx5.us.i.1.i = getelementptr inbounds i16, i16* %B, i32 %j.026.us.i.i
+  %B0 = load i16, i16* %arrayidx5.us.i.1.i, align 2
+  %conv6.us.i.1.i = sext i16 %B0 to i32
+  %mul7.us.i.1.i = mul nsw i32 %conv6.us.i.1.i, %conv.us.i.1.i
+  %add9.us.i.1.i = add nsw i32 %mul7.us.i.1.i, %A4
+  %inc.us.i.1.i = or i32 %j.026.us.i.i, 1
+  %add.us.i.2.i = add i32 %j.026.us.i.i, %mul.us.i118.2.i
+  %arrayidx4.us.i.2.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.2.i
+  %B1 = load i16, i16* %arrayidx4.us.i.2.i, align 2
+  %conv.us.i.2.i = sext i16 %B1 to i32
+  %arrayidx5.us.i.2.i = getelementptr inbounds i16, i16* %B, i32 %j.026.us.i.i
+  %B2 = load i16, i16* %arrayidx5.us.i.2.i, align 2
+  %conv6.us.i.2.i = sext i16 %B2 to i32
+  %mul7.us.i.2.i = mul nsw i32 %conv6.us.i.2.i, %conv.us.i.2.i
+  %add9.us.i.2.i = add nsw i32 %mul7.us.i.2.i, %A5
+  %inc.us.i.2.i = or i32 %j.026.us.i.i, 1
+  %add.us.i.3.i = add i32 %j.026.us.i.i, %mul.us.i118.3.i
+  %arrayidx4.us.i.3.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.3.i
+  %B3 = load i16, i16* %arrayidx4.us.i.3.i, align 2
+  %conv.us.i.3.i = sext i16 %B3 to i32
+  %arrayidx5.us.i.3.i = getelementptr inbounds i16, i16* %B, i32 %j.026.us.i.i
+  %B4 = load i16, i16* %arrayidx5.us.i.3.i, align 2
+  %conv6.us.i.3.i = sext i16 %B4 to i32
+  %mul7.us.i.3.i = mul nsw i32 %conv6.us.i.3.i, %conv.us.i.3.i
+  %add9.us.i.3.i = add nsw i32 %mul7.us.i.3.i, %A6
+  %inc.us.i.3.i = or i32 %j.026.us.i.i, 1
+  %add.us.i.1337.i = add i32 %inc.us.i.i, %mul.us.i118.i
+  %arrayidx4.us.i.1338.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.1337.i
+  %B5 = load i16, i16* %arrayidx4.us.i.1338.i, align 2
+  %conv.us.i.1339.i = sext i16 %B5 to i32
+  %arrayidx5.us.i.1340.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.i
+  %B6 = load i16, i16* %arrayidx5.us.i.1340.i, align 2
+  %conv6.us.i.1341.i = sext i16 %B6 to i32
+  %mul7.us.i.1342.i = mul nsw i32 %conv6.us.i.1341.i, %conv.us.i.1339.i
+  %add9.us.i.1343.i = add nsw i32 %mul7.us.i.1342.i, %add9.us.i.i
+  %inc.us.i.1344.i = or i32 %j.026.us.i.i, 2
+  %add.us.i.1.1.i = add i32 %inc.us.i.1.i, %mul.us.i118.1.i
+  %arrayidx4.us.i.1.1.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.1.1.i
+  %B7 = load i16, i16* %arrayidx4.us.i.1.1.i, align 2
+  %conv.us.i.1.1.i = sext i16 %B7 to i32
+  %arrayidx5.us.i.1.1.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.1.i
+  %B6.dup = load i16, i16* %arrayidx5.us.i.1.1.i, align 2
+  %conv6.us.i.1.1.i = sext i16 %B6.dup to i32
+  %mul7.us.i.1.1.i = mul nsw i32 %conv6.us.i.1.1.i, %conv.us.i.1.1.i
+  %add9.us.i.1.1.i = add nsw i32 %mul7.us.i.1.1.i, %add9.us.i.1.i
+  %inc.us.i.1.1.i = or i32 %j.026.us.i.i, 2
+  %add.us.i.2.1.i = add i32 %inc.us.i.2.i, %mul.us.i118.2.i
+  %arrayidx4.us.i.2.1.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.2.1.i
+  %B9 = load i16, i16* %arrayidx4.us.i.2.1.i, align 2
+  %conv.us.i.2.1.i = sext i16 %B9 to i32
+  %arrayidx5.us.i.2.1.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.2.i
+  %B6.dup.i = load i16, i16* %arrayidx5.us.i.2.1.i, align 2
+  %conv6.us.i.2.1.i = sext i16 %B6.dup.i to i32
+  %mul7.us.i.2.1.i = mul nsw i32 %conv6.us.i.2.1.i, %conv.us.i.2.1.i
+  %add9.us.i.2.1.i = add nsw i32 %mul7.us.i.2.1.i, %add9.us.i.2.i
+  %inc.us.i.2.1.i = or i32 %j.026.us.i.i, 2
+  %add.us.i.3.1.i = add i32 %inc.us.i.3.i, %mul.us.i118.3.i
+  %arrayidx4.us.i.3.1.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.3.1.i
+  %B11 = load i16, i16* %arrayidx4.us.i.3.1.i, align 2
+  %conv.us.i.3.1.i = sext i16 %B11 to i32
+  %arrayidx5.us.i.3.1.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.3.i
+  %B6.dup.i.i = load i16, i16* %arrayidx5.us.i.3.1.i, align 2
+  %conv6.us.i.3.1.i = sext i16 %B6.dup.i.i to i32
+  %mul7.us.i.3.1.i = mul nsw i32 %conv6.us.i.3.1.i, %conv.us.i.3.1.i
+  %add9.us.i.3.1.i = add nsw i32 %mul7.us.i.3.1.i, %add9.us.i.3.i
+  %inc.us.i.3.1.i = or i32 %j.026.us.i.i, 2
+  %add.us.i.2346.i = add i32 %inc.us.i.1344.i, %mul.us.i118.i
+  %arrayidx4.us.i.2347.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.2346.i
+  %B13 = load i16, i16* %arrayidx4.us.i.2347.i, align 2
+  %conv.us.i.2348.i = sext i16 %B13 to i32
+  %arrayidx5.us.i.2349.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.1344.i
+  %B14 = load i16, i16* %arrayidx5.us.i.2349.i, align 2
+  %conv6.us.i.2350.i = sext i16 %B14 to i32
+  %mul7.us.i.2351.i = mul nsw i32 %conv6.us.i.2350.i, %conv.us.i.2348.i
+  %add9.us.i.2352.i = add nsw i32 %mul7.us.i.2351.i, %add9.us.i.1343.i
+  %inc.us.i.2353.i = or i32 %j.026.us.i.i, 3
+  %add.us.i.1.2.i = add i32 %inc.us.i.1.1.i, %mul.us.i118.1.i
+  %arrayidx4.us.i.1.2.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.1.2.i
+  %B15 = load i16, i16* %arrayidx4.us.i.1.2.i, align 2
+  %conv.us.i.1.2.i = sext i16 %B15 to i32
+  %arrayidx5.us.i.1.2.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.1.1.i
+  %B14.dup = load i16, i16* %arrayidx5.us.i.1.2.i, align 2
+  %conv6.us.i.1.2.i = sext i16 %B14.dup to i32
+  %mul7.us.i.1.2.i = mul nsw i32 %conv6.us.i.1.2.i, %conv.us.i.1.2.i
+  %add9.us.i.1.2.i = add nsw i32 %mul7.us.i.1.2.i, %add9.us.i.1.1.i
+  %inc.us.i.1.2.i = or i32 %j.026.us.i.i, 3
+  %add.us.i.2.2.i = add i32 %inc.us.i.2.1.i, %mul.us.i118.2.i
+  %arrayidx4.us.i.2.2.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.2.2.i
+  %B17 = load i16, i16* %arrayidx4.us.i.2.2.i, align 2
+  %conv.us.i.2.2.i = sext i16 %B17 to i32
+  %arrayidx5.us.i.2.2.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.2.1.i
+  %B14.dup.i = load i16, i16* %arrayidx5.us.i.2.2.i, align 2
+  %conv6.us.i.2.2.i = sext i16 %B14.dup.i to i32
+  %mul7.us.i.2.2.i = mul nsw i32 %conv6.us.i.2.2.i, %conv.us.i.2.2.i
+  %add9.us.i.2.2.i = add nsw i32 %mul7.us.i.2.2.i, %add9.us.i.2.1.i
+  %inc.us.i.2.2.i = or i32 %j.026.us.i.i, 3
+  %add.us.i.3.2.i = add i32 %inc.us.i.3.1.i, %mul.us.i118.3.i
+  %arrayidx4.us.i.3.2.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.3.2.i
+  %B19 = load i16, i16* %arrayidx4.us.i.3.2.i, align 2
+  %conv.us.i.3.2.i = sext i16 %B19 to i32
+  %arrayidx5.us.i.3.2.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.3.1.i
+  %B14.dup.i.i = load i16, i16* %arrayidx5.us.i.3.2.i, align 2
+  %conv6.us.i.3.2.i = sext i16 %B14.dup.i.i to i32
+  %mul7.us.i.3.2.i = mul nsw i32 %conv6.us.i.3.2.i, %conv.us.i.3.2.i
+  %add9.us.i.3.2.i = add nsw i32 %mul7.us.i.3.2.i, %add9.us.i.3.1.i
+  %inc.us.i.3.2.i = or i32 %j.026.us.i.i, 3
+  %add.us.i.3355.i = add i32 %inc.us.i.2353.i, %mul.us.i118.i
+  %arrayidx4.us.i.3356.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.3355.i
+  %B21 = load i16, i16* %arrayidx4.us.i.3356.i, align 2
+  %conv.us.i.3357.i = sext i16 %B21 to i32
+  %arrayidx5.us.i.3358.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.2353.i
+  %B22 = load i16, i16* %arrayidx5.us.i.3358.i, align 2
+  %conv6.us.i.3359.i = sext i16 %B22 to i32
+  %mul7.us.i.3360.i = mul nsw i32 %conv6.us.i.3359.i, %conv.us.i.3357.i
+  %add9.us.i.3361.i = add nsw i32 %mul7.us.i.3360.i, %add9.us.i.2352.i
+  %inc.us.i.3362.i = add i32 %j.026.us.i.i, 4
+  %add.us.i.1.3.i = add i32 %inc.us.i.1.2.i, %mul.us.i118.1.i
+  %arrayidx4.us.i.1.3.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.1.3.i
+  %B23 = load i16, i16* %arrayidx4.us.i.1.3.i, align 2
+  %conv.us.i.1.3.i = sext i16 %B23 to i32
+  %arrayidx5.us.i.1.3.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.1.2.i
+  %B22.dup = load i16, i16* %arrayidx5.us.i.1.3.i, align 2
+  %conv6.us.i.1.3.i = sext i16 %B22.dup to i32
+  %mul7.us.i.1.3.i = mul nsw i32 %conv6.us.i.1.3.i, %conv.us.i.1.3.i
+  %add9.us.i.1.3.i = add nsw i32 %mul7.us.i.1.3.i, %add9.us.i.1.2.i
+  %add.us.i.2.3.i = add i32 %inc.us.i.2.2.i, %mul.us.i118.2.i
+  %arrayidx4.us.i.2.3.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.2.3.i
+  %B25 = load i16, i16* %arrayidx4.us.i.2.3.i, align 2
+  %conv.us.i.2.3.i = sext i16 %B25 to i32
+  %arrayidx5.us.i.2.3.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.2.2.i
+  %B22.dup.i = load i16, i16* %arrayidx5.us.i.2.3.i, align 2
+  %conv6.us.i.2.3.i = sext i16 %B22.dup.i to i32
+  %mul7.us.i.2.3.i = mul nsw i32 %conv6.us.i.2.3.i, %conv.us.i.2.3.i
+  %add9.us.i.2.3.i = add nsw i32 %mul7.us.i.2.3.i, %add9.us.i.2.2.i
+  %add.us.i.3.3.i = add i32 %inc.us.i.3.2.i, %mul.us.i118.3.i
+  %arrayidx4.us.i.3.3.i = getelementptr inbounds i16, i16* %A, i32 %add.us.i.3.3.i
+  %B27 = load i16, i16* %arrayidx4.us.i.3.3.i, align 2
+  %conv.us.i.3.3.i = sext i16 %B27 to i32
+  %arrayidx5.us.i.3.3.i = getelementptr inbounds i16, i16* %B, i32 %inc.us.i.3.2.i
+  %B22.dup.i.i = load i16, i16* %arrayidx5.us.i.3.3.i, align 2
+  %conv6.us.i.3.3.i = sext i16 %B22.dup.i.i to i32
+  %mul7.us.i.3.3.i = mul nsw i32 %conv6.us.i.3.3.i, %conv.us.i.3.3.i
+  %add9.us.i.3.3.i = add nsw i32 %mul7.us.i.3.3.i, %add9.us.i.3.2.i
+  %niter335.nsub.3.i = add i32 %niter335.i, -4
+  %niter335.ncmp.3.i = icmp eq i32 %niter335.nsub.3.i, 0
+  br i1 %niter335.ncmp.3.i, label %exit, label %for.body
+
+exit:
+  %arrayidx.out.i = getelementptr inbounds i32, i32* %res, i32 0
+  store i32 %add9.us.i.3361.i, i32* %arrayidx.out.i, align 4
+  %arrayidx.out.1.i = getelementptr inbounds i32, i32* %res, i32 1
+  store i32 %add9.us.i.1.3.i, i32* %arrayidx.out.1.i, align 4
+  %arrayidx.out.2.i = getelementptr inbounds i32, i32* %res, i32 2
+  store i32 %add9.us.i.2.3.i, i32* %arrayidx.out.2.i, align 4
+  %arrayidx.out.3.i = getelementptr inbounds i32, i32* %res, i32 3
+  store i32 %add9.us.i.3.3.i, i32* %arrayidx.out.3.i, align 4
+  ret void
+}

test/CodeGen/ARM/ParallelDSP/smlad0.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; The Cortex-M0 does not support unaligned accesses:
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+;
+; Check DSP extension:
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define dso_local i32 @OneReduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+;
+; CHECK-LABEL: @OneReduction
+; CHECK:  %mac1{{\.}}026 = phi i32 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
+; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
+; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
+; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
+; CHECK:  [[V8]] = call i32 @llvm.arm.smlad(i32 [[V5]], i32 [[V7]], i32 %mac1{{\.}}026)
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
+;
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+; One reduction statement here:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+
+; Here the Mul is the LHS, and the Add the RHS.
+  %add11 = add i32 %mul9, %add10
+
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+define dso_local arm_aapcs_vfpcc i32 @TwoReductions(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+;
+; CHECK-LABEL: @TwoReductions
+;
+; CHECK:  %mac1{{\.}}058 = phi i32 [ [[V10:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  %mac2{{\.}}057 = phi i32 [ [[V17:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  [[V10]] = call i32 @llvm.arm.smlad(i32 %{{.*}}, i32 %{{.*}}, i32 %mac1{{\.}}058)
+; CHECK:  [[V17]] = call i32 @llvm.arm.smlad(i32 %{{.*}}, i32 %{{.*}}, i32 %mac2{{\.}}057)
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+entry:
+  %cmp55 = icmp sgt i32 %arg, 0
+  br i1 %cmp55, label %for.body.preheader, label %for.cond.cleanup
+
+for.cond.cleanup:
+  %mac2.0.lcssa = phi i32 [ 0, %entry ], [ %add28, %for.body ]
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add16, %for.body ]
+  %add30 = add nsw i32 %mac1.0.lcssa, %mac2.0.lcssa
+  ret i32 %add30
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+; And two reduction statements here:
+  %mac1.058 = phi i32 [ %add16, %for.body ], [ 0, %for.body.preheader ]
+  %mac2.057 = phi i32 [ %add28, %for.body ], [ 0, %for.body.preheader ]
+
+  %i.056 = phi i32 [ %add29, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.056
+  %0 = load i16, i16* %arrayidx, align 2
+  %add1 = or i32 %i.056, 1
+  %arrayidx2 = getelementptr inbounds i16, i16* %arg3, i32 %add1
+  %1 = load i16, i16* %arrayidx2, align 2
+  %add3 = or i32 %i.056, 2
+  %arrayidx4 = getelementptr inbounds i16, i16* %arg3, i32 %add3
+  %2 = load i16, i16* %arrayidx4, align 2
+
+  %add5 = or i32 %i.056, 3
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg3, i32 %add5
+  %3 = load i16, i16* %arrayidx6, align 2
+  %arrayidx8 = getelementptr inbounds i16, i16* %arg2, i32 %i.056
+  %4 = load i16, i16* %arrayidx8, align 2
+  %conv = sext i16 %4 to i32
+  %conv9 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv9
+  %arrayidx11 = getelementptr inbounds i16, i16* %arg2, i32 %add1
+  %5 = load i16, i16* %arrayidx11, align 2
+  %conv12 = sext i16 %5 to i32
+  %conv13 = sext i16 %1 to i32
+  %mul14 = mul nsw i32 %conv12, %conv13
+  %add15 = add i32 %mul, %mac1.058
+  %add16 = add i32 %add15, %mul14
+  %arrayidx18 = getelementptr inbounds i16, i16* %arg2, i32 %add3
+  %6 = load i16, i16* %arrayidx18, align 2
+  %conv19 = sext i16 %6 to i32
+  %conv20 = sext i16 %2 to i32
+  %mul21 = mul nsw i32 %conv19, %conv20
+  %arrayidx23 = getelementptr inbounds i16, i16* %arg2, i32 %add5
+  %7 = load i16, i16* %arrayidx23, align 2
+  %conv24 = sext i16 %7 to i32
+  %conv25 = sext i16 %3 to i32
+  %mul26 = mul nsw i32 %conv24, %conv25
+  %add27 = add i32 %mul21, %mac2.057
+  %add28 = add i32 %add27, %mul26
+  %add29 = add nuw nsw i32 %i.056, 4
+  %cmp = icmp slt i32 %add29, %arg
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+define i32 @one_zext(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LABEL: @one_zext
+; CHECK-NOT: call i32 @llvm.arm.smlad
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = zext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = zext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %mul9, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+define i32 @two_zext(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LABEL: @two_zext
+; CHECK-NOT: call i32 @llvm.arm.smlad
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = zext i16 %2 to i32
+  %conv4 = zext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = zext i16 %3 to i32
+  %conv8 = zext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %mul9, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+define dso_local i32 @SextMultiUse(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LABEL: @SextMultiUse
+; CHECK-NOT: call i32 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  %count.final = phi i32 [ 0, %entry ], [ %count.next, %for.body ]
+  %res = add i32 %mac1.0.lcssa, %count.final
+  ret i32 %res
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %count = phi i32 [ %count.next, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %count.next = add i32 %conv4, %count
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %mul9, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}

test/CodeGen/ARM/ParallelDSP/smlad1.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+
+; CHECK-LABEL: @test1
+; CHECK:  %mac1{{\.}}026 = phi i32 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
+; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
+; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
+; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
+; CHECK:  [[V8]] = call i32 @llvm.arm.smlad(i32 [[V5]], i32 [[V7]], i32 %mac1{{\.}}026)
+
+define dso_local i32 @test1(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+
+; And here the Add is the LHS, the Mul the RHS
+  %add11 = add i32 %add10, %mul9
+
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+; Here we have i8 loads, which we do want to support, but don't handle yet.
+;
+; CHECK-LABEL: @test2
+; CHECK-NOT:   call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test2(i32 %arg, i32* nocapture readnone %arg1, i8* nocapture readonly %arg2, i8* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i8, i8* %arg3, align 2
+  %.pre27 = load i8, i8* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i8, i8* %arg3, i32 %i.025
+  %0 = load i8, i8* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i8, i8* %arg3, i32 %add
+  %1 = load i8, i8* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i8, i8* %arg2, i32 %i.025
+  %2 = load i8, i8* %arrayidx3, align 2
+  %conv = sext i8 %2 to i32
+  %conv4 = sext i8 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i8, i8* %arg2, i32 %add
+  %3 = load i8, i8* %arrayidx6, align 2
+  %conv7 = sext i8 %3 to i32
+  %conv8 = sext i8 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %add10, %mul9
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+

test/CodeGen/ARM/ParallelDSP/smlad10.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; Reduction statement is an i64 type: we only support i32 so check that the
+; rewrite isn't triggered.
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+define dso_local i64 @test(i64 %arg, i64* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i64 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
+  ret i64 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i64 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i64 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i64 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i64 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i64 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i64
+  %conv4 = sext i16 %0 to i64
+  %mul = mul nsw i64 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i64 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i64
+  %conv8 = sext i16 %1 to i64
+  %mul9 = mul nsw i64 %conv7, %conv8
+  %add10 = add i64 %mul, %mac1.026
+
+  %add11 = add i64 %mul9, %add10
+
+  %exitcond = icmp ne i64 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+

test/CodeGen/ARM/ParallelDSP/smlad11.ll

+; REQUIRES: asserts
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S -stats 2>&1 | FileCheck %s
+;
+; A more complicated chain: 4 mul operations, so we expect 2 smlad calls.
+;
+; CHECK:  %mac1{{\.}}054 = phi i32 [ [[V17:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  [[V8:%[0-9]+]] = bitcast i16* %arrayidx8 to i32*
+; CHECK:  [[V9:%[0-9]+]] = load i32, i32* [[V8]], align 2
+; CHECK:  [[V10:%[0-9]+]] = bitcast i16* %arrayidx to i32*
+; CHECK:  [[V11:%[0-9]+]] = load i32, i32* [[V10]], align 2
+; CHECK:  [[V12:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V9]], i32 [[V11]], i32 %mac1{{\.}}054)
+; CHECK:  [[V13:%[0-9]+]] = bitcast i16* %arrayidx17 to i32*
+; CHECK:  [[V14:%[0-9]+]] = load i32, i32* [[V13]], align 2
+; CHECK:  [[V15:%[0-9]+]] = bitcast i16* %arrayidx4 to i32*
+; CHECK:  [[V16:%[0-9]+]] = load i32, i32* [[V15]], align 2
+; CHECK:  [[V17:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V14]], i32 [[V16]], i32 [[V12]])
+;
+; And we don't want to see a 3rd smlad:
+; CHECK-NOT: call i32 @llvm.arm.smlad
+;
+; CHECK:  2 arm-parallel-dsp - Number of smlad instructions generated
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp52 = icmp sgt i32 %arg, 0
+  br i1 %cmp52, label %for.body.preheader, label %for.cond.cleanup
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add28, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+  %mac1.054 = phi i32 [ %add28, %for.body ], [ 0, %for.body.preheader ]
+  %i.053 = phi i32 [ %add29, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.053
+  %0 = load i16, i16* %arrayidx, align 2
+  %add1 = or i32 %i.053, 1
+  %arrayidx2 = getelementptr inbounds i16, i16* %arg3, i32 %add1
+  %1 = load i16, i16* %arrayidx2, align 2
+  %add3 = or i32 %i.053, 2
+  %arrayidx4 = getelementptr inbounds i16, i16* %arg3, i32 %add3
+  %2 = load i16, i16* %arrayidx4, align 2
+  %add5 = or i32 %i.053, 3
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg3, i32 %add5
+  %3 = load i16, i16* %arrayidx6, align 2
+  %arrayidx8 = getelementptr inbounds i16, i16* %arg2, i32 %i.053
+  %4 = load i16, i16* %arrayidx8, align 2
+  %conv = sext i16 %4 to i32
+  %conv9 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv9
+  %arrayidx11 = getelementptr inbounds i16, i16* %arg2, i32 %add1
+  %5 = load i16, i16* %arrayidx11, align 2
+  %conv12 = sext i16 %5 to i32
+  %conv13 = sext i16 %1 to i32
+  %mul14 = mul nsw i32 %conv12, %conv13
+  %arrayidx17 = getelementptr inbounds i16, i16* %arg2, i32 %add3
+  %6 = load i16, i16* %arrayidx17, align 2
+  %conv18 = sext i16 %6 to i32
+  %conv19 = sext i16 %2 to i32
+  %mul20 = mul nsw i32 %conv18, %conv19
+  %arrayidx23 = getelementptr inbounds i16, i16* %arg2, i32 %add5
+  %7 = load i16, i16* %arrayidx23, align 2
+  %conv24 = sext i16 %7 to i32
+  %conv25 = sext i16 %3 to i32
+  %mul26 = mul nsw i32 %conv24, %conv25
+  %add15 = add i32 %mul, %mac1.054
+  %add21 = add i32 %add15, %mul14
+  %add27 = add i32 %add21, %mul20
+  %add28 = add i32 %add27, %mul26
+  %add29 = add nuw nsw i32 %i.053, 4
+  %cmp = icmp slt i32 %add29, %arg
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}

test/CodeGen/ARM/ParallelDSP/smlad12.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; The loop header is not the loop latch.
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+; This is the loop header:
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body2 ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body2 ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %mul9, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body2, label %for.cond.cleanup
+
+; And this is the loop latch:
+for.body2:
+  br label %for.body
+}

test/CodeGen/ARM/ParallelDSP/smlad2.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; Operands of both muls are not symmetrical (see also comments inlined below), check
+; that the rewrite isn't triggered.
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+
+; This zero-extends the 2nd operand of %mul:
+  %conv4 = zext i16 %0 to i32
+
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+
+; And here we only have sign-extensions. Thus, the operands of
+; %mul and %mul9 are not symmetrical:
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %add10, %mul9
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}

test/CodeGen/ARM/ParallelDSP/smlad3.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; The loads are not consecutive: check that the rewrite isn't triggered.
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+
+; Here we add another constants offset of 2, to make sure the
+; loads to %3 and %2 are not consecutive:
+
+  %add5 = add nuw nsw i32 %i.025, 2
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add5
+  %3 = load i16, i16* %arrayidx6, align 2
+
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %add10, %mul9
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+

test/CodeGen/ARM/ParallelDSP/smlad4.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; The loads are not narrow loads: check that the rewrite isn't triggered.
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+; Arg2 is now an i32, while Arg3 is still and i16:
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i32* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp22 = icmp sgt i32 %arg, 0
+  br i1 %cmp22, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add9, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %0 = phi i16 [ %1, %for.body ], [ %.pre, %for.body.preheader ]
+  %mac1.024 = phi i32 [ %add9, %for.body ], [ 0, %for.body.preheader ]
+  %i.023 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %add = add nuw nsw i32 %i.023, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %conv = sext i16 %0 to i32
+
+; This is a 'normal' i32 load to %2:
+  %arrayidx3 = getelementptr inbounds i32, i32* %arg2, i32 %i.023
+  %2 = load i32, i32* %arrayidx3, align 4
+
+; This mul has now 1 operand which is a narrow load, and the other a normal
+; i32 load:
+  %mul = mul nsw i32 %2, %conv
+
+  %add4 = add nuw nsw i32 %i.023, 2
+  %arrayidx5 = getelementptr inbounds i32, i32* %arg2, i32 %add4
+  %3 = load i32, i32* %arrayidx5, align 4
+  %conv6 = sext i16 %1 to i32
+  %mul7 = mul nsw i32 %3, %conv6
+  %add8 = add i32 %mul, %mac1.024
+  %add9 = add i32 %add8, %mul7
+  %exitcond = icmp eq i32 %add, %arg
+  br i1 %exitcond, label %for.cond.cleanup, label %for.body
+}

test/CodeGen/ARM/ParallelDSP/smlad5.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; The loads are volatile loads: check that the rewrite isn't triggered.
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load volatile i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load volatile i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load volatile i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load volatile i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %add10, %mul9
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+

test/CodeGen/ARM/ParallelDSP/smlad6.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; Alias check: check that the rewrite isn't triggered when there's a store
+; instruction possibly aliasing any mul load operands; arguments are passed
+; without 'restrict' enabled.
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+
+; Store inserted here, aliasing with arrayidx, arrayidx1, arrayidx3
+  store i16 42, i16* %arrayidx, align 2
+
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %mul9, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+

test/CodeGen/ARM/ParallelDSP/smlad7.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; Alias check: check that the rewrite isn't triggered when there's a store
+; aliasing one of the mul load operands. Arguments are now annotated with
+; 'noalias'.
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* noalias %arg1, i16* noalias readonly %arg2, i16* noalias readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+
+; Store inserted here, aliasing only with loads from 'arrayidx'.
+  store i16 42, i16* %arrayidx, align 2
+
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %add10 = add i32 %mul, %mac1.026
+
+; Here the Mul is the LHS, and the Add the RHS.
+  %add11 = add i32 %mul9, %add10
+
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+

test/CodeGen/ARM/ParallelDSP/smlad8.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; Mul with operands that are not simple load and sext/zext chains: this is not
+; yet supported so the rewrite shouldn't trigger (but we do want to support this
+; soon).
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3, i16* %arg4) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  %gep0 = getelementptr inbounds i16, i16* %arg4, i32 0
+  %gep1 = getelementptr inbounds i16, i16* %arg4, i32 1
+  %.add4 = load i16, i16* %gep0, align 2
+  %.add5 = load i16, i16* %gep1, align 2
+  %.zext4 = zext i16 %.add4 to i32
+  %.zext5 = zext i16 %.add5 to i32
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %add1 = add i32 %conv, %.zext4
+
+; This mul has a more complicated pattern as an operand, %add1
+; is another add and load, which we don't support for now.
+  %mul = mul nsw i32 %add1, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %add2 = add i32 %conv7, %.zext5
+
+; Same here
+  %mul9 = mul nsw i32 %add2, %conv8
+  %add10 = add i32 %mul, %mac1.026
+
+  %add11 = add i32 %mul9, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}

test/CodeGen/ARM/ParallelDSP/smlad9.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; Muls with operands that are constants: not yet supported, so the rewrite
+; should not trigger (but we do want to add this soon).
+;
+; CHECK-NOT:  call i32 @llvm.arm.smlad
+;
+define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
+  ret i32 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %v2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %v2 to i32
+
+; RHS operand of this mul is a constant
+  %mul = mul nsw i32 %conv, 43
+
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %v3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %v3 to i32
+
+; And this RHS operand is a constant too.
+  %mul9 = mul nsw i32 %conv7, 42
+
+  %add10 = add i32 %mul, %mac1.026
+  %add11 = add i32 %mul9, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+

test/CodeGen/ARM/ParallelDSP/smladx-1.ll

+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define i32 @smladx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smladx
+; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
+; CHECK: [[ACC0:%[^ ]+]] = phi i32 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
+; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
+; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
+; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN23]], i32 [[IN12]], i32 [[ACC0]])
+; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
+; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
+; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
+; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN21]], i32 [[IN10]], i32 [[ACC1]])
+; CHECK-NOT: call i32 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i32 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i32 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sum.010.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sub.epil = sub i32 %j, %i.011.epil
+  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
+  %2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nsw i32 %mul.epil, %sum.010.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i32 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
+  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
+  %In2 = load i16, i16* %pIn2.0, align 2
+  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
+  %In1 = load i16, i16* %pIn1.0, align 2
+  %inc = or i32 %i.011, 1
+  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
+  %In2.1 = load i16, i16* %pIn2.1, align 2
+  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
+  %In1.1 = load i16, i16* %pIn1.1, align 2
+  %inc.1 = or i32 %i.011, 2
+  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
+  %In2.2 = load i16, i16* %pIn2.2, align 2
+  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
+  %In1.2 = load i16, i16* %pIn1.2, align 2
+  %inc.2 = or i32 %i.011, 3
+  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
+  %In2.3 = load i16, i16* %pIn2.3, align 2
+  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
+  %In1.3 = load i16, i16* %pIn1.3, align 2
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %mul = mul nsw i32 %sextIn1, %sextIn2
+  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
+  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
+  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
+  %add = add nsw i32 %mul, %sum.010
+  %add.1 = add nsw i32 %mul.1, %add
+  %add.2 = add nsw i32 %mul.2, %add.1
+  %add.3 = add nsw i32 %mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define i32 @smladx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smladx_swap
+; CHECK: for.body.preheader.new:
+; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
+; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
+
+; CHECK: for.body:
+; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
+; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
+; CHECK: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[ACC0:%[^ ]+]] = phi i32 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+
+; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
+; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
+
+
+; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
+; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
+; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
+; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN2_2]], i32 [[IN1]], i32 [[ACC0]])
+
+; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
+; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
+; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
+; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN2]], i32 [[IN1_2]], i32 [[ACC1]])
+
+; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
+; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
+
+; CHECK-NOT: call i32 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
+  %2 = add i32 %j, -1
+  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i32 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i32 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
+
+for.body.epil.preheader:
+  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
+  %3 = sub i32 %j, %i.011.unr
+  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
+  %4 = sub i32 0, %xtraiter
+  br label %for.body.epil
+
+for.body.epil:
+  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
+  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
+  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
+  %sum.010.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
+  %5 = load i16, i16* %lsr.iv3, align 2
+  %conv.epil = sext i16 %5 to i32
+  %6 = load i16, i16* %lsr.iv, align 2
+  %conv2.epil = sext i16 %6 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nsw i32 %mul.epil, %sum.010.epil
+  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
+  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
+  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
+  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %pin2 = phi i16* [ %pin2_sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
+  %pin1 = phi i16* [ %pin1_add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i32 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %pin2_add1 = getelementptr i16, i16* %pin2, i32 1
+  %In2 = load i16, i16* %pin2_add1, align 2
+  %pin1_sub2 = getelementptr i16, i16* %pin1, i32 -2
+  %In1 = load i16, i16* %pin1_sub2, align 2
+  %In2.1 = load i16, i16* %pin2, align 2
+  %pin1_sub1 = getelementptr i16, i16* %pin1, i32 -1
+  %In1.1 = load i16, i16* %pin1_sub1, align 2
+  %pin2_sub1 = getelementptr i16, i16* %pin2, i32 -1
+  %In2.2 = load i16, i16* %pin2_sub1, align 2
+  %In1.2 = load i16, i16* %pin1, align 2
+  %pin2_sub2 = getelementptr i16, i16* %pin2, i32 -2
+  %In2.3 = load i16, i16* %pin2_sub2, align 2
+  %pin1_add1 = getelementptr i16, i16* %pin1, i32 1
+  %In1.3 = load i16, i16* %pin1_add1, align 2
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %mul = mul nsw i32 %sextIn2, %sextIn1
+  %add = add nsw i32 %mul, %sum.010
+  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
+  %add.1 = add nsw i32 %mul.1, %add
+  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
+  %add.2 = add nsw i32 %mul.2, %add.1
+  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
+  %add.3 = add nsw i32 %mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pin1_add4 = getelementptr i16, i16* %pin1, i32 4
+  %pin2_sub4 = getelementptr i16, i16* %pin2, i32 -4
+  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}

test/CodeGen/ARM/ParallelDSP/smlald0.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; The Cortex-M0 does not support unaligned accesses:
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+;
+; Check DSP extension:
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define dso_local i64 @OneReduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+;
+; CHECK-LABEL: @OneReduction
+; CHECK:  %mac1{{\.}}026 = phi i64 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
+; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
+; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
+; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
+; CHECK:  [[V8]] = call i64 @llvm.arm.smlald(i32 [[V5]], i32 [[V7]], i64 %mac1{{\.}}026)
+; CHECK-NOT: call i64 @llvm.arm.smlald
+;
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlald
+;
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
+  ret i64 %mac1.0.lcssa
+
+for.body:
+; One reduction statement here:
+  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i64
+  %conv4 = sext i16 %0 to i64
+  %mul = mul nsw i64 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i64
+  %conv8 = sext i16 %1 to i64
+  %mul9 = mul nsw i64 %conv7, %conv8
+  %add10 = add i64 %mul, %mac1.026
+
+; Here the Mul is the LHS, and the Add the RHS.
+  %add11 = add i64 %mul9, %add10
+
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+define dso_local arm_aapcs_vfpcc i64 @TwoReductions(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+;
+; CHECK-LABEL: @TwoReductions
+;
+; CHECK:  %mac1{{\.}}058 = phi i64 [ [[V10:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  %mac2{{\.}}057 = phi i64 [ [[V17:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  [[V10]] = call i64 @llvm.arm.smlald(i32 %{{.*}}, i32 %{{.*}}, i64 %mac1{{\.}}058)
+; CHECK:  [[V17]] = call i64 @llvm.arm.smlald(i32 %{{.*}}, i32 %{{.*}}, i64 %mac2{{\.}}057)
+; CHECK-NOT: call i64 @llvm.arm.smlald
+;
+entry:
+  %cmp55 = icmp sgt i32 %arg, 0
+  br i1 %cmp55, label %for.body.preheader, label %for.cond.cleanup
+
+for.cond.cleanup:
+  %mac2.0.lcssa = phi i64 [ 0, %entry ], [ %add28, %for.body ]
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add16, %for.body ]
+  %add30 = add nsw i64 %mac1.0.lcssa, %mac2.0.lcssa
+  ret i64 %add30
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+; And two reduction statements here:
+  %mac1.058 = phi i64 [ %add16, %for.body ], [ 0, %for.body.preheader ]
+  %mac2.057 = phi i64 [ %add28, %for.body ], [ 0, %for.body.preheader ]
+
+  %i.056 = phi i32 [ %add29, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.056
+  %0 = load i16, i16* %arrayidx, align 2
+  %add1 = or i32 %i.056, 1
+  %arrayidx2 = getelementptr inbounds i16, i16* %arg3, i32 %add1
+  %1 = load i16, i16* %arrayidx2, align 2
+  %add3 = or i32 %i.056, 2
+  %arrayidx4 = getelementptr inbounds i16, i16* %arg3, i32 %add3
+  %2 = load i16, i16* %arrayidx4, align 2
+
+  %add5 = or i32 %i.056, 3
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg3, i32 %add5
+  %3 = load i16, i16* %arrayidx6, align 2
+  %arrayidx8 = getelementptr inbounds i16, i16* %arg2, i32 %i.056
+  %4 = load i16, i16* %arrayidx8, align 2
+  %conv = sext i16 %4 to i64
+  %conv9 = sext i16 %0 to i64
+  %mul = mul nsw i64 %conv, %conv9
+  %arrayidx11 = getelementptr inbounds i16, i16* %arg2, i32 %add1
+  %5 = load i16, i16* %arrayidx11, align 2
+  %conv12 = sext i16 %5 to i64
+  %conv13 = sext i16 %1 to i64
+  %mul14 = mul nsw i64 %conv12, %conv13
+  %add15 = add i64 %mul, %mac1.058
+  %add16 = add i64 %add15, %mul14
+  %arrayidx18 = getelementptr inbounds i16, i16* %arg2, i32 %add3
+  %6 = load i16, i16* %arrayidx18, align 2
+  %conv19 = sext i16 %6 to i64
+  %conv20 = sext i16 %2 to i64
+  %mul21 = mul nsw i64 %conv19, %conv20
+  %arrayidx23 = getelementptr inbounds i16, i16* %arg2, i32 %add5
+  %7 = load i16, i16* %arrayidx23, align 2
+  %conv24 = sext i16 %7 to i64
+  %conv25 = sext i16 %3 to i64
+  %mul26 = mul nsw i64 %conv24, %conv25
+  %add27 = add i64 %mul21, %mac2.057
+  %add28 = add i64 %add27, %mul26
+  %add29 = add nuw nsw i32 %i.056, 4
+  %cmp = icmp slt i32 %add29, %arg
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+define i64 @reduction_zext(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LABEL: @reduction_zext
+; CHECK-NOT: call i64 @llvm.arm.smlald
+; CHECK-NOT: call i32 @llvm.arm.smlad
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
+  ret i64 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i64
+  %conv4 = zext i16 %0 to i64
+  %mul = mul nsw i64 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i64
+  %conv8 = zext i16 %1 to i64
+  %mul9 = mul nsw i64 %conv7, %conv8
+  %add10 = add i64 %mul, %mac1.026
+  %add11 = add i64 %mul9, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}

test/CodeGen/ARM/ParallelDSP/smlald1.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+
+; CHECK-LABEL: @test1
+; CHECK:  %mac1{{\.}}026 = phi i64 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
+; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
+; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
+; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
+; CHECK:  [[V8]] = call i64 @llvm.arm.smlald(i32 [[V5]], i32 [[V7]], i64 %mac1{{\.}}026)
+
+define dso_local i64 @test1(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
+  ret i64 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i64
+  %conv4 = sext i16 %0 to i64
+  %mul = mul nsw i64 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i64
+  %conv8 = sext i16 %1 to i64
+  %mul9 = mul nsw i64 %conv7, %conv8
+  %add10 = add i64 %mul, %mac1.026
+
+; And here the Add is the LHS, the Mul the RHS
+  %add11 = add i64 %add10, %mul9
+
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+; Here we have i8 loads, which we do want to support, but don't handle yet.
+;
+; CHECK-LABEL: @test2
+; CHECK-NOT:   call i64 @llvm.arm.smlad
+;
+define dso_local i64 @test2(i32 %arg, i32* nocapture readnone %arg1, i8* nocapture readonly %arg2, i8* nocapture readonly %arg3) {
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i8, i8* %arg3, align 2
+  %.pre27 = load i8, i8* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
+  ret i64 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i8, i8* %arg3, i32 %i.025
+  %0 = load i8, i8* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i8, i8* %arg3, i32 %add
+  %1 = load i8, i8* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i8, i8* %arg2, i32 %i.025
+  %2 = load i8, i8* %arrayidx3, align 2
+  %conv = sext i8 %2 to i64
+  %conv4 = sext i8 %0 to i64
+  %mul = mul nsw i64 %conv, %conv4
+  %arrayidx6 = getelementptr inbounds i8, i8* %arg2, i32 %add
+  %3 = load i8, i8* %arrayidx6, align 2
+  %conv7 = sext i8 %3 to i64
+  %conv8 = sext i8 %1 to i64
+  %mul9 = mul nsw i64 %conv7, %conv8
+  %add10 = add i64 %mul, %mac1.026
+  %add11 = add i64 %add10, %mul9
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+

test/CodeGen/ARM/ParallelDSP/smlald2.ll

+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
+;
+; The Cortex-M0 does not support unaligned accesses:
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+;
+; Check DSP extension:
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define dso_local i64 @OneReduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+;
+; CHECK-LABEL: @OneReduction
+; CHECK:  %mac1{{\.}}026 = phi i64 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
+; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
+; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
+; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
+; CHECK:  [[V8]] = call i64 @llvm.arm.smlald(i32 [[V5]], i32 [[V7]], i64 %mac1{{\.}}026)
+; CHECK-NOT: call i64 @llvm.arm.smlald
+;
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlald
+;
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
+  ret i64 %mac1.0.lcssa
+
+for.body:
+; One reduction statement here:
+  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %sext0 = sext i32 %mul to i64
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %sext1 = sext i32 %mul9 to i64
+  %add10 = add i64 %sext0, %mac1.026
+
+; Here the Mul is the LHS, and the Add the RHS.
+  %add11 = add i64 %sext1, %add10
+
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+define dso_local arm_aapcs_vfpcc i64 @TwoReductions(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+;
+; CHECK-LABEL: @TwoReductions
+;
+; CHECK:  %mac1{{\.}}058 = phi i64 [ [[V10:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  %mac2{{\.}}057 = phi i64 [ [[V17:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
+; CHECK:  [[V10]] = call i64 @llvm.arm.smlald(i32 %{{.*}}, i32 %{{.*}}, i64 %mac1{{\.}}058)
+; CHECK:  [[V17]] = call i64 @llvm.arm.smlald(i32 %{{.*}}, i32 %{{.*}}, i64 %mac2{{\.}}057)
+; CHECK-NOT: call i64 @llvm.arm.smlald
+;
+entry:
+  %cmp55 = icmp sgt i32 %arg, 0
+  br i1 %cmp55, label %for.body.preheader, label %for.cond.cleanup
+
+for.cond.cleanup:
+  %mac2.0.lcssa = phi i64 [ 0, %entry ], [ %add28, %for.body ]
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add16, %for.body ]
+  %add30 = add nsw i64 %mac1.0.lcssa, %mac2.0.lcssa
+  ret i64 %add30
+
+for.body.preheader:
+  br label %for.body
+
+for.body:
+; And two reduction statements here:
+  %mac1.058 = phi i64 [ %add16, %for.body ], [ 0, %for.body.preheader ]
+  %mac2.057 = phi i64 [ %add28, %for.body ], [ 0, %for.body.preheader ]
+
+  %i.056 = phi i32 [ %add29, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.056
+  %0 = load i16, i16* %arrayidx, align 2
+  %add1 = or i32 %i.056, 1
+  %arrayidx2 = getelementptr inbounds i16, i16* %arg3, i32 %add1
+  %1 = load i16, i16* %arrayidx2, align 2
+  %add3 = or i32 %i.056, 2
+  %arrayidx4 = getelementptr inbounds i16, i16* %arg3, i32 %add3
+  %2 = load i16, i16* %arrayidx4, align 2
+
+  %add5 = or i32 %i.056, 3
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg3, i32 %add5
+  %3 = load i16, i16* %arrayidx6, align 2
+  %arrayidx8 = getelementptr inbounds i16, i16* %arg2, i32 %i.056
+  %4 = load i16, i16* %arrayidx8, align 2
+  %conv = sext i16 %4 to i32
+  %conv9 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv9
+  %sext0 = sext i32 %mul to i64
+  %arrayidx11 = getelementptr inbounds i16, i16* %arg2, i32 %add1
+  %5 = load i16, i16* %arrayidx11, align 2
+  %conv12 = sext i16 %5 to i32
+  %conv13 = sext i16 %1 to i32
+  %mul14 = mul nsw i32 %conv12, %conv13
+  %sext1 = sext i32 %mul14 to i64
+  %add15 = add i64 %sext0, %mac1.058
+  %add16 = add i64 %add15, %sext1
+  %arrayidx18 = getelementptr inbounds i16, i16* %arg2, i32 %add3
+  %6 = load i16, i16* %arrayidx18, align 2
+  %conv19 = sext i16 %6 to i32
+  %conv20 = sext i16 %2 to i32
+  %mul21 = mul nsw i32 %conv19, %conv20
+  %sext2 = sext i32 %mul21 to i64
+  %arrayidx23 = getelementptr inbounds i16, i16* %arg2, i32 %add5
+  %7 = load i16, i16* %arrayidx23, align 2
+  %conv24 = sext i16 %7 to i32
+  %conv25 = sext i16 %3 to i32
+  %mul26 = mul nsw i32 %conv24, %conv25
+  %sext3 = sext i32 %mul26 to i64
+  %add27 = add i64 %sext2, %mac2.057
+  %add28 = add i64 %add27, %sext3
+  %add29 = add nuw nsw i32 %i.056, 4
+  %cmp = icmp slt i32 %add29, %arg
+  br i1 %cmp, label %for.body, label %for.cond.cleanup
+}
+
+define i64 @zext_mul_reduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LABEL: @zext_mul_reduction
+; CHECK-NOT: call i64 @llvm.arm.smlald
+; CHECK-NOT: call i32 @llvm.arm.smlad
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
+  ret i64 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = zext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %sext0 = sext i32 %mul to i64
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = zext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %sext1 = sext i32 %mul9 to i64
+  %add10 = add i64 %sext0, %mac1.026
+  %add11 = add i64 %sext1, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}
+
+define i64 @zext_add_reduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
+; CHECK-LABEL: @zext_add_reduction
+; CHECK-NOT: call i64 @llvm.arm.smlald
+; CHECK-NOT: call i32 @llvm.arm.smlad
+entry:
+  %cmp24 = icmp sgt i32 %arg, 0
+  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
+
+for.body.preheader:
+  %.pre = load i16, i16* %arg3, align 2
+  %.pre27 = load i16, i16* %arg2, align 2
+  br label %for.body
+
+for.cond.cleanup:
+  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
+  ret i64 %mac1.0.lcssa
+
+for.body:
+  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
+  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
+  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
+  %0 = load i16, i16* %arrayidx, align 2
+  %add = add nuw nsw i32 %i.025, 1
+  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
+  %1 = load i16, i16* %arrayidx1, align 2
+  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
+  %2 = load i16, i16* %arrayidx3, align 2
+  %conv = sext i16 %2 to i32
+  %conv4 = sext i16 %0 to i32
+  %mul = mul nsw i32 %conv, %conv4
+  %sext0 = zext i32 %mul to i64
+  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
+  %3 = load i16, i16* %arrayidx6, align 2
+  %conv7 = sext i16 %3 to i32
+  %conv8 = sext i16 %1 to i32
+  %mul9 = mul nsw i32 %conv7, %conv8
+  %sext1 = zext i32 %mul9 to i64
+  %add10 = add i64 %sext0, %mac1.026
+  %add11 = add i64 %sext1, %add10
+  %exitcond = icmp ne i32 %add, %arg
+  br i1 %exitcond, label %for.body, label %for.cond.cleanup
+}

test/CodeGen/ARM/ParallelDSP/smlaldx-1.ll

+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smlaldx
+; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
+; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
+; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
+; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
+; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
+; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
+; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
+; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sub.epil = sub i32 %j, %i.011.epil
+  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
+  %2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+for.body:
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
+  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
+  %In2 = load i16, i16* %pIn2.0, align 2
+  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
+  %In1 = load i16, i16* %pIn1.0, align 2
+  %inc = or i32 %i.011, 1
+  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
+  %In2.1 = load i16, i16* %pIn2.1, align 2
+  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
+  %In1.1 = load i16, i16* %pIn1.1, align 2
+  %inc.1 = or i32 %i.011, 2
+  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
+  %In2.2 = load i16, i16* %pIn2.2, align 2
+  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
+  %In1.2 = load i16, i16* %pIn1.2, align 2
+  %inc.2 = or i32 %i.011, 3
+  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
+  %In2.3 = load i16, i16* %pIn2.3, align 2
+  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
+  %In1.3 = load i16, i16* %pIn1.3, align 2
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %mul = mul nsw i32 %sextIn1, %sextIn2
+  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
+  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
+  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
+  %sext.mul = sext i32 %mul to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add = add nsw i64 %sext.mul, %sum.010
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %add.2 = add nsw i64 %sext.mul.2, %add.1
+  %add.3 = add nsw i64 %sext.mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define i64 @smlaldx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
+  %2 = add i32 %j, -1
+  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
+
+for.body.epil.preheader:
+  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
+  %3 = sub i32 %j, %i.011.unr
+  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
+  %4 = sub i32 0, %xtraiter
+  br label %for.body.epil
+
+for.body.epil:
+  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
+  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
+  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
+  %5 = load i16, i16* %lsr.iv3, align 2
+  %conv.epil = sext i16 %5 to i32
+  %6 = load i16, i16* %lsr.iv, align 2
+  %conv2.epil = sext i16 %6 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
+  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
+  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
+  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+; CHECK-LABEL: smlaldx_swap
+; CHECK: for.body.preheader.new:
+; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
+; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
+
+; CHECK: for.body:
+; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
+; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
+; CHECK: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+
+; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
+; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
+
+; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
+; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
+; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
+; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2_2]], i32 [[IN1]], i64 [[ACC0]])
+
+; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
+; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
+; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
+; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC1]])
+
+; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
+; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
+
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+for.body:
+  %pin2 = phi i16* [ %pin2.sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
+  %pin1 = phi i16* [ %pin1.add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %pin2.add1 = getelementptr i16, i16* %pin2, i32 1
+  %In2 = load i16, i16* %pin2.add1, align 2
+  %pin1.sub2 = getelementptr i16, i16* %pin1, i32 -2
+  %In1 = load i16, i16* %pin1.sub2, align 2
+  %In2.1 = load i16, i16* %pin2, align 2
+  %pin1.sub1 = getelementptr i16, i16* %pin1, i32 -1
+  %In1.1 = load i16, i16* %pin1.sub1, align 2
+  %pin2.sub1 = getelementptr i16, i16* %pin2, i32 -1
+  %In2.2 = load i16, i16* %pin2.sub1, align 2
+  %In1.2 = load i16, i16* %pin1, align 2
+  %pin2.sub2 = getelementptr i16, i16* %pin2, i32 -2
+  %In2.3 = load i16, i16* %pin2.sub2, align 2
+  %pin1.add1 = getelementptr i16, i16* %pin1, i32 1
+  %In1.3 = load i16, i16* %pin1.add1, align 2
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %mul = mul nsw i32 %sextIn2, %sextIn1
+  %sext.mul = sext i32 %mul to i64
+  %add = add nsw i64 %sext.mul, %sum.010
+  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %add.2 = add nsw i64 %sext.mul.2, %add.1
+  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add.3 = add nsw i64 %sext.mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pin1.add4 = getelementptr i16, i16* %pin1, i32 4
+  %pin2.sub4 = getelementptr i16, i16* %pin2, i32 -4
+  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}

test/CodeGen/ARM/ParallelDSP/smlaldx-2.ll

+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smlaldx
+; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
+; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
+; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
+; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
+; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
+; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
+; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
+; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sub.epil = sub i32 %j, %i.011.epil
+  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
+  %2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+for.body:
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
+  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
+  %In2 = load i16, i16* %pIn2.0, align 2
+  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
+  %In1 = load i16, i16* %pIn1.0, align 2
+  %inc = or i32 %i.011, 1
+  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
+  %In2.1 = load i16, i16* %pIn2.1, align 2
+  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
+  %In1.1 = load i16, i16* %pIn1.1, align 2
+  %inc.1 = or i32 %i.011, 2
+  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
+  %In2.2 = load i16, i16* %pIn2.2, align 2
+  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
+  %In1.2 = load i16, i16* %pIn1.2, align 2
+  %inc.2 = or i32 %i.011, 3
+  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
+  %In2.3 = load i16, i16* %pIn2.3, align 2
+  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
+  %In1.3 = load i16, i16* %pIn1.3, align 2
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %mul = mul nsw i32 %sextIn1, %sextIn2
+  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
+  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
+  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
+  %sext.mul = sext i32 %mul to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add = add nsw i64 %sum.010, %sext.mul
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %add.2 = add nsw i64 %add.1, %sext.mul.2
+  %add.3 = add nsw i64 %sext.mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define i64 @smlaldx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
+  %2 = add i32 %j, -1
+  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
+
+for.body.epil.preheader:
+  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
+  %3 = sub i32 %j, %i.011.unr
+  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
+  %4 = sub i32 0, %xtraiter
+  br label %for.body.epil
+
+for.body.epil:
+  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
+  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
+  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
+  %5 = load i16, i16* %lsr.iv3, align 2
+  %conv.epil = sext i16 %5 to i32
+  %6 = load i16, i16* %lsr.iv, align 2
+  %conv2.epil = sext i16 %6 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
+  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
+  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
+  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+; CHECK-LABEL: smlaldx_swap
+; CHECK: for.body.preheader.new:
+; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
+; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
+
+; CHECK: for.body:
+; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
+; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
+; CHECK: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
+; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
+
+; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
+; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
+; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
+; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC0]])
+
+; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
+; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
+; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
+; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN1]], i32 [[IN2_2]], i64 [[ACC1]])
+
+; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
+; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
+
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+for.body:
+  %pin2 = phi i16* [ %pin2.sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
+  %pin1 = phi i16* [ %pin1.add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %pin2.add1 = getelementptr i16, i16* %pin2, i32 1
+  %In2 = load i16, i16* %pin2.add1, align 2
+  %pin1.sub2 = getelementptr i16, i16* %pin1, i32 -2
+  %In1 = load i16, i16* %pin1.sub2, align 2
+  %In2.1 = load i16, i16* %pin2, align 2
+  %pin1.sub1 = getelementptr i16, i16* %pin1, i32 -1
+  %In1.1 = load i16, i16* %pin1.sub1, align 2
+  %pin2.sub1 = getelementptr i16, i16* %pin2, i32 -1
+  %In2.2 = load i16, i16* %pin2.sub1, align 2
+  %In1.2 = load i16, i16* %pin1, align 2
+  %pin2.sub2 = getelementptr i16, i16* %pin2, i32 -2
+  %In2.3 = load i16, i16* %pin2.sub2, align 2
+  %pin1.add1 = getelementptr i16, i16* %pin1, i32 1
+  %In1.3 = load i16, i16* %pin1.add1, align 2
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %mul = mul nsw i32 %sextIn2, %sextIn1
+  %sext.mul = sext i32 %mul to i64
+  %add = add nsw i64 %sext.mul, %sum.010
+  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %add.2 = add nsw i64 %add.1, %sext.mul.2
+  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add.3 = add nsw i64 %add.2, %sext.mul.3
+  %inc.3 = add i32 %i.011, 4
+  %pin1.add4 = getelementptr i16, i16* %pin1, i32 4
+  %pin2.sub4 = getelementptr i16, i16* %pin2, i32 -4
+  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}

test/CodeGen/ARM/debug-info-branch-folding.ll

 ; RUN: llc < %s - | FileCheck %s
 target datalayout = "e-p:32:32:32-i1:8:32-i8:8:32-i16:16:32-i32:32:32-i64:32:32-f32:32:32-f64:32:32-v64:32:64-v128:32:128-a0:0:32-n32"
 target triple = "thumbv7-apple-macosx10.6.7"
 
-;CHECK:   vadd.f32  q4, q8, q9
+;CHECK:   vadd.f32  q0, q8, q9
 ;CHECK-NEXT: LBB0_1
 
-;CHECK:         @DEBUG_VALUE: x <- $q4{{$}}
-;CHECK-NEXT:    @DEBUG_VALUE: y <- $q4{{$}}
+;CHECK:         @DEBUG_VALUE: x <- $q0{{$}}
+;CHECK-NEXT:    @DEBUG_VALUE: y <- $q0{{$}}
 ;CHECK:         beq LBB0_1
 
 
 @.str = external constant [13 x i8]
 
 declare <4 x float> @test0001(float) nounwind readnone ssp
 
 define i32 @main(i32 %argc, i8** nocapture %argv, i1 %cond, <4 x float> %x) nounwind ssp !dbg !10 {

test/CodeGen/ARM/smlad0.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; The Cortex-M0 does not support unaligned accesses:
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-;
-; Check DSP extension:
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-
-define dso_local i32 @OneReduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-;
-; CHECK-LABEL: @OneReduction
-; CHECK:  %mac1{{\.}}026 = phi i32 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
-; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
-; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
-; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
-; CHECK:  [[V8]] = call i32 @llvm.arm.smlad(i32 [[V5]], i32 [[V7]], i32 %mac1{{\.}}026)
-; CHECK-NOT: call i32 @llvm.arm.smlad
-;
-; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
-;
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-; One reduction statement here:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-
-; Here the Mul is the LHS, and the Add the RHS.
-  %add11 = add i32 %mul9, %add10
-
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-
-define dso_local arm_aapcs_vfpcc i32 @TwoReductions(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-;
-; CHECK-LABEL: @TwoReductions
-;
-; CHECK:  %mac1{{\.}}058 = phi i32 [ [[V10:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  %mac2{{\.}}057 = phi i32 [ [[V17:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  [[V10]] = call i32 @llvm.arm.smlad(i32 %{{.*}}, i32 %{{.*}}, i32 %mac1{{\.}}058)
-; CHECK:  [[V17]] = call i32 @llvm.arm.smlad(i32 %{{.*}}, i32 %{{.*}}, i32 %mac2{{\.}}057)
-; CHECK-NOT: call i32 @llvm.arm.smlad
-;
-entry:
-  %cmp55 = icmp sgt i32 %arg, 0
-  br i1 %cmp55, label %for.body.preheader, label %for.cond.cleanup
-
-for.cond.cleanup:
-  %mac2.0.lcssa = phi i32 [ 0, %entry ], [ %add28, %for.body ]
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add16, %for.body ]
-  %add30 = add nsw i32 %mac1.0.lcssa, %mac2.0.lcssa
-  ret i32 %add30
-
-for.body.preheader:
-  br label %for.body
-
-for.body:
-; And two reduction statements here:
-  %mac1.058 = phi i32 [ %add16, %for.body ], [ 0, %for.body.preheader ]
-  %mac2.057 = phi i32 [ %add28, %for.body ], [ 0, %for.body.preheader ]
-
-  %i.056 = phi i32 [ %add29, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.056
-  %0 = load i16, i16* %arrayidx, align 2
-  %add1 = or i32 %i.056, 1
-  %arrayidx2 = getelementptr inbounds i16, i16* %arg3, i32 %add1
-  %1 = load i16, i16* %arrayidx2, align 2
-  %add3 = or i32 %i.056, 2
-  %arrayidx4 = getelementptr inbounds i16, i16* %arg3, i32 %add3
-  %2 = load i16, i16* %arrayidx4, align 2
-
-  %add5 = or i32 %i.056, 3
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg3, i32 %add5
-  %3 = load i16, i16* %arrayidx6, align 2
-  %arrayidx8 = getelementptr inbounds i16, i16* %arg2, i32 %i.056
-  %4 = load i16, i16* %arrayidx8, align 2
-  %conv = sext i16 %4 to i32
-  %conv9 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv9
-  %arrayidx11 = getelementptr inbounds i16, i16* %arg2, i32 %add1
-  %5 = load i16, i16* %arrayidx11, align 2
-  %conv12 = sext i16 %5 to i32
-  %conv13 = sext i16 %1 to i32
-  %mul14 = mul nsw i32 %conv12, %conv13
-  %add15 = add i32 %mul, %mac1.058
-  %add16 = add i32 %add15, %mul14
-  %arrayidx18 = getelementptr inbounds i16, i16* %arg2, i32 %add3
-  %6 = load i16, i16* %arrayidx18, align 2
-  %conv19 = sext i16 %6 to i32
-  %conv20 = sext i16 %2 to i32
-  %mul21 = mul nsw i32 %conv19, %conv20
-  %arrayidx23 = getelementptr inbounds i16, i16* %arg2, i32 %add5
-  %7 = load i16, i16* %arrayidx23, align 2
-  %conv24 = sext i16 %7 to i32
-  %conv25 = sext i16 %3 to i32
-  %mul26 = mul nsw i32 %conv24, %conv25
-  %add27 = add i32 %mul21, %mac2.057
-  %add28 = add i32 %add27, %mul26
-  %add29 = add nuw nsw i32 %i.056, 4
-  %cmp = icmp slt i32 %add29, %arg
-  br i1 %cmp, label %for.body, label %for.cond.cleanup
-}
-
-define i32 @one_zext(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-; CHECK-LABEL: @one_zext
-; CHECK-NOT: call i32 @llvm.arm.smlad
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = zext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = zext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-  %add11 = add i32 %mul9, %add10
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-
-define i32 @two_zext(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-; CHECK-LABEL: @two_zext
-; CHECK-NOT: call i32 @llvm.arm.smlad
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = zext i16 %2 to i32
-  %conv4 = zext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = zext i16 %3 to i32
-  %conv8 = zext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-  %add11 = add i32 %mul9, %add10
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}

test/CodeGen/ARM/smlad1.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-
-; CHECK-LABEL: @test1
-; CHECK:  %mac1{{\.}}026 = phi i32 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
-; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
-; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
-; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
-; CHECK:  [[V8]] = call i32 @llvm.arm.smlad(i32 [[V5]], i32 [[V7]], i32 %mac1{{\.}}026)
-
-define dso_local i32 @test1(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-
-; And here the Add is the LHS, the Mul the RHS
-  %add11 = add i32 %add10, %mul9
-
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-
-; Here we have i8 loads, which we do want to support, but don't handle yet.
-;
-; CHECK-LABEL: @test2
-; CHECK-NOT:   call i32 @llvm.arm.smlad
-;
-define dso_local i32 @test2(i32 %arg, i32* nocapture readnone %arg1, i8* nocapture readonly %arg2, i8* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i8, i8* %arg3, align 2
-  %.pre27 = load i8, i8* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i8, i8* %arg3, i32 %i.025
-  %0 = load i8, i8* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i8, i8* %arg3, i32 %add
-  %1 = load i8, i8* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i8, i8* %arg2, i32 %i.025
-  %2 = load i8, i8* %arrayidx3, align 2
-  %conv = sext i8 %2 to i32
-  %conv4 = sext i8 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i8, i8* %arg2, i32 %add
-  %3 = load i8, i8* %arrayidx6, align 2
-  %conv7 = sext i8 %3 to i32
-  %conv8 = sext i8 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-  %add11 = add i32 %add10, %mul9
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-
-

test/CodeGen/ARM/smlad10.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; Reduction statement is an i64 type: we only support i32 so check that the
-; rewrite isn't triggered.
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-define dso_local i64 @test(i64 %arg, i64* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i64 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
-  ret i64 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i64 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i64 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i64 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i64 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i64 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i64
-  %conv4 = sext i16 %0 to i64
-  %mul = mul nsw i64 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i64 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i64
-  %conv8 = sext i16 %1 to i64
-  %mul9 = mul nsw i64 %conv7, %conv8
-  %add10 = add i64 %mul, %mac1.026
-
-  %add11 = add i64 %mul9, %add10
-
-  %exitcond = icmp ne i64 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-

test/CodeGen/ARM/smlad11.ll

-; REQUIRES: asserts
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S -stats 2>&1 | FileCheck %s
-;
-; A more complicated chain: 4 mul operations, so we expect 2 smlad calls.
-;
-; CHECK:  %mac1{{\.}}054 = phi i32 [ [[V17:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  [[V8:%[0-9]+]] = bitcast i16* %arrayidx8 to i32*
-; CHECK:  [[V9:%[0-9]+]] = load i32, i32* [[V8]], align 2
-; CHECK:  [[V10:%[0-9]+]] = bitcast i16* %arrayidx to i32*
-; CHECK:  [[V11:%[0-9]+]] = load i32, i32* [[V10]], align 2
-; CHECK:  [[V12:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V9]], i32 [[V11]], i32 %mac1{{\.}}054)
-; CHECK:  [[V13:%[0-9]+]] = bitcast i16* %arrayidx17 to i32*
-; CHECK:  [[V14:%[0-9]+]] = load i32, i32* [[V13]], align 2
-; CHECK:  [[V15:%[0-9]+]] = bitcast i16* %arrayidx4 to i32*
-; CHECK:  [[V16:%[0-9]+]] = load i32, i32* [[V15]], align 2
-; CHECK:  [[V17:%[0-9]+]] = call i32 @llvm.arm.smlad(i32 [[V14]], i32 [[V16]], i32 [[V12]])
-;
-; And we don't want to see a 3rd smlad:
-; CHECK-NOT: call i32 @llvm.arm.smlad
-;
-; CHECK:  2 arm-parallel-dsp - Number of smlad instructions generated
-;
-define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp52 = icmp sgt i32 %arg, 0
-  br i1 %cmp52, label %for.body.preheader, label %for.cond.cleanup
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add28, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body.preheader:
-  br label %for.body
-
-for.body:
-  %mac1.054 = phi i32 [ %add28, %for.body ], [ 0, %for.body.preheader ]
-  %i.053 = phi i32 [ %add29, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.053
-  %0 = load i16, i16* %arrayidx, align 2
-  %add1 = or i32 %i.053, 1
-  %arrayidx2 = getelementptr inbounds i16, i16* %arg3, i32 %add1
-  %1 = load i16, i16* %arrayidx2, align 2
-  %add3 = or i32 %i.053, 2
-  %arrayidx4 = getelementptr inbounds i16, i16* %arg3, i32 %add3
-  %2 = load i16, i16* %arrayidx4, align 2
-  %add5 = or i32 %i.053, 3
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg3, i32 %add5
-  %3 = load i16, i16* %arrayidx6, align 2
-  %arrayidx8 = getelementptr inbounds i16, i16* %arg2, i32 %i.053
-  %4 = load i16, i16* %arrayidx8, align 2
-  %conv = sext i16 %4 to i32
-  %conv9 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv9
-  %arrayidx11 = getelementptr inbounds i16, i16* %arg2, i32 %add1
-  %5 = load i16, i16* %arrayidx11, align 2
-  %conv12 = sext i16 %5 to i32
-  %conv13 = sext i16 %1 to i32
-  %mul14 = mul nsw i32 %conv12, %conv13
-  %arrayidx17 = getelementptr inbounds i16, i16* %arg2, i32 %add3
-  %6 = load i16, i16* %arrayidx17, align 2
-  %conv18 = sext i16 %6 to i32
-  %conv19 = sext i16 %2 to i32
-  %mul20 = mul nsw i32 %conv18, %conv19
-  %arrayidx23 = getelementptr inbounds i16, i16* %arg2, i32 %add5
-  %7 = load i16, i16* %arrayidx23, align 2
-  %conv24 = sext i16 %7 to i32
-  %conv25 = sext i16 %3 to i32
-  %mul26 = mul nsw i32 %conv24, %conv25
-  %add15 = add i32 %mul, %mac1.054
-  %add21 = add i32 %add15, %mul14
-  %add27 = add i32 %add21, %mul20
-  %add28 = add i32 %add27, %mul26
-  %add29 = add nuw nsw i32 %i.053, 4
-  %cmp = icmp slt i32 %add29, %arg
-  br i1 %cmp, label %for.body, label %for.cond.cleanup
-}

test/CodeGen/ARM/smlad12.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; The loop header is not the loop latch.
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-; This is the loop header:
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body2 ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body2 ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-  %add11 = add i32 %mul9, %add10
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body2, label %for.cond.cleanup
-
-; And this is the loop latch:
-for.body2:
-  br label %for.body
-}

test/CodeGen/ARM/smlad2.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; Operands of both muls are not symmetrical (see also comments inlined below), check
-; that the rewrite isn't triggered.
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-
-; This zero-extends the 2nd operand of %mul:
-  %conv4 = zext i16 %0 to i32
-
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-
-; And here we only have sign-extensions. Thus, the operands of
-; %mul and %mul9 are not symmetrical:
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-  %add11 = add i32 %add10, %mul9
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}

test/CodeGen/ARM/smlad3.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; The loads are not consecutive: check that the rewrite isn't triggered.
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-
-; Here we add another constants offset of 2, to make sure the
-; loads to %3 and %2 are not consecutive:
-
-  %add5 = add nuw nsw i32 %i.025, 2
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add5
-  %3 = load i16, i16* %arrayidx6, align 2
-
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-  %add11 = add i32 %add10, %mul9
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-

test/CodeGen/ARM/smlad4.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; The loads are not narrow loads: check that the rewrite isn't triggered.
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-; Arg2 is now an i32, while Arg3 is still and i16:
-;
-define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i32* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp22 = icmp sgt i32 %arg, 0
-  br i1 %cmp22, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add9, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %0 = phi i16 [ %1, %for.body ], [ %.pre, %for.body.preheader ]
-  %mac1.024 = phi i32 [ %add9, %for.body ], [ 0, %for.body.preheader ]
-  %i.023 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %add = add nuw nsw i32 %i.023, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %conv = sext i16 %0 to i32
-
-; This is a 'normal' i32 load to %2:
-  %arrayidx3 = getelementptr inbounds i32, i32* %arg2, i32 %i.023
-  %2 = load i32, i32* %arrayidx3, align 4
-
-; This mul has now 1 operand which is a narrow load, and the other a normal
-; i32 load:
-  %mul = mul nsw i32 %2, %conv
-
-  %add4 = add nuw nsw i32 %i.023, 2
-  %arrayidx5 = getelementptr inbounds i32, i32* %arg2, i32 %add4
-  %3 = load i32, i32* %arrayidx5, align 4
-  %conv6 = sext i16 %1 to i32
-  %mul7 = mul nsw i32 %3, %conv6
-  %add8 = add i32 %mul, %mac1.024
-  %add9 = add i32 %add8, %mul7
-  %exitcond = icmp eq i32 %add, %arg
-  br i1 %exitcond, label %for.cond.cleanup, label %for.body
-}

test/CodeGen/ARM/smlad5.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; The loads are volatile loads: check that the rewrite isn't triggered.
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load volatile i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load volatile i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load volatile i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load volatile i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-  %add11 = add i32 %add10, %mul9
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-

test/CodeGen/ARM/smlad6.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; Alias check: check that the rewrite isn't triggered when there's a store
-; instruction possibly aliasing any mul load operands; arguments are passed
-; without 'restrict' enabled.
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-define dso_local i32 @test(i32 %arg, i32* nocapture %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-
-; Store inserted here, aliasing with arrayidx, arrayidx1, arrayidx3
-  store i16 42, i16* %arrayidx, align 2
-
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-  %add11 = add i32 %mul9, %add10
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-

test/CodeGen/ARM/smlad7.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; Alias check: check that the rewrite isn't triggered when there's a store
-; aliasing one of the mul load operands. Arguments are now annotated with
-; 'noalias'.
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-define dso_local i32 @test(i32 %arg, i32* noalias %arg1, i16* noalias readonly %arg2, i16* noalias readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-
-; Store inserted here, aliasing only with loads from 'arrayidx'.
-  store i16 42, i16* %arrayidx, align 2
-
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %add10 = add i32 %mul, %mac1.026
-
-; Here the Mul is the LHS, and the Add the RHS.
-  %add11 = add i32 %mul9, %add10
-
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-

test/CodeGen/ARM/smlad8.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; Mul with operands that are not simple load and sext/zext chains: this is not
-; yet supported so the rewrite shouldn't trigger (but we do want to support this
-; soon).
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3, i16* %arg4) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  %gep0 = getelementptr inbounds i16, i16* %arg4, i32 0
-  %gep1 = getelementptr inbounds i16, i16* %arg4, i32 1
-  %.add4 = load i16, i16* %gep0, align 2
-  %.add5 = load i16, i16* %gep1, align 2
-  %.zext4 = zext i16 %.add4 to i32
-  %.zext5 = zext i16 %.add5 to i32
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %add1 = add i32 %conv, %.zext4
-
-; This mul has a more complicated pattern as an operand, %add1
-; is another add and load, which we don't support for now.
-  %mul = mul nsw i32 %add1, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %add2 = add i32 %conv7, %.zext5
-
-; Same here
-  %mul9 = mul nsw i32 %add2, %conv8
-  %add10 = add i32 %mul, %mac1.026
-
-  %add11 = add i32 %mul9, %add10
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}

test/CodeGen/ARM/smlad9.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; Muls with operands that are constants: not yet supported, so the rewrite
-; should not trigger (but we do want to add this soon).
-;
-; CHECK-NOT:  call i32 @llvm.arm.smlad
-;
-define dso_local i32 @test(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i32 [ 0, %entry ], [ %add11, %for.body ]
-  ret i32 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i32 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %v2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %v2 to i32
-
-; RHS operand of this mul is a constant
-  %mul = mul nsw i32 %conv, 43
-
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %v3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %v3 to i32
-
-; And this RHS operand is a constant too.
-  %mul9 = mul nsw i32 %conv7, 42
-
-  %add10 = add i32 %mul, %mac1.026
-  %add11 = add i32 %mul9, %add10
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-

test/CodeGen/ARM/smladx-1.ll

-; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-
-define i32 @smladx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
-
-; CHECK-LABEL: smladx
-; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
-; CHECK: [[ACC0:%[^ ]+]] = phi i32 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
-; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
-; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
-; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
-; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
-; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN23]], i32 [[IN12]], i32 [[ACC0]])
-; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
-; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
-; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
-; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
-; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN21]], i32 [[IN10]], i32 [[ACC1]])
-; CHECK-NOT: call i32 @llvm.arm.smlad
-; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
-
-entry:
-  %cmp9 = icmp eq i32 %limit, 0
-  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
-
-for.body.preheader:
-  %0 = add i32 %limit, -1
-  %xtraiter = and i32 %limit, 3
-  %1 = icmp ult i32 %0, 3
-  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
-
-for.body.preheader.new:
-  %unroll_iter = sub i32 %limit, %xtraiter
-  br label %for.body
-
-for.cond.cleanup.loopexit.unr-lcssa:
-  %add.lcssa.ph = phi i32 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
-  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
-  %sum.010.unr = phi i32 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
-  %lcmp.mod = icmp eq i32 %xtraiter, 0
-  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
-
-for.body.epil:
-  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
-  %sum.010.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
-  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
-  %sub.epil = sub i32 %j, %i.011.epil
-  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
-  %2 = load i16, i16* %arrayidx.epil, align 2
-  %conv.epil = sext i16 %2 to i32
-  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
-  %3 = load i16, i16* %arrayidx1.epil, align 2
-  %conv2.epil = sext i16 %3 to i32
-  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
-  %add.epil = add nsw i32 %mul.epil, %sum.010.epil
-  %inc.epil = add nuw i32 %i.011.epil, 1
-  %epil.iter.sub = add i32 %epil.iter, -1
-  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
-  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
-
-for.cond.cleanup:
-  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
-  ret i32 %sum.0.lcssa
-
-for.body:
-  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
-  %sum.010 = phi i32 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
-  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
-  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
-  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
-  %In2 = load i16, i16* %pIn2.0, align 2
-  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
-  %In1 = load i16, i16* %pIn1.0, align 2
-  %inc = or i32 %i.011, 1
-  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
-  %In2.1 = load i16, i16* %pIn2.1, align 2
-  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
-  %In1.1 = load i16, i16* %pIn1.1, align 2
-  %inc.1 = or i32 %i.011, 2
-  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
-  %In2.2 = load i16, i16* %pIn2.2, align 2
-  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
-  %In1.2 = load i16, i16* %pIn1.2, align 2
-  %inc.2 = or i32 %i.011, 3
-  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
-  %In2.3 = load i16, i16* %pIn2.3, align 2
-  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
-  %In1.3 = load i16, i16* %pIn1.3, align 2
-  %sextIn1 = sext i16 %In1 to i32
-  %sextIn1.1 = sext i16 %In1.1 to i32
-  %sextIn1.2 = sext i16 %In1.2 to i32
-  %sextIn1.3 = sext i16 %In1.3 to i32
-  %sextIn2 = sext i16 %In2 to i32
-  %sextIn2.1 = sext i16 %In2.1 to i32
-  %sextIn2.2 = sext i16 %In2.2 to i32
-  %sextIn2.3 = sext i16 %In2.3 to i32
-  %mul = mul nsw i32 %sextIn1, %sextIn2
-  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
-  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
-  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
-  %add = add nsw i32 %mul, %sum.010
-  %add.1 = add nsw i32 %mul.1, %add
-  %add.2 = add nsw i32 %mul.2, %add.1
-  %add.3 = add nsw i32 %mul.3, %add.2
-  %inc.3 = add i32 %i.011, 4
-  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
-  %niter.nsub.3 = add i32 %niter, -4
-  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
-  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
-}
-
-define i32 @smladx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
-
-; CHECK-LABEL: smladx_swap
-; CHECK: for.body.preheader.new:
-; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
-; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
-
-; CHECK: for.body:
-; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
-; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
-; CHECK: [[IV:%[^ ]+]] = phi i32
-; CHECK: [[ACC0:%[^ ]+]] = phi i32 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
-
-; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
-; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
-
-
-; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
-; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
-; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
-; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
-; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN2_2]], i32 [[IN1]], i32 [[ACC0]])
-
-; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
-; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
-; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
-; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
-; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN2]], i32 [[IN1_2]], i32 [[ACC1]])
-
-; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
-; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
-
-; CHECK-NOT: call i32 @llvm.arm.smlad
-; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
-
-entry:
-  %cmp9 = icmp eq i32 %limit, 0
-  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
-
-for.body.preheader:
-  %0 = add i32 %limit, -1
-  %xtraiter = and i32 %limit, 3
-  %1 = icmp ult i32 %0, 3
-  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
-
-for.body.preheader.new:
-  %unroll_iter = sub i32 %limit, %xtraiter
-  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
-  %2 = add i32 %j, -1
-  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
-  br label %for.body
-
-for.cond.cleanup.loopexit.unr-lcssa:
-  %add.lcssa.ph = phi i32 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
-  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
-  %sum.010.unr = phi i32 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
-  %lcmp.mod = icmp eq i32 %xtraiter, 0
-  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
-
-for.body.epil.preheader:
-  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
-  %3 = sub i32 %j, %i.011.unr
-  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
-  %4 = sub i32 0, %xtraiter
-  br label %for.body.epil
-
-for.body.epil:
-  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
-  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
-  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
-  %sum.010.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
-  %5 = load i16, i16* %lsr.iv3, align 2
-  %conv.epil = sext i16 %5 to i32
-  %6 = load i16, i16* %lsr.iv, align 2
-  %conv2.epil = sext i16 %6 to i32
-  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
-  %add.epil = add nsw i32 %mul.epil, %sum.010.epil
-  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
-  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
-  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
-  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
-  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
-
-for.cond.cleanup:
-  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
-  ret i32 %sum.0.lcssa
-
-for.body:
-  %pin2 = phi i16* [ %pin2_sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
-  %pin1 = phi i16* [ %pin1_add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
-  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
-  %sum.010 = phi i32 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
-  %pin2_add1 = getelementptr i16, i16* %pin2, i32 1
-  %In2 = load i16, i16* %pin2_add1, align 2
-  %pin1_sub2 = getelementptr i16, i16* %pin1, i32 -2
-  %In1 = load i16, i16* %pin1_sub2, align 2
-  %In2.1 = load i16, i16* %pin2, align 2
-  %pin1_sub1 = getelementptr i16, i16* %pin1, i32 -1
-  %In1.1 = load i16, i16* %pin1_sub1, align 2
-  %pin2_sub1 = getelementptr i16, i16* %pin2, i32 -1
-  %In2.2 = load i16, i16* %pin2_sub1, align 2
-  %In1.2 = load i16, i16* %pin1, align 2
-  %pin2_sub2 = getelementptr i16, i16* %pin2, i32 -2
-  %In2.3 = load i16, i16* %pin2_sub2, align 2
-  %pin1_add1 = getelementptr i16, i16* %pin1, i32 1
-  %In1.3 = load i16, i16* %pin1_add1, align 2
-  %sextIn2 = sext i16 %In2 to i32
-  %sextIn1 = sext i16 %In1 to i32
-  %sextIn2.1 = sext i16 %In2.1 to i32
-  %sextIn1.1 = sext i16 %In1.1 to i32
-  %sextIn2.2 = sext i16 %In2.2 to i32
-  %sextIn1.2 = sext i16 %In1.2 to i32
-  %sextIn2.3 = sext i16 %In2.3 to i32
-  %sextIn1.3 = sext i16 %In1.3 to i32
-  %mul = mul nsw i32 %sextIn2, %sextIn1
-  %add = add nsw i32 %mul, %sum.010
-  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
-  %add.1 = add nsw i32 %mul.1, %add
-  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
-  %add.2 = add nsw i32 %mul.2, %add.1
-  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
-  %add.3 = add nsw i32 %mul.3, %add.2
-  %inc.3 = add i32 %i.011, 4
-  %pin1_add4 = getelementptr i16, i16* %pin1, i32 4
-  %pin2_sub4 = getelementptr i16, i16* %pin2, i32 -4
-  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
-  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
-}

test/CodeGen/ARM/smlald0.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; The Cortex-M0 does not support unaligned accesses:
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-;
-; Check DSP extension:
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-
-define dso_local i64 @OneReduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-;
-; CHECK-LABEL: @OneReduction
-; CHECK:  %mac1{{\.}}026 = phi i64 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
-; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
-; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
-; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
-; CHECK:  [[V8]] = call i64 @llvm.arm.smlald(i32 [[V5]], i32 [[V7]], i64 %mac1{{\.}}026)
-; CHECK-NOT: call i64 @llvm.arm.smlald
-;
-; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlald
-;
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
-  ret i64 %mac1.0.lcssa
-
-for.body:
-; One reduction statement here:
-  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i64
-  %conv4 = sext i16 %0 to i64
-  %mul = mul nsw i64 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i64
-  %conv8 = sext i16 %1 to i64
-  %mul9 = mul nsw i64 %conv7, %conv8
-  %add10 = add i64 %mul, %mac1.026
-
-; Here the Mul is the LHS, and the Add the RHS.
-  %add11 = add i64 %mul9, %add10
-
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-
-define dso_local arm_aapcs_vfpcc i64 @TwoReductions(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-;
-; CHECK-LABEL: @TwoReductions
-;
-; CHECK:  %mac1{{\.}}058 = phi i64 [ [[V10:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  %mac2{{\.}}057 = phi i64 [ [[V17:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  [[V10]] = call i64 @llvm.arm.smlald(i32 %{{.*}}, i32 %{{.*}}, i64 %mac1{{\.}}058)
-; CHECK:  [[V17]] = call i64 @llvm.arm.smlald(i32 %{{.*}}, i32 %{{.*}}, i64 %mac2{{\.}}057)
-; CHECK-NOT: call i64 @llvm.arm.smlald
-;
-entry:
-  %cmp55 = icmp sgt i32 %arg, 0
-  br i1 %cmp55, label %for.body.preheader, label %for.cond.cleanup
-
-for.cond.cleanup:
-  %mac2.0.lcssa = phi i64 [ 0, %entry ], [ %add28, %for.body ]
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add16, %for.body ]
-  %add30 = add nsw i64 %mac1.0.lcssa, %mac2.0.lcssa
-  ret i64 %add30
-
-for.body.preheader:
-  br label %for.body
-
-for.body:
-; And two reduction statements here:
-  %mac1.058 = phi i64 [ %add16, %for.body ], [ 0, %for.body.preheader ]
-  %mac2.057 = phi i64 [ %add28, %for.body ], [ 0, %for.body.preheader ]
-
-  %i.056 = phi i32 [ %add29, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.056
-  %0 = load i16, i16* %arrayidx, align 2
-  %add1 = or i32 %i.056, 1
-  %arrayidx2 = getelementptr inbounds i16, i16* %arg3, i32 %add1
-  %1 = load i16, i16* %arrayidx2, align 2
-  %add3 = or i32 %i.056, 2
-  %arrayidx4 = getelementptr inbounds i16, i16* %arg3, i32 %add3
-  %2 = load i16, i16* %arrayidx4, align 2
-
-  %add5 = or i32 %i.056, 3
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg3, i32 %add5
-  %3 = load i16, i16* %arrayidx6, align 2
-  %arrayidx8 = getelementptr inbounds i16, i16* %arg2, i32 %i.056
-  %4 = load i16, i16* %arrayidx8, align 2
-  %conv = sext i16 %4 to i64
-  %conv9 = sext i16 %0 to i64
-  %mul = mul nsw i64 %conv, %conv9
-  %arrayidx11 = getelementptr inbounds i16, i16* %arg2, i32 %add1
-  %5 = load i16, i16* %arrayidx11, align 2
-  %conv12 = sext i16 %5 to i64
-  %conv13 = sext i16 %1 to i64
-  %mul14 = mul nsw i64 %conv12, %conv13
-  %add15 = add i64 %mul, %mac1.058
-  %add16 = add i64 %add15, %mul14
-  %arrayidx18 = getelementptr inbounds i16, i16* %arg2, i32 %add3
-  %6 = load i16, i16* %arrayidx18, align 2
-  %conv19 = sext i16 %6 to i64
-  %conv20 = sext i16 %2 to i64
-  %mul21 = mul nsw i64 %conv19, %conv20
-  %arrayidx23 = getelementptr inbounds i16, i16* %arg2, i32 %add5
-  %7 = load i16, i16* %arrayidx23, align 2
-  %conv24 = sext i16 %7 to i64
-  %conv25 = sext i16 %3 to i64
-  %mul26 = mul nsw i64 %conv24, %conv25
-  %add27 = add i64 %mul21, %mac2.057
-  %add28 = add i64 %add27, %mul26
-  %add29 = add nuw nsw i32 %i.056, 4
-  %cmp = icmp slt i32 %add29, %arg
-  br i1 %cmp, label %for.body, label %for.cond.cleanup
-}
-
-define i64 @reduction_zext(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-; CHECK-LABEL: @reduction_zext
-; CHECK-NOT: call i64 @llvm.arm.smlald
-; CHECK-NOT: call i32 @llvm.arm.smlad
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
-  ret i64 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i64
-  %conv4 = zext i16 %0 to i64
-  %mul = mul nsw i64 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i64
-  %conv8 = zext i16 %1 to i64
-  %mul9 = mul nsw i64 %conv7, %conv8
-  %add10 = add i64 %mul, %mac1.026
-  %add11 = add i64 %mul9, %add10
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}

test/CodeGen/ARM/smlald1.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-
-; CHECK-LABEL: @test1
-; CHECK:  %mac1{{\.}}026 = phi i64 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
-; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
-; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
-; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
-; CHECK:  [[V8]] = call i64 @llvm.arm.smlald(i32 [[V5]], i32 [[V7]], i64 %mac1{{\.}}026)
-
-define dso_local i64 @test1(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
-  ret i64 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i64
-  %conv4 = sext i16 %0 to i64
-  %mul = mul nsw i64 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i64
-  %conv8 = sext i16 %1 to i64
-  %mul9 = mul nsw i64 %conv7, %conv8
-  %add10 = add i64 %mul, %mac1.026
-
-; And here the Add is the LHS, the Mul the RHS
-  %add11 = add i64 %add10, %mul9
-
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-
-; Here we have i8 loads, which we do want to support, but don't handle yet.
-;
-; CHECK-LABEL: @test2
-; CHECK-NOT:   call i64 @llvm.arm.smlad
-;
-define dso_local i64 @test2(i32 %arg, i32* nocapture readnone %arg1, i8* nocapture readonly %arg2, i8* nocapture readonly %arg3) {
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i8, i8* %arg3, align 2
-  %.pre27 = load i8, i8* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
-  ret i64 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i8, i8* %arg3, i32 %i.025
-  %0 = load i8, i8* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i8, i8* %arg3, i32 %add
-  %1 = load i8, i8* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i8, i8* %arg2, i32 %i.025
-  %2 = load i8, i8* %arrayidx3, align 2
-  %conv = sext i8 %2 to i64
-  %conv4 = sext i8 %0 to i64
-  %mul = mul nsw i64 %conv, %conv4
-  %arrayidx6 = getelementptr inbounds i8, i8* %arg2, i32 %add
-  %3 = load i8, i8* %arrayidx6, align 2
-  %conv7 = sext i8 %3 to i64
-  %conv8 = sext i8 %1 to i64
-  %mul9 = mul nsw i64 %conv7, %conv8
-  %add10 = add i64 %mul, %mac1.026
-  %add11 = add i64 %add10, %mul9
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-

test/CodeGen/ARM/smlald2.ll

-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 < %s -arm-parallel-dsp -S | FileCheck %s
-;
-; The Cortex-M0 does not support unaligned accesses:
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-;
-; Check DSP extension:
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-
-define dso_local i64 @OneReduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-;
-; CHECK-LABEL: @OneReduction
-; CHECK:  %mac1{{\.}}026 = phi i64 [ [[V8:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  [[V4:%[0-9]+]] = bitcast i16* %arrayidx3 to i32*
-; CHECK:  [[V5:%[0-9]+]] = load i32, i32* [[V4]], align 2
-; CHECK:  [[V6:%[0-9]+]] = bitcast i16* %arrayidx to i32*
-; CHECK:  [[V7:%[0-9]+]] = load i32, i32* [[V6]], align 2
-; CHECK:  [[V8]] = call i64 @llvm.arm.smlald(i32 [[V5]], i32 [[V7]], i64 %mac1{{\.}}026)
-; CHECK-NOT: call i64 @llvm.arm.smlald
-;
-; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlald
-;
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
-  ret i64 %mac1.0.lcssa
-
-for.body:
-; One reduction statement here:
-  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %sext0 = sext i32 %mul to i64
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %sext1 = sext i32 %mul9 to i64
-  %add10 = add i64 %sext0, %mac1.026
-
-; Here the Mul is the LHS, and the Add the RHS.
-  %add11 = add i64 %sext1, %add10
-
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-
-define dso_local arm_aapcs_vfpcc i64 @TwoReductions(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-;
-; CHECK-LABEL: @TwoReductions
-;
-; CHECK:  %mac1{{\.}}058 = phi i64 [ [[V10:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  %mac2{{\.}}057 = phi i64 [ [[V17:%[0-9]+]], %for.body ], [ 0, %for.body.preheader ]
-; CHECK:  [[V10]] = call i64 @llvm.arm.smlald(i32 %{{.*}}, i32 %{{.*}}, i64 %mac1{{\.}}058)
-; CHECK:  [[V17]] = call i64 @llvm.arm.smlald(i32 %{{.*}}, i32 %{{.*}}, i64 %mac2{{\.}}057)
-; CHECK-NOT: call i64 @llvm.arm.smlald
-;
-entry:
-  %cmp55 = icmp sgt i32 %arg, 0
-  br i1 %cmp55, label %for.body.preheader, label %for.cond.cleanup
-
-for.cond.cleanup:
-  %mac2.0.lcssa = phi i64 [ 0, %entry ], [ %add28, %for.body ]
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add16, %for.body ]
-  %add30 = add nsw i64 %mac1.0.lcssa, %mac2.0.lcssa
-  ret i64 %add30
-
-for.body.preheader:
-  br label %for.body
-
-for.body:
-; And two reduction statements here:
-  %mac1.058 = phi i64 [ %add16, %for.body ], [ 0, %for.body.preheader ]
-  %mac2.057 = phi i64 [ %add28, %for.body ], [ 0, %for.body.preheader ]
-
-  %i.056 = phi i32 [ %add29, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.056
-  %0 = load i16, i16* %arrayidx, align 2
-  %add1 = or i32 %i.056, 1
-  %arrayidx2 = getelementptr inbounds i16, i16* %arg3, i32 %add1
-  %1 = load i16, i16* %arrayidx2, align 2
-  %add3 = or i32 %i.056, 2
-  %arrayidx4 = getelementptr inbounds i16, i16* %arg3, i32 %add3
-  %2 = load i16, i16* %arrayidx4, align 2
-
-  %add5 = or i32 %i.056, 3
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg3, i32 %add5
-  %3 = load i16, i16* %arrayidx6, align 2
-  %arrayidx8 = getelementptr inbounds i16, i16* %arg2, i32 %i.056
-  %4 = load i16, i16* %arrayidx8, align 2
-  %conv = sext i16 %4 to i32
-  %conv9 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv9
-  %sext0 = sext i32 %mul to i64
-  %arrayidx11 = getelementptr inbounds i16, i16* %arg2, i32 %add1
-  %5 = load i16, i16* %arrayidx11, align 2
-  %conv12 = sext i16 %5 to i32
-  %conv13 = sext i16 %1 to i32
-  %mul14 = mul nsw i32 %conv12, %conv13
-  %sext1 = sext i32 %mul14 to i64
-  %add15 = add i64 %sext0, %mac1.058
-  %add16 = add i64 %add15, %sext1
-  %arrayidx18 = getelementptr inbounds i16, i16* %arg2, i32 %add3
-  %6 = load i16, i16* %arrayidx18, align 2
-  %conv19 = sext i16 %6 to i32
-  %conv20 = sext i16 %2 to i32
-  %mul21 = mul nsw i32 %conv19, %conv20
-  %sext2 = sext i32 %mul21 to i64
-  %arrayidx23 = getelementptr inbounds i16, i16* %arg2, i32 %add5
-  %7 = load i16, i16* %arrayidx23, align 2
-  %conv24 = sext i16 %7 to i32
-  %conv25 = sext i16 %3 to i32
-  %mul26 = mul nsw i32 %conv24, %conv25
-  %sext3 = sext i32 %mul26 to i64
-  %add27 = add i64 %sext2, %mac2.057
-  %add28 = add i64 %add27, %sext3
-  %add29 = add nuw nsw i32 %i.056, 4
-  %cmp = icmp slt i32 %add29, %arg
-  br i1 %cmp, label %for.body, label %for.cond.cleanup
-}
-
-define i64 @zext_mul_reduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-; CHECK-LABEL: @zext_mul_reduction
-; CHECK-NOT: call i64 @llvm.arm.smlald
-; CHECK-NOT: call i32 @llvm.arm.smlad
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
-  ret i64 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = zext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %sext0 = sext i32 %mul to i64
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = zext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %sext1 = sext i32 %mul9 to i64
-  %add10 = add i64 %sext0, %mac1.026
-  %add11 = add i64 %sext1, %add10
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}
-
-define i64 @zext_add_reduction(i32 %arg, i32* nocapture readnone %arg1, i16* nocapture readonly %arg2, i16* nocapture readonly %arg3) {
-; CHECK-LABEL: @zext_add_reduction
-; CHECK-NOT: call i64 @llvm.arm.smlald
-; CHECK-NOT: call i32 @llvm.arm.smlad
-entry:
-  %cmp24 = icmp sgt i32 %arg, 0
-  br i1 %cmp24, label %for.body.preheader, label %for.cond.cleanup
-
-for.body.preheader:
-  %.pre = load i16, i16* %arg3, align 2
-  %.pre27 = load i16, i16* %arg2, align 2
-  br label %for.body
-
-for.cond.cleanup:
-  %mac1.0.lcssa = phi i64 [ 0, %entry ], [ %add11, %for.body ]
-  ret i64 %mac1.0.lcssa
-
-for.body:
-  %mac1.026 = phi i64 [ %add11, %for.body ], [ 0, %for.body.preheader ]
-  %i.025 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
-  %arrayidx = getelementptr inbounds i16, i16* %arg3, i32 %i.025
-  %0 = load i16, i16* %arrayidx, align 2
-  %add = add nuw nsw i32 %i.025, 1
-  %arrayidx1 = getelementptr inbounds i16, i16* %arg3, i32 %add
-  %1 = load i16, i16* %arrayidx1, align 2
-  %arrayidx3 = getelementptr inbounds i16, i16* %arg2, i32 %i.025
-  %2 = load i16, i16* %arrayidx3, align 2
-  %conv = sext i16 %2 to i32
-  %conv4 = sext i16 %0 to i32
-  %mul = mul nsw i32 %conv, %conv4
-  %sext0 = zext i32 %mul to i64
-  %arrayidx6 = getelementptr inbounds i16, i16* %arg2, i32 %add
-  %3 = load i16, i16* %arrayidx6, align 2
-  %conv7 = sext i16 %3 to i32
-  %conv8 = sext i16 %1 to i32
-  %mul9 = mul nsw i32 %conv7, %conv8
-  %sext1 = zext i32 %mul9 to i64
-  %add10 = add i64 %sext0, %mac1.026
-  %add11 = add i64 %sext1, %add10
-  %exitcond = icmp ne i32 %add, %arg
-  br i1 %exitcond, label %for.body, label %for.cond.cleanup
-}

test/CodeGen/ARM/smlaldx-1.ll

-; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-
-define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
-
-; CHECK-LABEL: smlaldx
-; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
-; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
-; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
-; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
-; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
-; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
-; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
-; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
-; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
-; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
-; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
-; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
-; CHECK-NOT: call i64 @llvm.arm.smlad
-; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
-
-entry:
-  %cmp9 = icmp eq i32 %limit, 0
-  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
-
-for.body.preheader:
-  %0 = add i32 %limit, -1
-  %xtraiter = and i32 %limit, 3
-  %1 = icmp ult i32 %0, 3
-  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
-
-for.body.preheader.new:
-  %unroll_iter = sub i32 %limit, %xtraiter
-  br label %for.body
-
-for.cond.cleanup.loopexit.unr-lcssa:
-  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
-  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
-  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
-  %lcmp.mod = icmp eq i32 %xtraiter, 0
-  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
-
-for.body.epil:
-  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
-  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
-  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
-  %sub.epil = sub i32 %j, %i.011.epil
-  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
-  %2 = load i16, i16* %arrayidx.epil, align 2
-  %conv.epil = sext i16 %2 to i32
-  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
-  %3 = load i16, i16* %arrayidx1.epil, align 2
-  %conv2.epil = sext i16 %3 to i32
-  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
-  %sext.mul.epil = sext i32 %mul.epil to i64
-  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
-  %inc.epil = add nuw i32 %i.011.epil, 1
-  %epil.iter.sub = add i32 %epil.iter, -1
-  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
-  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
-
-for.cond.cleanup:
-  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
-  ret i64 %sum.0.lcssa
-
-for.body:
-  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
-  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
-  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
-  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
-  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
-  %In2 = load i16, i16* %pIn2.0, align 2
-  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
-  %In1 = load i16, i16* %pIn1.0, align 2
-  %inc = or i32 %i.011, 1
-  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
-  %In2.1 = load i16, i16* %pIn2.1, align 2
-  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
-  %In1.1 = load i16, i16* %pIn1.1, align 2
-  %inc.1 = or i32 %i.011, 2
-  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
-  %In2.2 = load i16, i16* %pIn2.2, align 2
-  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
-  %In1.2 = load i16, i16* %pIn1.2, align 2
-  %inc.2 = or i32 %i.011, 3
-  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
-  %In2.3 = load i16, i16* %pIn2.3, align 2
-  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
-  %In1.3 = load i16, i16* %pIn1.3, align 2
-  %sextIn1 = sext i16 %In1 to i32
-  %sextIn1.1 = sext i16 %In1.1 to i32
-  %sextIn1.2 = sext i16 %In1.2 to i32
-  %sextIn1.3 = sext i16 %In1.3 to i32
-  %sextIn2 = sext i16 %In2 to i32
-  %sextIn2.1 = sext i16 %In2.1 to i32
-  %sextIn2.2 = sext i16 %In2.2 to i32
-  %sextIn2.3 = sext i16 %In2.3 to i32
-  %mul = mul nsw i32 %sextIn1, %sextIn2
-  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
-  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
-  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
-  %sext.mul = sext i32 %mul to i64
-  %sext.mul.1 = sext i32 %mul.1 to i64
-  %sext.mul.2 = sext i32 %mul.2 to i64
-  %sext.mul.3 = sext i32 %mul.3 to i64
-  %add = add nsw i64 %sext.mul, %sum.010
-  %add.1 = add nsw i64 %sext.mul.1, %add
-  %add.2 = add nsw i64 %sext.mul.2, %add.1
-  %add.3 = add nsw i64 %sext.mul.3, %add.2
-  %inc.3 = add i32 %i.011, 4
-  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
-  %niter.nsub.3 = add i32 %niter, -4
-  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
-  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
-}
-
-define i64 @smlaldx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
-
-entry:
-  %cmp9 = icmp eq i32 %limit, 0
-  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
-
-for.body.preheader:
-  %0 = add i32 %limit, -1
-  %xtraiter = and i32 %limit, 3
-  %1 = icmp ult i32 %0, 3
-  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
-
-for.body.preheader.new:
-  %unroll_iter = sub i32 %limit, %xtraiter
-  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
-  %2 = add i32 %j, -1
-  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
-  br label %for.body
-
-for.cond.cleanup.loopexit.unr-lcssa:
-  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
-  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
-  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
-  %lcmp.mod = icmp eq i32 %xtraiter, 0
-  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
-
-for.body.epil.preheader:
-  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
-  %3 = sub i32 %j, %i.011.unr
-  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
-  %4 = sub i32 0, %xtraiter
-  br label %for.body.epil
-
-for.body.epil:
-  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
-  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
-  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
-  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
-  %5 = load i16, i16* %lsr.iv3, align 2
-  %conv.epil = sext i16 %5 to i32
-  %6 = load i16, i16* %lsr.iv, align 2
-  %conv2.epil = sext i16 %6 to i32
-  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
-  %sext.mul.epil = sext i32 %mul.epil to i64
-  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
-  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
-  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
-  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
-  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
-  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
-
-for.cond.cleanup:
-  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
-  ret i64 %sum.0.lcssa
-
-; CHECK-LABEL: smlaldx_swap
-; CHECK: for.body.preheader.new:
-; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
-; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
-
-; CHECK: for.body:
-; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
-; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
-; CHECK: [[IV:%[^ ]+]] = phi i32
-; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
-
-; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
-; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
-
-; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
-; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
-; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
-; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
-; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2_2]], i32 [[IN1]], i64 [[ACC0]])
-
-; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
-; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
-; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
-; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
-; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC1]])
-
-; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
-; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
-
-; CHECK-NOT: call i64 @llvm.arm.smlad
-; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
-
-for.body:
-  %pin2 = phi i16* [ %pin2.sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
-  %pin1 = phi i16* [ %pin1.add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
-  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
-  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
-  %pin2.add1 = getelementptr i16, i16* %pin2, i32 1
-  %In2 = load i16, i16* %pin2.add1, align 2
-  %pin1.sub2 = getelementptr i16, i16* %pin1, i32 -2
-  %In1 = load i16, i16* %pin1.sub2, align 2
-  %In2.1 = load i16, i16* %pin2, align 2
-  %pin1.sub1 = getelementptr i16, i16* %pin1, i32 -1
-  %In1.1 = load i16, i16* %pin1.sub1, align 2
-  %pin2.sub1 = getelementptr i16, i16* %pin2, i32 -1
-  %In2.2 = load i16, i16* %pin2.sub1, align 2
-  %In1.2 = load i16, i16* %pin1, align 2
-  %pin2.sub2 = getelementptr i16, i16* %pin2, i32 -2
-  %In2.3 = load i16, i16* %pin2.sub2, align 2
-  %pin1.add1 = getelementptr i16, i16* %pin1, i32 1
-  %In1.3 = load i16, i16* %pin1.add1, align 2
-  %sextIn2 = sext i16 %In2 to i32
-  %sextIn1 = sext i16 %In1 to i32
-  %sextIn2.1 = sext i16 %In2.1 to i32
-  %sextIn1.1 = sext i16 %In1.1 to i32
-  %sextIn2.2 = sext i16 %In2.2 to i32
-  %sextIn1.2 = sext i16 %In1.2 to i32
-  %sextIn2.3 = sext i16 %In2.3 to i32
-  %sextIn1.3 = sext i16 %In1.3 to i32
-  %mul = mul nsw i32 %sextIn2, %sextIn1
-  %sext.mul = sext i32 %mul to i64
-  %add = add nsw i64 %sext.mul, %sum.010
-  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
-  %sext.mul.1 = sext i32 %mul.1 to i64
-  %add.1 = add nsw i64 %sext.mul.1, %add
-  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
-  %sext.mul.2 = sext i32 %mul.2 to i64
-  %add.2 = add nsw i64 %sext.mul.2, %add.1
-  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
-  %sext.mul.3 = sext i32 %mul.3 to i64
-  %add.3 = add nsw i64 %sext.mul.3, %add.2
-  %inc.3 = add i32 %i.011, 4
-  %pin1.add4 = getelementptr i16, i16* %pin1, i32 4
-  %pin2.sub4 = getelementptr i16, i16* %pin2, i32 -4
-  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
-  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
-}

test/CodeGen/ARM/smlaldx-2.ll

-; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
-
-define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
-
-; CHECK-LABEL: smlaldx
-; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
-; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
-; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
-; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
-; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
-; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
-; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
-; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
-; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
-; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
-; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
-; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
-; CHECK-NOT: call i64 @llvm.arm.smlad
-; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
-
-entry:
-  %cmp9 = icmp eq i32 %limit, 0
-  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
-
-for.body.preheader:
-  %0 = add i32 %limit, -1
-  %xtraiter = and i32 %limit, 3
-  %1 = icmp ult i32 %0, 3
-  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
-
-for.body.preheader.new:
-  %unroll_iter = sub i32 %limit, %xtraiter
-  br label %for.body
-
-for.cond.cleanup.loopexit.unr-lcssa:
-  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
-  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
-  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
-  %lcmp.mod = icmp eq i32 %xtraiter, 0
-  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
-
-for.body.epil:
-  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
-  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
-  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
-  %sub.epil = sub i32 %j, %i.011.epil
-  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
-  %2 = load i16, i16* %arrayidx.epil, align 2
-  %conv.epil = sext i16 %2 to i32
-  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
-  %3 = load i16, i16* %arrayidx1.epil, align 2
-  %conv2.epil = sext i16 %3 to i32
-  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
-  %sext.mul.epil = sext i32 %mul.epil to i64
-  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
-  %inc.epil = add nuw i32 %i.011.epil, 1
-  %epil.iter.sub = add i32 %epil.iter, -1
-  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
-  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
-
-for.cond.cleanup:
-  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
-  ret i64 %sum.0.lcssa
-
-for.body:
-  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
-  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
-  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
-  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
-  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
-  %In2 = load i16, i16* %pIn2.0, align 2
-  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
-  %In1 = load i16, i16* %pIn1.0, align 2
-  %inc = or i32 %i.011, 1
-  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
-  %In2.1 = load i16, i16* %pIn2.1, align 2
-  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
-  %In1.1 = load i16, i16* %pIn1.1, align 2
-  %inc.1 = or i32 %i.011, 2
-  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
-  %In2.2 = load i16, i16* %pIn2.2, align 2
-  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
-  %In1.2 = load i16, i16* %pIn1.2, align 2
-  %inc.2 = or i32 %i.011, 3
-  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
-  %In2.3 = load i16, i16* %pIn2.3, align 2
-  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
-  %In1.3 = load i16, i16* %pIn1.3, align 2
-  %sextIn1 = sext i16 %In1 to i32
-  %sextIn1.1 = sext i16 %In1.1 to i32
-  %sextIn1.2 = sext i16 %In1.2 to i32
-  %sextIn1.3 = sext i16 %In1.3 to i32
-  %sextIn2 = sext i16 %In2 to i32
-  %sextIn2.1 = sext i16 %In2.1 to i32
-  %sextIn2.2 = sext i16 %In2.2 to i32
-  %sextIn2.3 = sext i16 %In2.3 to i32
-  %mul = mul nsw i32 %sextIn1, %sextIn2
-  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
-  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
-  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
-  %sext.mul = sext i32 %mul to i64
-  %sext.mul.1 = sext i32 %mul.1 to i64
-  %sext.mul.2 = sext i32 %mul.2 to i64
-  %sext.mul.3 = sext i32 %mul.3 to i64
-  %add = add nsw i64 %sum.010, %sext.mul
-  %add.1 = add nsw i64 %sext.mul.1, %add
-  %add.2 = add nsw i64 %add.1, %sext.mul.2
-  %add.3 = add nsw i64 %sext.mul.3, %add.2
-  %inc.3 = add i32 %i.011, 4
-  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
-  %niter.nsub.3 = add i32 %niter, -4
-  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
-  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
-}
-
-define i64 @smlaldx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
-
-entry:
-  %cmp9 = icmp eq i32 %limit, 0
-  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
-
-for.body.preheader:
-  %0 = add i32 %limit, -1
-  %xtraiter = and i32 %limit, 3
-  %1 = icmp ult i32 %0, 3
-  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
-
-for.body.preheader.new:
-  %unroll_iter = sub i32 %limit, %xtraiter
-  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
-  %2 = add i32 %j, -1
-  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
-  br label %for.body
-
-for.cond.cleanup.loopexit.unr-lcssa:
-  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
-  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
-  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
-  %lcmp.mod = icmp eq i32 %xtraiter, 0
-  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
-
-for.body.epil.preheader:
-  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
-  %3 = sub i32 %j, %i.011.unr
-  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
-  %4 = sub i32 0, %xtraiter
-  br label %for.body.epil
-
-for.body.epil:
-  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
-  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
-  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
-  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
-  %5 = load i16, i16* %lsr.iv3, align 2
-  %conv.epil = sext i16 %5 to i32
-  %6 = load i16, i16* %lsr.iv, align 2
-  %conv2.epil = sext i16 %6 to i32
-  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
-  %sext.mul.epil = sext i32 %mul.epil to i64
-  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
-  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
-  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
-  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
-  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
-  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
-
-for.cond.cleanup:
-  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
-  ret i64 %sum.0.lcssa
-
-; CHECK-LABEL: smlaldx_swap
-; CHECK: for.body.preheader.new:
-; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
-; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
-
-; CHECK: for.body:
-; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
-; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
-; CHECK: [[IV:%[^ ]+]] = phi i32
-; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
-; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
-; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
-
-; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
-; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
-; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
-; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
-; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC0]])
-
-; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
-; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
-; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
-; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
-; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN1]], i32 [[IN2_2]], i64 [[ACC1]])
-
-; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
-; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
-
-; CHECK-NOT: call i64 @llvm.arm.smlad
-; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
-
-for.body:
-  %pin2 = phi i16* [ %pin2.sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
-  %pin1 = phi i16* [ %pin1.add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
-  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
-  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
-  %pin2.add1 = getelementptr i16, i16* %pin2, i32 1
-  %In2 = load i16, i16* %pin2.add1, align 2
-  %pin1.sub2 = getelementptr i16, i16* %pin1, i32 -2
-  %In1 = load i16, i16* %pin1.sub2, align 2
-  %In2.1 = load i16, i16* %pin2, align 2
-  %pin1.sub1 = getelementptr i16, i16* %pin1, i32 -1
-  %In1.1 = load i16, i16* %pin1.sub1, align 2
-  %pin2.sub1 = getelementptr i16, i16* %pin2, i32 -1
-  %In2.2 = load i16, i16* %pin2.sub1, align 2
-  %In1.2 = load i16, i16* %pin1, align 2
-  %pin2.sub2 = getelementptr i16, i16* %pin2, i32 -2
-  %In2.3 = load i16, i16* %pin2.sub2, align 2
-  %pin1.add1 = getelementptr i16, i16* %pin1, i32 1
-  %In1.3 = load i16, i16* %pin1.add1, align 2
-  %sextIn2 = sext i16 %In2 to i32
-  %sextIn1 = sext i16 %In1 to i32
-  %sextIn2.1 = sext i16 %In2.1 to i32
-  %sextIn1.1 = sext i16 %In1.1 to i32
-  %sextIn2.2 = sext i16 %In2.2 to i32
-  %sextIn1.2 = sext i16 %In1.2 to i32
-  %sextIn2.3 = sext i16 %In2.3 to i32
-  %sextIn1.3 = sext i16 %In1.3 to i32
-  %mul = mul nsw i32 %sextIn2, %sextIn1
-  %sext.mul = sext i32 %mul to i64
-  %add = add nsw i64 %sext.mul, %sum.010
-  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
-  %sext.mul.1 = sext i32 %mul.1 to i64
-  %add.1 = add nsw i64 %sext.mul.1, %add
-  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
-  %sext.mul.2 = sext i32 %mul.2 to i64
-  %add.2 = add nsw i64 %add.1, %sext.mul.2
-  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
-  %sext.mul.3 = sext i32 %mul.3 to i64
-  %add.3 = add nsw i64 %add.2, %sext.mul.3
-  %inc.3 = add i32 %i.011, 4
-  %pin1.add4 = getelementptr i16, i16* %pin1, i32 4
-  %pin2.sub4 = getelementptr i16, i16* %pin2, i32 -4
-  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
-  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
-}