Index: llvm/trunk/lib/Target/ARM/ARMParallelDSP.cpp
===================================================================
--- llvm/trunk/lib/Target/ARM/ARMParallelDSP.cpp
+++ llvm/trunk/lib/Target/ARM/ARMParallelDSP.cpp
@@ -92,6 +92,7 @@
   struct BinOpChain : public OpChain {
     ValueList     LHS;      // List of all (narrow) left hand operands.
     ValueList     RHS;      // List of all (narrow) right hand operands.
+    bool Exchange = false;
 
     BinOpChain(Instruction *I, ValueList &lhs, ValueList &rhs) :
       OpChain(I, lhs), LHS(lhs), RHS(rhs) {
@@ -125,7 +126,8 @@
     bool AreSequentialLoads(LoadInst *Ld0, LoadInst *Ld1, MemInstList &VecMem);
     PMACPairList CreateParallelMACPairs(OpChainList &Candidates);
     Instruction *CreateSMLADCall(LoadInst *VecLd0, LoadInst *VecLd1,
-                                 Instruction *Acc, Instruction *InsertAfter);
+                                 Instruction *Acc, bool Exchange,
+                                 Instruction *InsertAfter);
 
     /// Try to match and generate: SMLAD, SMLADX - Signed Multiply Accumulate
     /// Dual performs two signed 16x16-bit multiplications. It adds the
@@ -198,7 +200,8 @@
       LoopAccessInfo LAI(L, SE, TLI, AA, DT, LI);
       bool Changes = false;
 
-      LLVM_DEBUG(dbgs() << "\n== Parallel DSP pass ==\n\n");
+      LLVM_DEBUG(dbgs() << "\n== Parallel DSP pass ==\n");
+      LLVM_DEBUG(dbgs() << " - " << F.getName() << "\n\n");
       Changes = MatchSMLAD(F);
       return Changes;
     }
@@ -300,6 +303,7 @@
     return false;
   }
   if (isConsecutiveAccess(MemOp0, MemOp1, DL, SE)) {
+    VecMem.clear();
     VecMem.push_back(MemOp0);
     VecMem.push_back(MemOp1);
     LLVM_DEBUG(dbgs() << "OK: accesses are consecutive.\n");
@@ -335,53 +339,90 @@
   if (Elems < 2)
     return PMACPairs;
 
-  // TODO: for now we simply try to match consecutive pairs i and i+1.
-  // We can compare all elements, but then we need to compare and evaluate
-  // different solutions.
-  for(unsigned i=0; i<Elems-1; i+=2) {
+  SmallPtrSet<const Instruction*, 4> Paired;
+  for (unsigned i = 0; i < Elems; ++i) {
     BinOpChain *PMul0 = static_cast<BinOpChain*>(Candidates[i].get());
-    BinOpChain *PMul1 = static_cast<BinOpChain*>(Candidates[i+1].get());
-    const Instruction *Mul0 = PMul0->Root;
-    const Instruction *Mul1 = PMul1->Root;
-
-    if (Mul0 == Mul1)
+    if (Paired.count(PMul0->Root))
       continue;
 
-    LLVM_DEBUG(dbgs() << "\nCheck parallel muls:\n";
-               dbgs() << "- "; Mul0->dump();
-               dbgs() << "- "; Mul1->dump());
-
-    const ValueList &Mul0_LHS = PMul0->LHS;
-    const ValueList &Mul0_RHS = PMul0->RHS;
-    const ValueList &Mul1_LHS = PMul1->LHS;
-    const ValueList &Mul1_RHS = PMul1->RHS;
-
-    if (!AreSymmetrical(Mul0_LHS, Mul1_LHS) ||
-        !AreSymmetrical(Mul0_RHS, Mul1_RHS))
-      continue;
-
-    LLVM_DEBUG(dbgs() << "OK: mul operands list match:\n");
-    // The first elements of each vector should be loads with sexts. If we find
-    // that its two pairs of consecutive loads, then these can be transformed
-    // into two wider loads and the users can be replaced with DSP
-    // intrinsics.
-    for (unsigned x = 0; x < Mul0_LHS.size(); x += 2) {
-      auto *Ld0 = dyn_cast<LoadInst>(Mul0_LHS[x]);
-      auto *Ld1 = dyn_cast<LoadInst>(Mul1_LHS[x]);
-      auto *Ld2 = dyn_cast<LoadInst>(Mul0_RHS[x]);
-      auto *Ld3 = dyn_cast<LoadInst>(Mul1_RHS[x]);
-
-      LLVM_DEBUG(dbgs() << "Looking at operands " << x << ":\n";
-                 dbgs() << "\t mul1: "; Mul0_LHS[x]->dump();
-                 dbgs() << "\t mul2: "; Mul1_LHS[x]->dump();
-                 dbgs() << "and operands " << x + 2 << ":\n";
-                 dbgs() << "\t mul1: "; Mul0_RHS[x]->dump();
-                 dbgs() << "\t mul2: "; Mul1_RHS[x]->dump());
-
-      if (AreSequentialLoads(Ld0, Ld1, PMul0->VecLd) &&
-          AreSequentialLoads(Ld2, Ld3, PMul1->VecLd)) {
-        LLVM_DEBUG(dbgs() << "OK: found two pairs of parallel loads!\n");
-        PMACPairs.push_back(std::make_pair(PMul0, PMul1));
+    for (unsigned j = 0; j < Elems; ++j) {
+      if (i == j)
+        continue;
+
+      BinOpChain *PMul1 = static_cast<BinOpChain*>(Candidates[j].get());
+      if (Paired.count(PMul1->Root))
+        continue;
+
+      const Instruction *Mul0 = PMul0->Root;
+      const Instruction *Mul1 = PMul1->Root;
+      if (Mul0 == Mul1)
+        continue;
+
+      assert(PMul0 != PMul1 && "expected different chains");
+
+      LLVM_DEBUG(dbgs() << "\nCheck parallel muls:\n";
+                 dbgs() << "- "; Mul0->dump();
+                 dbgs() << "- "; Mul1->dump());
+
+      const ValueList &Mul0_LHS = PMul0->LHS;
+      const ValueList &Mul0_RHS = PMul0->RHS;
+      const ValueList &Mul1_LHS = PMul1->LHS;
+      const ValueList &Mul1_RHS = PMul1->RHS;
+
+      if (!AreSymmetrical(Mul0_LHS, Mul1_LHS) ||
+          !AreSymmetrical(Mul0_RHS, Mul1_RHS))
+        continue;
+
+      LLVM_DEBUG(dbgs() << "OK: mul operands list match:\n");
+      // The first elements of each vector should be loads with sexts. If we
+      // find that its two pairs of consecutive loads, then these can be
+      // transformed into two wider loads and the users can be replaced with
+      // DSP intrinsics.
+      bool Found = false;
+      for (unsigned x = 0; x < Mul0_LHS.size(); x += 2) {
+        auto *Ld0 = dyn_cast<LoadInst>(Mul0_LHS[x]);
+        auto *Ld1 = dyn_cast<LoadInst>(Mul1_LHS[x]);
+        auto *Ld2 = dyn_cast<LoadInst>(Mul0_RHS[x]);
+        auto *Ld3 = dyn_cast<LoadInst>(Mul1_RHS[x]);
+
+        if (!Ld0 || !Ld1 || !Ld2 || !Ld3)
+          continue;
+
+        LLVM_DEBUG(dbgs() << "Looking at operands " << x << ":\n"
+                   << "\t Ld0: " << *Ld0 << "\n"
+                   << "\t Ld1: " << *Ld1 << "\n"
+                   << "and operands " << x + 2 << ":\n"
+                   << "\t Ld2: " << *Ld2 << "\n"
+                   << "\t Ld3: " << *Ld3 << "\n");
+
+        if (AreSequentialLoads(Ld0, Ld1, PMul0->VecLd)) {
+          if (AreSequentialLoads(Ld2, Ld3, PMul1->VecLd)) {
+            LLVM_DEBUG(dbgs() << "OK: found two pairs of parallel loads!\n");
+            PMACPairs.push_back(std::make_pair(PMul0, PMul1));
+            Found = true;
+          } else if (AreSequentialLoads(Ld3, Ld2, PMul1->VecLd)) {
+            LLVM_DEBUG(dbgs() << "OK: found two pairs of parallel loads!\n");
+            LLVM_DEBUG(dbgs() << "    exchanging Ld2 and Ld3\n");
+            PMul1->Exchange = true;
+            PMACPairs.push_back(std::make_pair(PMul0, PMul1));
+            Found = true;
+          }
+        } else if (AreSequentialLoads(Ld1, Ld0, PMul0->VecLd)) {
+          if (AreSequentialLoads(Ld2, Ld3, PMul1->VecLd)) {
+            LLVM_DEBUG(dbgs() << "OK: found two pairs of parallel loads!\n");
+            LLVM_DEBUG(dbgs() << "    exchanging Ld0 and Ld1\n");
+            LLVM_DEBUG(dbgs() << "    and swapping muls\n");
+            PMul0->Exchange = true;
+            // Only the second operand can be exchanged, so swap the muls.
+            PMACPairs.push_back(std::make_pair(PMul1, PMul0));
+            Found = true;
+          }
+        }
+      }
+      if (Found) {
+        Paired.insert(Mul0);
+        Paired.insert(Mul1);
+        break;
       }
     }
   }
@@ -394,12 +435,15 @@
   Instruction *InsertAfter = Reduction.AccIntAdd;
 
   for (auto &Pair : PMACPairs) {
+    BinOpChain *PMul0 = Pair.first;
+    BinOpChain *PMul1 = Pair.second;
     LLVM_DEBUG(dbgs() << "Found parallel MACs!!\n";
-               dbgs() << "- "; Pair.first->Root->dump();
-               dbgs() << "- "; Pair.second->Root->dump());
-    auto *VecLd0 = cast<LoadInst>(Pair.first->VecLd[0]);
-    auto *VecLd1 = cast<LoadInst>(Pair.second->VecLd[0]);
-    Acc = CreateSMLADCall(VecLd0, VecLd1, Acc, InsertAfter);
+               dbgs() << "- "; PMul0->Root->dump();
+               dbgs() << "- "; PMul1->Root->dump());
+
+    auto *VecLd0 = cast<LoadInst>(PMul0->VecLd[0]);
+    auto *VecLd1 = cast<LoadInst>(PMul1->VecLd[0]);
+    Acc = CreateSMLADCall(VecLd0, VecLd1, Acc, PMul1->Exchange, InsertAfter);
     InsertAfter = Acc;
   }
 
@@ -453,7 +497,6 @@
 }
 
 static void AddMACCandidate(OpChainList &Candidates,
-                            const Instruction *Acc,
                             Instruction *Mul,
                             Value *MulOp0, Value *MulOp1) {
   LLVM_DEBUG(dbgs() << "OK, found acc mul:\t"; Mul->dump());
@@ -470,55 +513,44 @@
 
 static void MatchParallelMACSequences(Reduction &R,
                                       OpChainList &Candidates) {
-  const Instruction *Acc = R.AccIntAdd;
-  Value *A, *MulOp0, *MulOp1;
-  LLVM_DEBUG(dbgs() << "\n- Analysing:\t"; Acc->dump());
-
-  // Pattern 1: the accumulator is the RHS of the mul.
-  while(match(Acc, m_Add(m_Mul(m_Value(MulOp0), m_Value(MulOp1)),
-                         m_Value(A)))){
-    Instruction *Mul = cast<Instruction>(Acc->getOperand(0));
-    AddMACCandidate(Candidates, Acc, Mul, MulOp0, MulOp1);
-    Acc = dyn_cast<Instruction>(A);
-  }
-  // Pattern 2: the accumulator is the LHS of the mul.
-  while(match(Acc, m_Add(m_Value(A),
-                         m_Mul(m_Value(MulOp0), m_Value(MulOp1))))) {
-    Instruction *Mul = cast<Instruction>(Acc->getOperand(1));
-    AddMACCandidate(Candidates, Acc, Mul, MulOp0, MulOp1);
-    Acc = dyn_cast<Instruction>(A);
-  }
-
-  // The last mul in the chain has a slightly different pattern:
-  // the mul is the first operand
-  if (match(Acc, m_Add(m_Mul(m_Value(MulOp0), m_Value(MulOp1)), m_Value(A))))
-    AddMACCandidate(Candidates, Acc, cast<Instruction>(Acc->getOperand(0)),
-                    MulOp0, MulOp1);
-
-  // Same as above, but SMLALD may perform 32-bit muls, sext the results and
-  // then accumulate.
-  while(match(Acc, m_Add(m_SExt(m_Mul(m_Value(MulOp0), m_Value(MulOp1))),
-                        m_Value(A)))) {
-    Value *Mul = cast<Instruction>(Acc->getOperand(0))->getOperand(0);
-    AddMACCandidate(Candidates, Acc, cast<Instruction>(Mul), MulOp0, MulOp1);
-    Acc = dyn_cast<Instruction>(A);
-  }
-  while(match(Acc, m_Add(m_Value(A),
-                         m_SExt(m_Mul(m_Value(MulOp0), m_Value(MulOp1)))))) {
-    Value *Mul = cast<Instruction>(Acc->getOperand(1))->getOperand(0);
-    AddMACCandidate(Candidates, Acc, cast<Instruction>(Mul), MulOp0, MulOp1);
-    Acc = dyn_cast<Instruction>(A);
-  }
-  if (match(Acc, m_Add(m_SExt(m_Mul(m_Value(MulOp0), m_Value(MulOp1))),
-                       m_Value(A)))) {
-    Value *Mul = cast<Instruction>(
-      cast<Instruction>(Acc)->getOperand(0))->getOperand(0);
-    AddMACCandidate(Candidates, Acc, cast<Instruction>(Mul), MulOp0, MulOp1);
-  }
-
-  // Because we start at the bottom of the chain, and we work our way up,
-  // the muls are added in reverse program order to the list.
-  std::reverse(Candidates.begin(), Candidates.end());
+  Instruction *Acc = R.AccIntAdd;
+  LLVM_DEBUG(dbgs() << "\n- Analysing:\t" << *Acc);
+
+  // Returns false to signal the search should be stopped.
+  std::function<bool(Value*)> Match =
+    [&Candidates, &Match](Value *V) -> bool {
+
+    auto *I = dyn_cast<Instruction>(V);
+    if (!V)
+      return false;
+
+    Value *MulOp0, *MulOp1;
+
+    switch (I->getOpcode()) {
+    case Instruction::Add:
+      if (Match(I->getOperand(0)) || (Match(I->getOperand(1))))
+        return true;
+      break;
+    case Instruction::Mul:
+      if (match (I, (m_Mul(m_Value(MulOp0), m_Value(MulOp1))))) {
+        AddMACCandidate(Candidates, I, MulOp0, MulOp1);
+        return false;
+      }
+      break;
+    case Instruction::SExt:
+      if (match (I, (m_SExt(m_Mul(m_Value(MulOp0), m_Value(MulOp1)))))) {
+        Instruction *Mul = cast<Instruction>(I->getOperand(0));
+        AddMACCandidate(Candidates, Mul, MulOp0, MulOp1);
+        return false;
+      }
+      break;
+    }
+    return false;
+  };
+
+  while (Match (Acc));
+  LLVM_DEBUG(dbgs() << "Finished matching MAC sequences, found "
+             << Candidates.size() << " candidates.\n");
 }
 
 // Collects all instructions that are not part of the MAC chains, which is the
@@ -661,34 +693,41 @@
   return Changed;
 }
 
-static void CreateLoadIns(IRBuilder<NoFolder> &IRB, LoadInst **VecLd,
-                          const Type *LoadTy) {
-  const unsigned AddrSpace = (*VecLd)->getPointerAddressSpace();
+static LoadInst *CreateLoadIns(IRBuilder<NoFolder> &IRB, LoadInst &BaseLoad,
+                               const Type *LoadTy) {
+  const unsigned AddrSpace = BaseLoad.getPointerAddressSpace();
 
-  Value *VecPtr = IRB.CreateBitCast((*VecLd)->getPointerOperand(),
+  Value *VecPtr = IRB.CreateBitCast(BaseLoad.getPointerOperand(),
                                     LoadTy->getPointerTo(AddrSpace));
-  *VecLd = IRB.CreateAlignedLoad(VecPtr, (*VecLd)->getAlignment());
+  return IRB.CreateAlignedLoad(VecPtr, BaseLoad.getAlignment());
 }
 
 Instruction *ARMParallelDSP::CreateSMLADCall(LoadInst *VecLd0, LoadInst *VecLd1,
-                                             Instruction *Acc,
+                                             Instruction *Acc, bool Exchange,
                                              Instruction *InsertAfter) {
-  LLVM_DEBUG(dbgs() << "Create SMLAD intrinsic using:\n";
-             dbgs() << "- "; VecLd0->dump();
-             dbgs() << "- "; VecLd1->dump();
-             dbgs() << "- "; Acc->dump());
+  LLVM_DEBUG(dbgs() << "Create SMLAD intrinsic using:\n"
+             << "- " << *VecLd0 << "\n"
+             << "- " << *VecLd1 << "\n"
+             << "- " << *Acc << "\n"
+             << "Exchange: " << Exchange << "\n");
 
   IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
                               ++BasicBlock::iterator(InsertAfter));
 
   // Replace the reduction chain with an intrinsic call
   const Type *Ty = IntegerType::get(M->getContext(), 32);
-  CreateLoadIns(Builder, &VecLd0, Ty);
-  CreateLoadIns(Builder, &VecLd1, Ty);
-  Value* Args[] = { VecLd0, VecLd1, Acc };
-  Function *SMLAD = Acc->getType()->isIntegerTy(32) ?
-    Intrinsic::getDeclaration(M, Intrinsic::arm_smlad) :
-    Intrinsic::getDeclaration(M, Intrinsic::arm_smlald);
+  LoadInst *NewLd0 = CreateLoadIns(Builder, VecLd0[0], Ty);
+  LoadInst *NewLd1 = CreateLoadIns(Builder, VecLd1[0], Ty);
+  Value* Args[] = { NewLd0, NewLd1, Acc };
+  Function *SMLAD = nullptr;
+  if (Exchange)
+    SMLAD = Acc->getType()->isIntegerTy(32) ?
+      Intrinsic::getDeclaration(M, Intrinsic::arm_smladx) :
+      Intrinsic::getDeclaration(M, Intrinsic::arm_smlaldx);
+  else
+    SMLAD = Acc->getType()->isIntegerTy(32) ?
+      Intrinsic::getDeclaration(M, Intrinsic::arm_smlad) :
+      Intrinsic::getDeclaration(M, Intrinsic::arm_smlald);
   CallInst *Call = Builder.CreateCall(SMLAD, Args);
   NumSMLAD++;
   return Call;
Index: llvm/trunk/test/CodeGen/ARM/smladx-1.ll
===================================================================
--- llvm/trunk/test/CodeGen/ARM/smladx-1.ll
+++ llvm/trunk/test/CodeGen/ARM/smladx-1.ll
@@ -0,0 +1,240 @@
+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define i32 @smladx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smladx
+; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
+; CHECK: [[ACC0:%[^ ]+]] = phi i32 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
+; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
+; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
+; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN23]], i32 [[IN12]], i32 [[ACC0]])
+; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
+; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
+; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
+; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN21]], i32 [[IN10]], i32 [[ACC1]])
+; CHECK-NOT: call i32 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i32 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i32 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sum.010.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sub.epil = sub i32 %j, %i.011.epil
+  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
+  %2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nsw i32 %mul.epil, %sum.010.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i32 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
+  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
+  %In2 = load i16, i16* %pIn2.0, align 2
+  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
+  %In1 = load i16, i16* %pIn1.0, align 2
+  %inc = or i32 %i.011, 1
+  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
+  %In2.1 = load i16, i16* %pIn2.1, align 2
+  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
+  %In1.1 = load i16, i16* %pIn1.1, align 2
+  %inc.1 = or i32 %i.011, 2
+  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
+  %In2.2 = load i16, i16* %pIn2.2, align 2
+  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
+  %In1.2 = load i16, i16* %pIn1.2, align 2
+  %inc.2 = or i32 %i.011, 3
+  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
+  %In2.3 = load i16, i16* %pIn2.3, align 2
+  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
+  %In1.3 = load i16, i16* %pIn1.3, align 2
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %mul = mul nsw i32 %sextIn1, %sextIn2
+  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
+  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
+  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
+  %add = add nsw i32 %mul, %sum.010
+  %add.1 = add nsw i32 %mul.1, %add
+  %add.2 = add nsw i32 %mul.2, %add.1
+  %add.3 = add nsw i32 %mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define i32 @smladx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smladx_swap
+; CHECK: for.body.preheader.new:
+; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
+; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
+
+; CHECK: for.body:
+; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
+; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
+; CHECK: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[ACC0:%[^ ]+]] = phi i32 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+
+; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
+; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
+
+
+; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
+; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
+; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
+; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN2_2]], i32 [[IN1]], i32 [[ACC0]])
+
+; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
+; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
+; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
+; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN2]], i32 [[IN1_2]], i32 [[ACC1]])
+
+; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
+; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
+
+; CHECK-NOT: call i32 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
+  %2 = add i32 %j, -1
+  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i32 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i32 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
+
+for.body.epil.preheader:
+  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
+  %3 = sub i32 %j, %i.011.unr
+  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
+  %4 = sub i32 0, %xtraiter
+  br label %for.body.epil
+
+for.body.epil:
+  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
+  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
+  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
+  %sum.010.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
+  %5 = load i16, i16* %lsr.iv3, align 2
+  %conv.epil = sext i16 %5 to i32
+  %6 = load i16, i16* %lsr.iv, align 2
+  %conv2.epil = sext i16 %6 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nsw i32 %mul.epil, %sum.010.epil
+  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
+  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
+  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
+  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %pin2 = phi i16* [ %pin2_sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
+  %pin1 = phi i16* [ %pin1_add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i32 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %pin2_add1 = getelementptr i16, i16* %pin2, i32 1
+  %In2 = load i16, i16* %pin2_add1, align 2
+  %pin1_sub2 = getelementptr i16, i16* %pin1, i32 -2
+  %In1 = load i16, i16* %pin1_sub2, align 2
+  %In2.1 = load i16, i16* %pin2, align 2
+  %pin1_sub1 = getelementptr i16, i16* %pin1, i32 -1
+  %In1.1 = load i16, i16* %pin1_sub1, align 2
+  %pin2_sub1 = getelementptr i16, i16* %pin2, i32 -1
+  %In2.2 = load i16, i16* %pin2_sub1, align 2
+  %In1.2 = load i16, i16* %pin1, align 2
+  %pin2_sub2 = getelementptr i16, i16* %pin2, i32 -2
+  %In2.3 = load i16, i16* %pin2_sub2, align 2
+  %pin1_add1 = getelementptr i16, i16* %pin1, i32 1
+  %In1.3 = load i16, i16* %pin1_add1, align 2
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %mul = mul nsw i32 %sextIn2, %sextIn1
+  %add = add nsw i32 %mul, %sum.010
+  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
+  %add.1 = add nsw i32 %mul.1, %add
+  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
+  %add.2 = add nsw i32 %mul.2, %add.1
+  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
+  %add.3 = add nsw i32 %mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pin1_add4 = getelementptr i16, i16* %pin1, i32 4
+  %pin2_sub4 = getelementptr i16, i16* %pin2, i32 -4
+  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
Index: llvm/trunk/test/CodeGen/ARM/smlaldx-1.ll
===================================================================
--- llvm/trunk/test/CodeGen/ARM/smlaldx-1.ll
+++ llvm/trunk/test/CodeGen/ARM/smlaldx-1.ll
@@ -0,0 +1,249 @@
+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smlaldx
+; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
+; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
+; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
+; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
+; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
+; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
+; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
+; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sub.epil = sub i32 %j, %i.011.epil
+  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
+  %2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+for.body:
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
+  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
+  %In2 = load i16, i16* %pIn2.0, align 2
+  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
+  %In1 = load i16, i16* %pIn1.0, align 2
+  %inc = or i32 %i.011, 1
+  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
+  %In2.1 = load i16, i16* %pIn2.1, align 2
+  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
+  %In1.1 = load i16, i16* %pIn1.1, align 2
+  %inc.1 = or i32 %i.011, 2
+  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
+  %In2.2 = load i16, i16* %pIn2.2, align 2
+  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
+  %In1.2 = load i16, i16* %pIn1.2, align 2
+  %inc.2 = or i32 %i.011, 3
+  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
+  %In2.3 = load i16, i16* %pIn2.3, align 2
+  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
+  %In1.3 = load i16, i16* %pIn1.3, align 2
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %mul = mul nsw i32 %sextIn1, %sextIn2
+  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
+  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
+  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
+  %sext.mul = sext i32 %mul to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add = add nsw i64 %sext.mul, %sum.010
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %add.2 = add nsw i64 %sext.mul.2, %add.1
+  %add.3 = add nsw i64 %sext.mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define i64 @smlaldx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
+  %2 = add i32 %j, -1
+  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
+
+for.body.epil.preheader:
+  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
+  %3 = sub i32 %j, %i.011.unr
+  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
+  %4 = sub i32 0, %xtraiter
+  br label %for.body.epil
+
+for.body.epil:
+  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
+  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
+  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
+  %5 = load i16, i16* %lsr.iv3, align 2
+  %conv.epil = sext i16 %5 to i32
+  %6 = load i16, i16* %lsr.iv, align 2
+  %conv2.epil = sext i16 %6 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
+  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
+  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
+  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+; CHECK-LABEL: smlaldx_swap
+; CHECK: for.body.preheader.new:
+; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
+; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
+
+; CHECK: for.body:
+; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
+; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
+; CHECK: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+
+; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
+; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
+
+; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
+; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
+; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
+; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2_2]], i32 [[IN1]], i64 [[ACC0]])
+
+; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
+; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
+; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
+; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC1]])
+
+; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
+; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
+
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+for.body:
+  %pin2 = phi i16* [ %pin2.sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
+  %pin1 = phi i16* [ %pin1.add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %pin2.add1 = getelementptr i16, i16* %pin2, i32 1
+  %In2 = load i16, i16* %pin2.add1, align 2
+  %pin1.sub2 = getelementptr i16, i16* %pin1, i32 -2
+  %In1 = load i16, i16* %pin1.sub2, align 2
+  %In2.1 = load i16, i16* %pin2, align 2
+  %pin1.sub1 = getelementptr i16, i16* %pin1, i32 -1
+  %In1.1 = load i16, i16* %pin1.sub1, align 2
+  %pin2.sub1 = getelementptr i16, i16* %pin2, i32 -1
+  %In2.2 = load i16, i16* %pin2.sub1, align 2
+  %In1.2 = load i16, i16* %pin1, align 2
+  %pin2.sub2 = getelementptr i16, i16* %pin2, i32 -2
+  %In2.3 = load i16, i16* %pin2.sub2, align 2
+  %pin1.add1 = getelementptr i16, i16* %pin1, i32 1
+  %In1.3 = load i16, i16* %pin1.add1, align 2
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %mul = mul nsw i32 %sextIn2, %sextIn1
+  %sext.mul = sext i32 %mul to i64
+  %add = add nsw i64 %sext.mul, %sum.010
+  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %add.2 = add nsw i64 %sext.mul.2, %add.1
+  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add.3 = add nsw i64 %sext.mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pin1.add4 = getelementptr i16, i16* %pin1, i32 4
+  %pin2.sub4 = getelementptr i16, i16* %pin2, i32 -4
+  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
Index: llvm/trunk/test/CodeGen/ARM/smlaldx-2.ll
===================================================================
--- llvm/trunk/test/CodeGen/ARM/smlaldx-2.ll
+++ llvm/trunk/test/CodeGen/ARM/smlaldx-2.ll
@@ -0,0 +1,248 @@
+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smlaldx
+; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
+; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
+; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
+; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
+; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
+; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
+; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
+; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sub.epil = sub i32 %j, %i.011.epil
+  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
+  %2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+for.body:
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
+  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
+  %In2 = load i16, i16* %pIn2.0, align 2
+  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
+  %In1 = load i16, i16* %pIn1.0, align 2
+  %inc = or i32 %i.011, 1
+  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
+  %In2.1 = load i16, i16* %pIn2.1, align 2
+  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
+  %In1.1 = load i16, i16* %pIn1.1, align 2
+  %inc.1 = or i32 %i.011, 2
+  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
+  %In2.2 = load i16, i16* %pIn2.2, align 2
+  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
+  %In1.2 = load i16, i16* %pIn1.2, align 2
+  %inc.2 = or i32 %i.011, 3
+  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
+  %In2.3 = load i16, i16* %pIn2.3, align 2
+  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
+  %In1.3 = load i16, i16* %pIn1.3, align 2
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %mul = mul nsw i32 %sextIn1, %sextIn2
+  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
+  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
+  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
+  %sext.mul = sext i32 %mul to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add = add nsw i64 %sum.010, %sext.mul
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %add.2 = add nsw i64 %add.1, %sext.mul.2
+  %add.3 = add nsw i64 %sext.mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define i64 @smlaldx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
+  %2 = add i32 %j, -1
+  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
+
+for.body.epil.preheader:
+  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
+  %3 = sub i32 %j, %i.011.unr
+  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
+  %4 = sub i32 0, %xtraiter
+  br label %for.body.epil
+
+for.body.epil:
+  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
+  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
+  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
+  %5 = load i16, i16* %lsr.iv3, align 2
+  %conv.epil = sext i16 %5 to i32
+  %6 = load i16, i16* %lsr.iv, align 2
+  %conv2.epil = sext i16 %6 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
+  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
+  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
+  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+; CHECK-LABEL: smlaldx_swap
+; CHECK: for.body.preheader.new:
+; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
+; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
+
+; CHECK: for.body:
+; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
+; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
+; CHECK: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
+; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
+
+; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
+; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
+; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
+; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC0]])
+
+; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
+; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
+; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
+; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN1]], i32 [[IN2_2]], i64 [[ACC1]])
+
+; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
+; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
+
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+for.body:
+  %pin2 = phi i16* [ %pin2.sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
+  %pin1 = phi i16* [ %pin1.add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %pin2.add1 = getelementptr i16, i16* %pin2, i32 1
+  %In2 = load i16, i16* %pin2.add1, align 2
+  %pin1.sub2 = getelementptr i16, i16* %pin1, i32 -2
+  %In1 = load i16, i16* %pin1.sub2, align 2
+  %In2.1 = load i16, i16* %pin2, align 2
+  %pin1.sub1 = getelementptr i16, i16* %pin1, i32 -1
+  %In1.1 = load i16, i16* %pin1.sub1, align 2
+  %pin2.sub1 = getelementptr i16, i16* %pin2, i32 -1
+  %In2.2 = load i16, i16* %pin2.sub1, align 2
+  %In1.2 = load i16, i16* %pin1, align 2
+  %pin2.sub2 = getelementptr i16, i16* %pin2, i32 -2
+  %In2.3 = load i16, i16* %pin2.sub2, align 2
+  %pin1.add1 = getelementptr i16, i16* %pin1, i32 1
+  %In1.3 = load i16, i16* %pin1.add1, align 2
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %mul = mul nsw i32 %sextIn2, %sextIn1
+  %sext.mul = sext i32 %mul to i64
+  %add = add nsw i64 %sext.mul, %sum.010
+  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %add.2 = add nsw i64 %add.1, %sext.mul.2
+  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add.3 = add nsw i64 %add.2, %sext.mul.3
+  %inc.3 = add i32 %i.011, 4
+  %pin1.add4 = getelementptr i16, i16* %pin1, i32 4
+  %pin2.sub4 = getelementptr i16, i16* %pin2, i32 -4
+  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}