[ARM] Exchange MAC operands in ARMParallelDSP

[samparker]

Summary

SMLAD and SMLALD instructions also come in the form of SMLADX and SMLALDX which perform an exchange on their second operand. To support this, more of the loads in the MAC candidates are compared for sequential access and a boolean value has been added to BinOpChain.

AddMACCandiate has been refactored into a small pattern matching state machine to reduce the amount of duplicated code, but also to enable the matching to be more flexible. CreateParallelMACPairs now iterates through all the candidates to find parallel ones.

[SjoerdMeijer]

That's a cool new trick!

llvm/trunk/lib/Target/ARM/ARMParallelDSP.cpp

namespace {
 
   // 'BinOpChain' and 'Reduction' are just some bookkeeping data structures.
   // 'Reduction' contains the phi-node and accumulator statement from where we
   // start pattern matching, and 'BinOpChain' the multiplication
   // instructions that are candidates for parallel execution.
   struct BinOpChain : public OpChain {
     ValueList     LHS;      // List of all (narrow) left hand operands.
     ValueList     RHS;      // List of all (narrow) right hand operands.
+    bool Exchange = false;
 
     BinOpChain(Instruction *I, ValueList &lhs, ValueList &rhs) :
       OpChain(I, lhs), LHS(lhs), RHS(rhs) {
         for (auto *V : RHS)
           AllValues.push_back(V);
       }
   };
 
class ARMParallelDSP : public LoopPass {
     Loop              *L;
     const DataLayout  *DL;
     Module            *M;
 
     bool InsertParallelMACs(Reduction &Reduction, PMACPairList &PMACPairs);
     bool AreSequentialLoads(LoadInst *Ld0, LoadInst *Ld1, MemInstList &VecMem);
     PMACPairList CreateParallelMACPairs(OpChainList &Candidates);
     Instruction *CreateSMLADCall(LoadInst *VecLd0, LoadInst *VecLd1,
-                                 Instruction *Acc, Instruction *InsertAfter);
+                                 Instruction *Acc, bool Exchange,
+                                 Instruction *InsertAfter);
 
     /// Try to match and generate: SMLAD, SMLADX - Signed Multiply Accumulate
     /// Dual performs two signed 16x16-bit multiplications. It adds the
     /// products to a 32-bit accumulate operand. Optionally, the instruction can
     /// exchange the halfwords of the second operand before performing the
     /// arithmetic.
     bool MatchSMLAD(Function &F);
 
bool runOnLoop(Loop *TheLoop, LPPassManager &) override {
         LLVM_DEBUG(dbgs() << "DSP extension not enabled: not running pass "
                              "ARMParallelDSP\n");
         return false;
       }
 
       LoopAccessInfo LAI(L, SE, TLI, AA, DT, LI);
       bool Changes = false;
 
-      LLVM_DEBUG(dbgs() << "\n== Parallel DSP pass ==\n\n");
+      LLVM_DEBUG(dbgs() << "\n== Parallel DSP pass ==\n");
+      LLVM_DEBUG(dbgs() << " - " << F.getName() << "\n\n");
       Changes = MatchSMLAD(F);
       return Changes;
     }
   };
 }
 
 // MaxBitwidth: the maximum supported bitwidth of the elements in the DSP
 // instructions, which is set to 16. So here we should collect all i8 and i16

 static bool AreSequentialAccesses(MemInst *MemOp0, MemInst *MemOp1,
                                   MemInstList &VecMem, const DataLayout &DL,
                                   ScalarEvolution &SE) {
   if (!MemOp0->isSimple() || !MemOp1->isSimple()) {
     LLVM_DEBUG(dbgs() << "No, not touching volatile access\n");
     return false;
   }
   if (isConsecutiveAccess(MemOp0, MemOp1, DL, SE)) {
+    VecMem.clear();
     VecMem.push_back(MemOp0);
     VecMem.push_back(MemOp1);
     LLVM_DEBUG(dbgs() << "OK: accesses are consecutive.\n");
     return true;
   }
   LLVM_DEBUG(dbgs() << "No, accesses aren't consecutive.\n");
   return false;
 }

 PMACPairList
 ARMParallelDSP::CreateParallelMACPairs(OpChainList &Candidates) {
   const unsigned Elems = Candidates.size();
   PMACPairList PMACPairs;
 
   if (Elems < 2)
     return PMACPairs;
 
-  // TODO: for now we simply try to match consecutive pairs i and i+1.
-  // We can compare all elements, but then we need to compare and evaluate
-  // different solutions.
-  for(unsigned i=0; i<Elems-1; i+=2) {
+  SmallPtrSet<const Instruction*, 4> Paired;
+  for (unsigned i = 0; i < Elems; ++i) {
     BinOpChain *PMul0 = static_cast<BinOpChain*>(Candidates[i].get());
-    BinOpChain *PMul1 = static_cast<BinOpChain*>(Candidates[i+1].get());
+    if (Paired.count(PMul0->Root))
+      continue;
+
+    for (unsigned j = 0; j < Elems; ++j) {
+      if (i == j)
+        continue;
+
+      BinOpChain *PMul1 = static_cast<BinOpChain*>(Candidates[j].get());
+      if (Paired.count(PMul1->Root))
+        continue;
+
       const Instruction *Mul0 = PMul0->Root;
       const Instruction *Mul1 = PMul1->Root;
-
       if (Mul0 == Mul1)
         continue;
 
+      assert(PMul0 != PMul1 && "expected different chains");
+
       LLVM_DEBUG(dbgs() << "\nCheck parallel muls:\n";
                  dbgs() << "- "; Mul0->dump();
                  dbgs() << "- "; Mul1->dump());
 
       const ValueList &Mul0_LHS = PMul0->LHS;
       const ValueList &Mul0_RHS = PMul0->RHS;
       const ValueList &Mul1_LHS = PMul1->LHS;
       const ValueList &Mul1_RHS = PMul1->RHS;
 
       if (!AreSymmetrical(Mul0_LHS, Mul1_LHS) ||
           !AreSymmetrical(Mul0_RHS, Mul1_RHS))
         continue;
 
       LLVM_DEBUG(dbgs() << "OK: mul operands list match:\n");
-    // The first elements of each vector should be loads with sexts. If we find
-    // that its two pairs of consecutive loads, then these can be transformed
-    // into two wider loads and the users can be replaced with DSP
-    // intrinsics.
+      // The first elements of each vector should be loads with sexts. If we
+      // find that its two pairs of consecutive loads, then these can be
+      // transformed into two wider loads and the users can be replaced with
+      // DSP intrinsics.
+      bool Found = false;
       for (unsigned x = 0; x < Mul0_LHS.size(); x += 2) {
         auto *Ld0 = dyn_cast<LoadInst>(Mul0_LHS[x]);
         auto *Ld1 = dyn_cast<LoadInst>(Mul1_LHS[x]);
         auto *Ld2 = dyn_cast<LoadInst>(Mul0_RHS[x]);
         auto *Ld3 = dyn_cast<LoadInst>(Mul1_RHS[x]);
 
-      LLVM_DEBUG(dbgs() << "Looking at operands " << x << ":\n";
-                 dbgs() << "\t mul1: "; Mul0_LHS[x]->dump();
-                 dbgs() << "\t mul2: "; Mul1_LHS[x]->dump();
-                 dbgs() << "and operands " << x + 2 << ":\n";
-                 dbgs() << "\t mul1: "; Mul0_RHS[x]->dump();
-                 dbgs() << "\t mul2: "; Mul1_RHS[x]->dump());
+        if (!Ld0 || !Ld1 || !Ld2 || !Ld3)
+          continue;
 
-      if (AreSequentialLoads(Ld0, Ld1, PMul0->VecLd) &&
-          AreSequentialLoads(Ld2, Ld3, PMul1->VecLd)) {
+        LLVM_DEBUG(dbgs() << "Looking at operands " << x << ":\n"
+                   << "\t Ld0: " << *Ld0 << "\n"
+                   << "\t Ld1: " << *Ld1 << "\n"
+                   << "and operands " << x + 2 << ":\n"
+                   << "\t Ld2: " << *Ld2 << "\n"
+                   << "\t Ld3: " << *Ld3 << "\n");
+
+        if (AreSequentialLoads(Ld0, Ld1, PMul0->VecLd)) {
+          if (AreSequentialLoads(Ld2, Ld3, PMul1->VecLd)) {
+            LLVM_DEBUG(dbgs() << "OK: found two pairs of parallel loads!\n");
+            PMACPairs.push_back(std::make_pair(PMul0, PMul1));
+            Found = true;
+          } else if (AreSequentialLoads(Ld3, Ld2, PMul1->VecLd)) {
             LLVM_DEBUG(dbgs() << "OK: found two pairs of parallel loads!\n");
+            LLVM_DEBUG(dbgs() << "    exchanging Ld2 and Ld3\n");
+            PMul1->Exchange = true;
             PMACPairs.push_back(std::make_pair(PMul0, PMul1));
+            Found = true;
+          }
+        } else if (AreSequentialLoads(Ld1, Ld0, PMul0->VecLd)) {
+          if (AreSequentialLoads(Ld2, Ld3, PMul1->VecLd)) {
+            LLVM_DEBUG(dbgs() << "OK: found two pairs of parallel loads!\n");
+            LLVM_DEBUG(dbgs() << "    exchanging Ld0 and Ld1\n");
+            LLVM_DEBUG(dbgs() << "    and swapping muls\n");
+            PMul0->Exchange = true;
+            // Only the second operand can be exchanged, so swap the muls.
+            PMACPairs.push_back(std::make_pair(PMul1, PMul0));
+            Found = true;
+          }
+        }
+      }
+      if (Found) {
+        Paired.insert(Mul0);
+        Paired.insert(Mul1);
+        break;
       }
     }
   }
   return PMACPairs;
 }
 
 bool ARMParallelDSP::InsertParallelMACs(Reduction &Reduction,
                                         PMACPairList &PMACPairs) {
   Instruction *Acc = Reduction.Phi;
   Instruction *InsertAfter = Reduction.AccIntAdd;
 
   for (auto &Pair : PMACPairs) {
+    BinOpChain *PMul0 = Pair.first;
+    BinOpChain *PMul1 = Pair.second;
     LLVM_DEBUG(dbgs() << "Found parallel MACs!!\n";
-               dbgs() << "- "; Pair.first->Root->dump();
-               dbgs() << "- "; Pair.second->Root->dump());
-    auto *VecLd0 = cast<LoadInst>(Pair.first->VecLd[0]);
-    auto *VecLd1 = cast<LoadInst>(Pair.second->VecLd[0]);
-    Acc = CreateSMLADCall(VecLd0, VecLd1, Acc, InsertAfter);
+               dbgs() << "- "; PMul0->Root->dump();
+               dbgs() << "- "; PMul1->Root->dump());
+
+    auto *VecLd0 = cast<LoadInst>(PMul0->VecLd[0]);
+    auto *VecLd1 = cast<LoadInst>(PMul1->VecLd[0]);
+    Acc = CreateSMLADCall(VecLd0, VecLd1, Acc, PMul1->Exchange, InsertAfter);
     InsertAfter = Acc;
   }
 
   if (Acc != Reduction.Phi) {
     LLVM_DEBUG(dbgs() << "Replace Accumulate: "; Acc->dump());
     Reduction.AccIntAdd->replaceAllUsesWith(Acc);
     return true;
   }
LLVM_DEBUG(
     for (auto &R : Reductions) {
       dbgs() << "-  "; R.Phi->dump();
       dbgs() << "-> "; R.AccIntAdd->dump();
     }
   );
 }
 
 static void AddMACCandidate(OpChainList &Candidates,
-                            const Instruction *Acc,
                             Instruction *Mul,
                             Value *MulOp0, Value *MulOp1) {
   LLVM_DEBUG(dbgs() << "OK, found acc mul:\t"; Mul->dump());
   assert(Mul->getOpcode() == Instruction::Mul &&
          "expected mul instruction");
   ValueList LHS;
   ValueList RHS;
   if (IsNarrowSequence<16>(MulOp0, LHS) &&
       IsNarrowSequence<16>(MulOp1, RHS)) {
     LLVM_DEBUG(dbgs() << "OK, found narrow mul: "; Mul->dump());
     Candidates.push_back(make_unique<BinOpChain>(Mul, LHS, RHS));
   }
 }
 
 static void MatchParallelMACSequences(Reduction &R,
                                       OpChainList &Candidates) {
-  const Instruction *Acc = R.AccIntAdd;
-  Value *A, *MulOp0, *MulOp1;
-  LLVM_DEBUG(dbgs() << "\n- Analysing:\t"; Acc->dump());
-
-  // Pattern 1: the accumulator is the RHS of the mul.
-  while(match(Acc, m_Add(m_Mul(m_Value(MulOp0), m_Value(MulOp1)),
-                         m_Value(A)))){
-    Instruction *Mul = cast<Instruction>(Acc->getOperand(0));
-    AddMACCandidate(Candidates, Acc, Mul, MulOp0, MulOp1);
-    Acc = dyn_cast<Instruction>(A);
-  }
-  // Pattern 2: the accumulator is the LHS of the mul.
-  while(match(Acc, m_Add(m_Value(A),
-                         m_Mul(m_Value(MulOp0), m_Value(MulOp1))))) {
-    Instruction *Mul = cast<Instruction>(Acc->getOperand(1));
-    AddMACCandidate(Candidates, Acc, Mul, MulOp0, MulOp1);
-    Acc = dyn_cast<Instruction>(A);
-  }
-
-  // The last mul in the chain has a slightly different pattern:
-  // the mul is the first operand
-  if (match(Acc, m_Add(m_Mul(m_Value(MulOp0), m_Value(MulOp1)), m_Value(A))))
-    AddMACCandidate(Candidates, Acc, cast<Instruction>(Acc->getOperand(0)),
-                    MulOp0, MulOp1);
-
-  // Same as above, but SMLALD may perform 32-bit muls, sext the results and
-  // then accumulate.
-  while(match(Acc, m_Add(m_SExt(m_Mul(m_Value(MulOp0), m_Value(MulOp1))),
-                        m_Value(A)))) {
-    Value *Mul = cast<Instruction>(Acc->getOperand(0))->getOperand(0);
-    AddMACCandidate(Candidates, Acc, cast<Instruction>(Mul), MulOp0, MulOp1);
-    Acc = dyn_cast<Instruction>(A);
-  }
-  while(match(Acc, m_Add(m_Value(A),
-                         m_SExt(m_Mul(m_Value(MulOp0), m_Value(MulOp1)))))) {
-    Value *Mul = cast<Instruction>(Acc->getOperand(1))->getOperand(0);
-    AddMACCandidate(Candidates, Acc, cast<Instruction>(Mul), MulOp0, MulOp1);
-    Acc = dyn_cast<Instruction>(A);
-  }
-  if (match(Acc, m_Add(m_SExt(m_Mul(m_Value(MulOp0), m_Value(MulOp1))),
-                       m_Value(A)))) {
-    Value *Mul = cast<Instruction>(
-      cast<Instruction>(Acc)->getOperand(0))->getOperand(0);
-    AddMACCandidate(Candidates, Acc, cast<Instruction>(Mul), MulOp0, MulOp1);
-  }
-
-  // Because we start at the bottom of the chain, and we work our way up,
-  // the muls are added in reverse program order to the list.
-  std::reverse(Candidates.begin(), Candidates.end());
+  Instruction *Acc = R.AccIntAdd;
+  LLVM_DEBUG(dbgs() << "\n- Analysing:\t" << *Acc);
+
+  // Returns false to signal the search should be stopped.
+  std::function<bool(Value*)> Match =
+    [&Candidates, &Match](Value *V) -> bool {
+
+    auto *I = dyn_cast<Instruction>(V);
+    if (!V)
+      return false;
+
+    Value *MulOp0, *MulOp1;
+
+    switch (I->getOpcode()) {
+    case Instruction::Add:
+      if (Match(I->getOperand(0)) || (Match(I->getOperand(1))))
+        return true;
+      break;
+    case Instruction::Mul:
+      if (match (I, (m_Mul(m_Value(MulOp0), m_Value(MulOp1))))) {
+        AddMACCandidate(Candidates, I, MulOp0, MulOp1);
+        return false;
+      }
+      break;
+    case Instruction::SExt:
+      if (match (I, (m_SExt(m_Mul(m_Value(MulOp0), m_Value(MulOp1)))))) {
+        Instruction *Mul = cast<Instruction>(I->getOperand(0));
+        AddMACCandidate(Candidates, Mul, MulOp0, MulOp1);
+        return false;
+      }
+      break;
+    }
+    return false;
+  };
+
+  while (Match (Acc));
+  LLVM_DEBUG(dbgs() << "Finished matching MAC sequences, found "
+             << Candidates.size() << " candidates.\n");
 }
 
 // Collects all instructions that are not part of the MAC chains, which is the
 // set of instructions that can potentially alias with the MAC operands.
 static void AliasCandidates(BasicBlock *Header, Instructions &Reads,
                             Instructions &Writes) {
   for (auto &I : *Header) {
     if (I.mayReadFromMemory())
for (auto &R : Reductions) {
     PMACPairList PMACPairs = CreateParallelMACPairs(R.MACCandidates);
     Changed |= InsertParallelMACs(R, PMACPairs);
   }
 
   LLVM_DEBUG(if (Changed) dbgs() << "Header block:\n"; Header->dump(););
   return Changed;
 }
 
-static void CreateLoadIns(IRBuilder<NoFolder> &IRB, LoadInst **VecLd,
+static LoadInst *CreateLoadIns(IRBuilder<NoFolder> &IRB, LoadInst &BaseLoad,
                                const Type *LoadTy) {
-  const unsigned AddrSpace = (*VecLd)->getPointerAddressSpace();
+  const unsigned AddrSpace = BaseLoad.getPointerAddressSpace();
 
-  Value *VecPtr = IRB.CreateBitCast((*VecLd)->getPointerOperand(),
+  Value *VecPtr = IRB.CreateBitCast(BaseLoad.getPointerOperand(),
                                     LoadTy->getPointerTo(AddrSpace));
-  *VecLd = IRB.CreateAlignedLoad(VecPtr, (*VecLd)->getAlignment());
+  return IRB.CreateAlignedLoad(VecPtr, BaseLoad.getAlignment());
 }
 
 Instruction *ARMParallelDSP::CreateSMLADCall(LoadInst *VecLd0, LoadInst *VecLd1,
-                                             Instruction *Acc,
+                                             Instruction *Acc, bool Exchange,
                                              Instruction *InsertAfter) {
-  LLVM_DEBUG(dbgs() << "Create SMLAD intrinsic using:\n";
-             dbgs() << "- "; VecLd0->dump();
-             dbgs() << "- "; VecLd1->dump();
-             dbgs() << "- "; Acc->dump());
+  LLVM_DEBUG(dbgs() << "Create SMLAD intrinsic using:\n"
+             << "- " << *VecLd0 << "\n"
+             << "- " << *VecLd1 << "\n"
+             << "- " << *Acc << "\n"
+             << "Exchange: " << Exchange << "\n");
 
   IRBuilder<NoFolder> Builder(InsertAfter->getParent(),
                               ++BasicBlock::iterator(InsertAfter));
 
   // Replace the reduction chain with an intrinsic call
   const Type *Ty = IntegerType::get(M->getContext(), 32);
-  CreateLoadIns(Builder, &VecLd0, Ty);
-  CreateLoadIns(Builder, &VecLd1, Ty);
-  Value* Args[] = { VecLd0, VecLd1, Acc };
-  Function *SMLAD = Acc->getType()->isIntegerTy(32) ?
+  LoadInst *NewLd0 = CreateLoadIns(Builder, VecLd0[0], Ty);
+  LoadInst *NewLd1 = CreateLoadIns(Builder, VecLd1[0], Ty);
+  Value* Args[] = { NewLd0, NewLd1, Acc };
+  Function *SMLAD = nullptr;
+  if (Exchange)
+    SMLAD = Acc->getType()->isIntegerTy(32) ?
+      Intrinsic::getDeclaration(M, Intrinsic::arm_smladx) :
+      Intrinsic::getDeclaration(M, Intrinsic::arm_smlaldx);
+  else
+    SMLAD = Acc->getType()->isIntegerTy(32) ?
       Intrinsic::getDeclaration(M, Intrinsic::arm_smlad) :
       Intrinsic::getDeclaration(M, Intrinsic::arm_smlald);
   CallInst *Call = Builder.CreateCall(SMLAD, Args);
   NumSMLAD++;
   return Call;
 }
 
 Pass *llvm::createARMParallelDSPPass() {
   return new ARMParallelDSP();
 }
 
 char ARMParallelDSP::ID = 0;
 
 INITIALIZE_PASS_BEGIN(ARMParallelDSP, "arm-parallel-dsp",
                 "Transform loops to use DSP intrinsics", false, false)
 INITIALIZE_PASS_END(ARMParallelDSP, "arm-parallel-dsp",
                 "Transform loops to use DSP intrinsics", false, false)

llvm/trunk/test/CodeGen/ARM/smladx-1.ll

+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define i32 @smladx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smladx
+; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
+; CHECK: [[ACC0:%[^ ]+]] = phi i32 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
+; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
+; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
+; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN23]], i32 [[IN12]], i32 [[ACC0]])
+; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
+; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
+; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
+; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN21]], i32 [[IN10]], i32 [[ACC1]])
+; CHECK-NOT: call i32 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i32 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i32 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sum.010.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sub.epil = sub i32 %j, %i.011.epil
+  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
+  %2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nsw i32 %mul.epil, %sum.010.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i32 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
+  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
+  %In2 = load i16, i16* %pIn2.0, align 2
+  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
+  %In1 = load i16, i16* %pIn1.0, align 2
+  %inc = or i32 %i.011, 1
+  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
+  %In2.1 = load i16, i16* %pIn2.1, align 2
+  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
+  %In1.1 = load i16, i16* %pIn1.1, align 2
+  %inc.1 = or i32 %i.011, 2
+  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
+  %In2.2 = load i16, i16* %pIn2.2, align 2
+  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
+  %In1.2 = load i16, i16* %pIn1.2, align 2
+  %inc.2 = or i32 %i.011, 3
+  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
+  %In2.3 = load i16, i16* %pIn2.3, align 2
+  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
+  %In1.3 = load i16, i16* %pIn1.3, align 2
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %mul = mul nsw i32 %sextIn1, %sextIn2
+  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
+  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
+  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
+  %add = add nsw i32 %mul, %sum.010
+  %add.1 = add nsw i32 %mul.1, %add
+  %add.2 = add nsw i32 %mul.2, %add.1
+  %add.3 = add nsw i32 %mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define i32 @smladx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smladx_swap
+; CHECK: for.body.preheader.new:
+; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
+; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
+
+; CHECK: for.body:
+; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
+; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
+; CHECK: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[ACC0:%[^ ]+]] = phi i32 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+
+; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
+; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
+
+
+; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
+; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
+; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
+; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i32 @llvm.arm.smladx(i32 [[IN2_2]], i32 [[IN1]], i32 [[ACC0]])
+
+; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
+; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
+; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
+; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC2]] = call i32 @llvm.arm.smladx(i32 [[IN2]], i32 [[IN1_2]], i32 [[ACC1]])
+
+; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
+; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
+
+; CHECK-NOT: call i32 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i32 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
+  %2 = add i32 %j, -1
+  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i32 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i32 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
+
+for.body.epil.preheader:
+  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
+  %3 = sub i32 %j, %i.011.unr
+  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
+  %4 = sub i32 0, %xtraiter
+  br label %for.body.epil
+
+for.body.epil:
+  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
+  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
+  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
+  %sum.010.epil = phi i32 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
+  %5 = load i16, i16* %lsr.iv3, align 2
+  %conv.epil = sext i16 %5 to i32
+  %6 = load i16, i16* %lsr.iv, align 2
+  %conv2.epil = sext i16 %6 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %add.epil = add nsw i32 %mul.epil, %sum.010.epil
+  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
+  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
+  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
+  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i32 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i32 %sum.0.lcssa
+
+for.body:
+  %pin2 = phi i16* [ %pin2_sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
+  %pin1 = phi i16* [ %pin1_add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i32 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %pin2_add1 = getelementptr i16, i16* %pin2, i32 1
+  %In2 = load i16, i16* %pin2_add1, align 2
+  %pin1_sub2 = getelementptr i16, i16* %pin1, i32 -2
+  %In1 = load i16, i16* %pin1_sub2, align 2
+  %In2.1 = load i16, i16* %pin2, align 2
+  %pin1_sub1 = getelementptr i16, i16* %pin1, i32 -1
+  %In1.1 = load i16, i16* %pin1_sub1, align 2
+  %pin2_sub1 = getelementptr i16, i16* %pin2, i32 -1
+  %In2.2 = load i16, i16* %pin2_sub1, align 2
+  %In1.2 = load i16, i16* %pin1, align 2
+  %pin2_sub2 = getelementptr i16, i16* %pin2, i32 -2
+  %In2.3 = load i16, i16* %pin2_sub2, align 2
+  %pin1_add1 = getelementptr i16, i16* %pin1, i32 1
+  %In1.3 = load i16, i16* %pin1_add1, align 2
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %mul = mul nsw i32 %sextIn2, %sextIn1
+  %add = add nsw i32 %mul, %sum.010
+  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
+  %add.1 = add nsw i32 %mul.1, %add
+  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
+  %add.2 = add nsw i32 %mul.2, %add.1
+  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
+  %add.3 = add nsw i32 %mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pin1_add4 = getelementptr i16, i16* %pin1, i32 4
+  %pin2_sub4 = getelementptr i16, i16* %pin2, i32 -4
+  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}

llvm/trunk/test/CodeGen/ARM/smlaldx-1.ll

+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smlaldx
+; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
+; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
+; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
+; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
+; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
+; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
+; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
+; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sub.epil = sub i32 %j, %i.011.epil
+  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
+  %2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+for.body:
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
+  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
+  %In2 = load i16, i16* %pIn2.0, align 2
+  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
+  %In1 = load i16, i16* %pIn1.0, align 2
+  %inc = or i32 %i.011, 1
+  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
+  %In2.1 = load i16, i16* %pIn2.1, align 2
+  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
+  %In1.1 = load i16, i16* %pIn1.1, align 2
+  %inc.1 = or i32 %i.011, 2
+  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
+  %In2.2 = load i16, i16* %pIn2.2, align 2
+  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
+  %In1.2 = load i16, i16* %pIn1.2, align 2
+  %inc.2 = or i32 %i.011, 3
+  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
+  %In2.3 = load i16, i16* %pIn2.3, align 2
+  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
+  %In1.3 = load i16, i16* %pIn1.3, align 2
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %mul = mul nsw i32 %sextIn1, %sextIn2
+  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
+  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
+  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
+  %sext.mul = sext i32 %mul to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add = add nsw i64 %sext.mul, %sum.010
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %add.2 = add nsw i64 %sext.mul.2, %add.1
+  %add.3 = add nsw i64 %sext.mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define i64 @smlaldx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
+  %2 = add i32 %j, -1
+  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
+
+for.body.epil.preheader:
+  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
+  %3 = sub i32 %j, %i.011.unr
+  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
+  %4 = sub i32 0, %xtraiter
+  br label %for.body.epil
+
+for.body.epil:
+  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
+  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
+  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
+  %5 = load i16, i16* %lsr.iv3, align 2
+  %conv.epil = sext i16 %5 to i32
+  %6 = load i16, i16* %lsr.iv, align 2
+  %conv2.epil = sext i16 %6 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
+  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
+  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
+  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+; CHECK-LABEL: smlaldx_swap
+; CHECK: for.body.preheader.new:
+; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
+; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
+
+; CHECK: for.body:
+; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
+; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
+; CHECK: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+
+; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
+; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
+
+; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
+; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
+; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
+; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2_2]], i32 [[IN1]], i64 [[ACC0]])
+
+; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
+; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
+; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
+; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC1]])
+
+; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
+; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
+
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+for.body:
+  %pin2 = phi i16* [ %pin2.sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
+  %pin1 = phi i16* [ %pin1.add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %pin2.add1 = getelementptr i16, i16* %pin2, i32 1
+  %In2 = load i16, i16* %pin2.add1, align 2
+  %pin1.sub2 = getelementptr i16, i16* %pin1, i32 -2
+  %In1 = load i16, i16* %pin1.sub2, align 2
+  %In2.1 = load i16, i16* %pin2, align 2
+  %pin1.sub1 = getelementptr i16, i16* %pin1, i32 -1
+  %In1.1 = load i16, i16* %pin1.sub1, align 2
+  %pin2.sub1 = getelementptr i16, i16* %pin2, i32 -1
+  %In2.2 = load i16, i16* %pin2.sub1, align 2
+  %In1.2 = load i16, i16* %pin1, align 2
+  %pin2.sub2 = getelementptr i16, i16* %pin2, i32 -2
+  %In2.3 = load i16, i16* %pin2.sub2, align 2
+  %pin1.add1 = getelementptr i16, i16* %pin1, i32 1
+  %In1.3 = load i16, i16* %pin1.add1, align 2
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %mul = mul nsw i32 %sextIn2, %sextIn1
+  %sext.mul = sext i32 %mul to i64
+  %add = add nsw i64 %sext.mul, %sum.010
+  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %add.2 = add nsw i64 %sext.mul.2, %add.1
+  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add.3 = add nsw i64 %sext.mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pin1.add4 = getelementptr i16, i16* %pin1, i32 4
+  %pin2.sub4 = getelementptr i16, i16* %pin2, i32 -4
+  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}

llvm/trunk/test/CodeGen/ARM/smlaldx-2.ll

+; RUN: opt -mtriple=thumbv8m.main -mcpu=cortex-m33 -arm-parallel-dsp %s -S -o - | FileCheck %s
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m0 < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+; RUN: opt -mtriple=arm-arm-eabi -mcpu=cortex-m33 -mattr=-dsp < %s -arm-parallel-dsp -S | FileCheck %s --check-prefix=CHECK-UNSUPPORTED
+
+define i64 @smlaldx(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+; CHECK-LABEL: smlaldx
+; CHECK: = phi i32 [ 0, %for.body.preheader.new ],
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN23:%[^ ]+]] = bitcast i16* %pIn2.3 to i32*
+; CHECK: [[IN23:%[^ ]+]] = load i32, i32* [[PIN23]], align 2
+; CHECK: [[PIN12:%[^ ]+]] = bitcast i16* %pIn1.2 to i32*
+; CHECK: [[IN12:%[^ ]+]] = load i32, i32* [[PIN12]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN23]], i32 [[IN12]], i64 [[ACC0]])
+; CHECK: [[PIN21:%[^ ]+]] = bitcast i16* %pIn2.1 to i32*
+; CHECK: [[IN21:%[^ ]+]] = load i32, i32* [[PIN21]], align 2
+; CHECK: [[PIN10:%[^ ]+]] = bitcast i16* %pIn1.0 to i32*
+; CHECK: [[IN10:%[^ ]+]] = load i32, i32* [[PIN10]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN21]], i32 [[IN10]], i64 [[ACC1]])
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil
+
+for.body.epil:
+  %i.011.epil = phi i32 [ %inc.epil, %for.body.epil ], [ %i.011.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %epil.iter = phi i32 [ %epil.iter.sub, %for.body.epil ], [ %xtraiter, %for.cond.cleanup.loopexit.unr-lcssa ]
+  %sub.epil = sub i32 %j, %i.011.epil
+  %arrayidx.epil = getelementptr inbounds i16, i16* %pIn2, i32 %sub.epil
+  %2 = load i16, i16* %arrayidx.epil, align 2
+  %conv.epil = sext i16 %2 to i32
+  %arrayidx1.epil = getelementptr inbounds i16, i16* %pIn1, i32 %i.011.epil
+  %3 = load i16, i16* %arrayidx1.epil, align 2
+  %conv2.epil = sext i16 %3 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %inc.epil = add nuw i32 %i.011.epil, 1
+  %epil.iter.sub = add i32 %epil.iter, -1
+  %epil.iter.cmp = icmp eq i32 %epil.iter.sub, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+for.body:
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %niter = phi i32 [ %unroll_iter, %for.body.preheader.new ], [ %niter.nsub.3, %for.body ]
+  %pIn2Base = phi i16* [ %pIn2, %for.body.preheader.new ], [ %pIn2.4, %for.body ]
+  %pIn2.0 = getelementptr inbounds i16, i16* %pIn2Base, i32 0
+  %In2 = load i16, i16* %pIn2.0, align 2
+  %pIn1.0 = getelementptr inbounds i16, i16* %pIn1, i32 %i.011
+  %In1 = load i16, i16* %pIn1.0, align 2
+  %inc = or i32 %i.011, 1
+  %pIn2.1 = getelementptr inbounds i16, i16* %pIn2Base, i32 -1
+  %In2.1 = load i16, i16* %pIn2.1, align 2
+  %pIn1.1 = getelementptr inbounds i16, i16* %pIn1, i32 %inc
+  %In1.1 = load i16, i16* %pIn1.1, align 2
+  %inc.1 = or i32 %i.011, 2
+  %pIn2.2 = getelementptr inbounds i16, i16* %pIn2Base, i32 -2
+  %In2.2 = load i16, i16* %pIn2.2, align 2
+  %pIn1.2 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.1
+  %In1.2 = load i16, i16* %pIn1.2, align 2
+  %inc.2 = or i32 %i.011, 3
+  %pIn2.3 = getelementptr inbounds i16, i16* %pIn2Base, i32 -3
+  %In2.3 = load i16, i16* %pIn2.3, align 2
+  %pIn1.3 = getelementptr inbounds i16, i16* %pIn1, i32 %inc.2
+  %In1.3 = load i16, i16* %pIn1.3, align 2
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %mul = mul nsw i32 %sextIn1, %sextIn2
+  %mul.1 = mul nsw i32 %sextIn1.1, %sextIn2.1
+  %mul.2 = mul nsw i32 %sextIn1.2, %sextIn2.2
+  %mul.3 = mul nsw i32 %sextIn1.3, %sextIn2.3
+  %sext.mul = sext i32 %mul to i64
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add = add nsw i64 %sum.010, %sext.mul
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %add.2 = add nsw i64 %add.1, %sext.mul.2
+  %add.3 = add nsw i64 %sext.mul.3, %add.2
+  %inc.3 = add i32 %i.011, 4
+  %pIn2.4 = getelementptr inbounds i16, i16* %pIn2Base, i32 -4
+  %niter.nsub.3 = add i32 %niter, -4
+  %niter.ncmp.3 = icmp eq i32 %niter.nsub.3, 0
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}
+
+define i64 @smlaldx_swap(i16* nocapture readonly %pIn1, i16* nocapture readonly %pIn2, i32 %j, i32 %limit) {
+
+entry:
+  %cmp9 = icmp eq i32 %limit, 0
+  br i1 %cmp9, label %for.cond.cleanup, label %for.body.preheader
+
+for.body.preheader:
+  %0 = add i32 %limit, -1
+  %xtraiter = and i32 %limit, 3
+  %1 = icmp ult i32 %0, 3
+  br i1 %1, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body.preheader.new
+
+for.body.preheader.new:
+  %unroll_iter = sub i32 %limit, %xtraiter
+  %scevgep6 = getelementptr i16, i16* %pIn1, i32 2
+  %2 = add i32 %j, -1
+  %scevgep11 = getelementptr i16, i16* %pIn2, i32 %2
+  br label %for.body
+
+for.cond.cleanup.loopexit.unr-lcssa:
+  %add.lcssa.ph = phi i64 [ undef, %for.body.preheader ], [ %add.3, %for.body ]
+  %i.011.unr = phi i32 [ 0, %for.body.preheader ], [ %inc.3, %for.body ]
+  %sum.010.unr = phi i64 [ 0, %for.body.preheader ], [ %add.3, %for.body ]
+  %lcmp.mod = icmp eq i32 %xtraiter, 0
+  br i1 %lcmp.mod, label %for.cond.cleanup, label %for.body.epil.preheader
+
+for.body.epil.preheader:
+  %scevgep = getelementptr i16, i16* %pIn1, i32 %i.011.unr
+  %3 = sub i32 %j, %i.011.unr
+  %scevgep2 = getelementptr i16, i16* %pIn2, i32 %3
+  %4 = sub i32 0, %xtraiter
+  br label %for.body.epil
+
+for.body.epil:
+  %lsr.iv5 = phi i32 [ %4, %for.body.epil.preheader ], [ %lsr.iv.next, %for.body.epil ]
+  %lsr.iv3 = phi i16* [ %scevgep2, %for.body.epil.preheader ], [ %scevgep4, %for.body.epil ]
+  %lsr.iv = phi i16* [ %scevgep, %for.body.epil.preheader ], [ %scevgep1, %for.body.epil ]
+  %sum.010.epil = phi i64 [ %add.epil, %for.body.epil ], [ %sum.010.unr, %for.body.epil.preheader ]
+  %5 = load i16, i16* %lsr.iv3, align 2
+  %conv.epil = sext i16 %5 to i32
+  %6 = load i16, i16* %lsr.iv, align 2
+  %conv2.epil = sext i16 %6 to i32
+  %mul.epil = mul nsw i32 %conv2.epil, %conv.epil
+  %sext.mul.epil = sext i32 %mul.epil to i64
+  %add.epil = add nsw i64 %sext.mul.epil, %sum.010.epil
+  %scevgep1 = getelementptr i16, i16* %lsr.iv, i32 1
+  %scevgep4 = getelementptr i16, i16* %lsr.iv3, i32 -1
+  %lsr.iv.next = add nsw i32 %lsr.iv5, 1
+  %epil.iter.cmp = icmp eq i32 %lsr.iv.next, 0
+  br i1 %epil.iter.cmp, label %for.cond.cleanup, label %for.body.epil
+
+for.cond.cleanup:
+  %sum.0.lcssa = phi i64 [ 0, %entry ], [ %add.lcssa.ph, %for.cond.cleanup.loopexit.unr-lcssa ], [ %add.epil, %for.body.epil ]
+  ret i64 %sum.0.lcssa
+
+; CHECK-LABEL: smlaldx_swap
+; CHECK: for.body.preheader.new:
+; CHECK: [[PIN1Base:[^ ]+]] = getelementptr i16, i16* %pIn1
+; CHECK: [[PIN2Base:[^ ]+]] = getelementptr i16, i16* %pIn2
+
+; CHECK: for.body:
+; CHECK: [[PIN2:%[^ ]+]] = phi i16* [ [[PIN2_NEXT:%[^ ]+]], %for.body ], [ [[PIN2Base]], %for.body.preheader.new ]
+; CHECK: [[PIN1:%[^ ]+]] = phi i16* [ [[PIN1_NEXT:%[^ ]+]], %for.body ], [ [[PIN1Base]], %for.body.preheader.new ]
+; CHECK: [[IV:%[^ ]+]] = phi i32
+; CHECK: [[ACC0:%[^ ]+]] = phi i64 [ 0, %for.body.preheader.new ], [ [[ACC2:%[^ ]+]], %for.body ]
+; CHECK: [[PIN1_2:%[^ ]+]] = getelementptr i16, i16* [[PIN1]], i32 -2
+; CHECK: [[PIN2_2:%[^ ]+]] = getelementptr i16, i16* [[PIN2]], i32 -2
+
+; CHECK: [[PIN2_CAST:%[^ ]+]] = bitcast i16* [[PIN2]] to i32*
+; CHECK: [[IN2:%[^ ]+]] = load i32, i32* [[PIN2_CAST]], align 2
+; CHECK: [[PIN1_2_CAST:%[^ ]+]] = bitcast i16* [[PIN1_2]] to i32*
+; CHECK: [[IN1_2:%[^ ]+]] = load i32, i32* [[PIN1_2_CAST]], align 2
+; CHECK: [[ACC1:%[^ ]+]] = call i64 @llvm.arm.smlaldx(i32 [[IN2]], i32 [[IN1_2]], i64 [[ACC0]])
+
+; CHECK: [[PIN1_CAST:%[^ ]+]] = bitcast i16* [[PIN1]] to i32*
+; CHECK: [[IN1:%[^ ]+]] = load i32, i32* [[PIN1_CAST]], align 2
+; CHECK: [[PIN2_2_CAST:%[^ ]+]] = bitcast i16* [[PIN2_2]] to i32*
+; CHECK: [[IN2_2:%[^ ]+]] = load i32, i32* [[PIN2_2_CAST]], align 2
+; CHECK: [[ACC2]] = call i64 @llvm.arm.smlaldx(i32 [[IN1]], i32 [[IN2_2]], i64 [[ACC1]])
+
+; CHECK: [[PIN1_NEXT]] = getelementptr i16, i16* [[PIN1]], i32 4
+; CHECK: [[PIN2_NEXT]] = getelementptr i16, i16* [[PIN2]], i32 -4
+
+; CHECK-NOT: call i64 @llvm.arm.smlad
+; CHECK-UNSUPPORTED-NOT:  call i64 @llvm.arm.smlad
+
+for.body:
+  %pin2 = phi i16* [ %pin2.sub4, %for.body ], [ %scevgep11, %for.body.preheader.new ]
+  %pin1 = phi i16* [ %pin1.add4, %for.body ], [ %scevgep6, %for.body.preheader.new ]
+  %i.011 = phi i32 [ 0, %for.body.preheader.new ], [ %inc.3, %for.body ]
+  %sum.010 = phi i64 [ 0, %for.body.preheader.new ], [ %add.3, %for.body ]
+  %pin2.add1 = getelementptr i16, i16* %pin2, i32 1
+  %In2 = load i16, i16* %pin2.add1, align 2
+  %pin1.sub2 = getelementptr i16, i16* %pin1, i32 -2
+  %In1 = load i16, i16* %pin1.sub2, align 2
+  %In2.1 = load i16, i16* %pin2, align 2
+  %pin1.sub1 = getelementptr i16, i16* %pin1, i32 -1
+  %In1.1 = load i16, i16* %pin1.sub1, align 2
+  %pin2.sub1 = getelementptr i16, i16* %pin2, i32 -1
+  %In2.2 = load i16, i16* %pin2.sub1, align 2
+  %In1.2 = load i16, i16* %pin1, align 2
+  %pin2.sub2 = getelementptr i16, i16* %pin2, i32 -2
+  %In2.3 = load i16, i16* %pin2.sub2, align 2
+  %pin1.add1 = getelementptr i16, i16* %pin1, i32 1
+  %In1.3 = load i16, i16* %pin1.add1, align 2
+  %sextIn2 = sext i16 %In2 to i32
+  %sextIn1 = sext i16 %In1 to i32
+  %sextIn2.1 = sext i16 %In2.1 to i32
+  %sextIn1.1 = sext i16 %In1.1 to i32
+  %sextIn2.2 = sext i16 %In2.2 to i32
+  %sextIn1.2 = sext i16 %In1.2 to i32
+  %sextIn2.3 = sext i16 %In2.3 to i32
+  %sextIn1.3 = sext i16 %In1.3 to i32
+  %mul = mul nsw i32 %sextIn2, %sextIn1
+  %sext.mul = sext i32 %mul to i64
+  %add = add nsw i64 %sext.mul, %sum.010
+  %mul.1 = mul nsw i32 %sextIn2.1, %sextIn1.1
+  %sext.mul.1 = sext i32 %mul.1 to i64
+  %add.1 = add nsw i64 %sext.mul.1, %add
+  %mul.2 = mul nsw i32 %sextIn2.2, %sextIn1.2
+  %sext.mul.2 = sext i32 %mul.2 to i64
+  %add.2 = add nsw i64 %add.1, %sext.mul.2
+  %mul.3 = mul nsw i32 %sextIn2.3, %sextIn1.3
+  %sext.mul.3 = sext i32 %mul.3 to i64
+  %add.3 = add nsw i64 %add.2, %sext.mul.3
+  %inc.3 = add i32 %i.011, 4
+  %pin1.add4 = getelementptr i16, i16* %pin1, i32 4
+  %pin2.sub4 = getelementptr i16, i16* %pin2, i32 -4
+  %niter.ncmp.3 = icmp eq i32 %unroll_iter, %inc.3
+  br i1 %niter.ncmp.3, label %for.cond.cleanup.loopexit.unr-lcssa, label %for.body
+}