[LoopVectorization] Add an API to make the LoopVectorizer publically accessible

[jmolloy]

Summary

There's currently only one way to vectorize a loop - by using the LoopVectorize driver pass. But there's no real reason to have this restriction.

Taking examples from elsewhere in the codebase, I've created an API that takes all its dependencies explicitly and can be called from anywhere in the codebase (subject to layering violations of course).

LoopVectorize turns into a shell wrapper pass, which takes its dependencies from the pass manager and dumbly passes them on to the new LoopVectorization class.

The API makes a distinction between loop vectorization and loop interleaving. Even though these are very conflated within the loop vectorizer, I felt it was important to keep them separate at the API level.

[anemet]

Hi James,

Can you please explain the plan here? I am a bit confused how interleaving is now done as part of LoopVectorization::vectorize even though you create a new parallel class LoopInterleaving. What is the plan here?

Besides, I also have some more questions and nitpicks below but probably this first question is the most critical one.

Thanks,
Adam

I am assuming this is NFC. Please mention that for each patch.

Do you have plans to move out the implementation to its own module? If not immediately, you may want to explain the difficulty in the header or somewhere.

include/llvm/Transforms/Utils/LoopWidening.h

60

s/Add/add

lib/Transforms/Vectorize/LoopVectorize.cpp

1609–1611

This does not look like properly clang-formatted

1614–1618

I think it's probably better to only add these arguments when they become supported.

include/llvm/Transforms/Utils/LoopWidening.h

+//===- LoopWidening.h - Loop vectorization and interleaving ---------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines utility classes for widening loops - coalescing multiple
+// iterations together. It provides utilities for vectorizing and interleaving
+// loops.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_TRANSFORMS_UTILS_LOOP_WIDENING_H
+#define LLVM_TRANSFORMS_UTILS_LOOP_WIDENING_H
+
+namespace llvm {
+
+class Loop;
+class ScalarEvolution;
+class LoopInfo;
+class DominatorTree;
+class TargetLibraryInfo;
+class TargetTransformInfo;
+class LoopAccessAnalysis;
+class BlockFrequencyInfo;
+class BlockFrequency;
+class AssumptionCache;
+
+/// Performs vectorization on a loop.
+class LoopVectorization {
+public:
+  /// Creates a new LoopVectorization utility.
+  /// \param DisableUnrolling Disable interleaving completely, overriding the
+  ///        cost model and loop hints.
+  /// \param AlwaysVectorize Forces vectorization, like #pragma vectorize enable.
+  LoopVectorization(ScalarEvolution *SE, LoopInfo *LI,
+                    DominatorTree *DT,
+                    const TargetTransformInfo *TTI,
+                    LoopAccessAnalysis *LAA,
+                    AliasAnalysis *AA,
+                    AssumptionCache *AC,
+                    BlockFrequencyInfo *BFI = nullptr,
+                    TargetLibraryInfo *TLI = nullptr,
+                    bool DisableUnrolling = false,
+                    bool AlwaysVectorize = false);
+
+  /// Perform the vectorization of the loop. Return true if the loop was
+  /// vectorized successfully.
+  /// \param L The loop to vectorize.
+  /// \param VectorWidth An override for the vector factor, or -1 to select
+  ///        based on the cost model and loop hints.
+  /// \param UnrollFactor An override for the interleave factor, or -1 to select
+  ///        based on the cost model and loop hints.
+  bool vectorize(Loop *L, int VectorWidth = -1, int UnrollFactor = -1);
+
+protected:
+  void AddRuntimeUnrollDisableMetaData(Loop *L);
+  
+  ScalarEvolution *SE;
+  LoopInfo *LI;
+  DominatorTree *DT;
+  const TargetTransformInfo *TTI;
+  LoopAccessAnalysis *LAA;
+  AliasAnalysis *AA;
+  AssumptionCache *AC;
+  BlockFrequencyInfo *BFI;
+  TargetLibraryInfo *TLI;
+  BlockFrequency *ColdEntryFreq;
+  bool DisableUnrolling, AlwaysVectorize;
+};
+
+/// Performs an interleaving unroll on a loop.
+class LoopInterleaving : public LoopVectorization {
+public:
+  using LoopVectorization::LoopVectorization;
+
+  /// Perform the interleaving of the loop. Return true if the loop was
+  /// interleaved successfully.
+  /// \param L The loop to vectorize.
+  /// \param UnrollFactor An override for the interleave factor, or -1 to select
+  ///        based on the cost model and loop hints.
+  bool interleave(Loop *L, int UnrollFactor=-1) {
+    return vectorize(L, 1, UnrollFactor);
+  }
+
+};
+  
+}
+
+#endif

lib/Transforms/Vectorize/LoopVectorize.cpp

 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Analysis/VectorUtils.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/LoopWidening.h"
 #include <algorithm>
 #include <map>
 #include <tuple>
 
 using namespace llvm;
 using namespace llvm::PatternMatch;
 
 #define LV_NAME "loop-vectorize"
struct LoopVectorize : public FunctionPass {
   BlockFrequencyInfo *BFI;
   TargetLibraryInfo *TLI;
   AliasAnalysis *AA;
   AssumptionCache *AC;
   LoopAccessAnalysis *LAA;
   bool DisableUnrolling;
   bool AlwaysVectorize;
 
-  BlockFrequency ColdEntryFreq;
-
   bool runOnFunction(Function &F) override {
     SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
     LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
     TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
     DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
     BFI = &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI();
     auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
     TLI = TLIP ? &TLIP->getTLI() : nullptr;
     AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
     AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
     LAA = &getAnalysis<LoopAccessAnalysis>();
 
-    // Compute some weights outside of the loop over the loops. Compute this
-    // using a BranchProbability to re-use its scaling math.
-    const BranchProbability ColdProb(1, 5); // 20%
-    ColdEntryFreq = BlockFrequency(BFI->getEntryFreq()) * ColdProb;
-
     // Don't attempt if
     // 1. the target claims to have no vector registers, and
     // 2. interleaving won't help ILP.
     //
     // The second condition is necessary because, even if the target has no
     // vector registers, loop vectorization may still enable scalar
     // interleaving.
     if (!TTI->getNumberOfRegisters(true) && TTI->getMaxInterleaveFactor(1) < 2)
       return false;
 
     // Build up a worklist of inner-loops to vectorize. This is necessary as
     // the act of vectorizing or partially unrolling a loop creates new loops
     // and can invalidate iterators across the loops.
     SmallVector<Loop *, 8> Worklist;
 
     for (Loop *L : *LI)
       addInnerLoop(*L, Worklist);
 
     LoopsAnalyzed += Worklist.size();
 
     // Now walk the identified inner loops.
     bool Changed = false;
-    while (!Worklist.empty())
-      Changed |= processLoop(Worklist.pop_back_val());
+    while (!Worklist.empty()) {
+      LoopVectorization LV(SE, LI, DT, TTI, LAA, AA, AC, BFI, TLI,
+                           DisableUnrolling, AlwaysVectorize);
+      Changed |= LV.vectorize(Worklist.pop_back_val());
+    }
 
     // Process each loop nest in the function.
     return Changed;
   }
 
-  static void AddRuntimeUnrollDisableMetaData(Loop *L) {
-    SmallVector<Metadata *, 4> MDs;
-    // Reserve first location for self reference to the LoopID metadata node.
-    MDs.push_back(nullptr);
-    bool IsUnrollMetadata = false;
-    MDNode *LoopID = L->getLoopID();
-    if (LoopID) {
-      // First find existing loop unrolling disable metadata.
-      for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
-        MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
-        if (MD) {
-          const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
-          IsUnrollMetadata =
-              S && S->getString().startswith("llvm.loop.unroll.disable");
-        }
-        MDs.push_back(LoopID->getOperand(i));
-      }
-    }
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<AssumptionCacheTracker>();
+    AU.addRequiredID(LoopSimplifyID);
+    AU.addRequiredID(LCSSAID);
+    AU.addRequired<BlockFrequencyInfoWrapperPass>();
+    AU.addRequired<DominatorTreeWrapperPass>();
+    AU.addRequired<LoopInfoWrapperPass>();
+    AU.addRequired<ScalarEvolutionWrapperPass>();
+    AU.addRequired<TargetTransformInfoWrapperPass>();
+    AU.addRequired<AAResultsWrapperPass>();
+    AU.addRequired<LoopAccessAnalysis>();
+    AU.addPreserved<LoopInfoWrapperPass>();
+    AU.addPreserved<DominatorTreeWrapperPass>();
+    AU.addPreserved<BasicAAWrapperPass>();
+    AU.addPreserved<AAResultsWrapperPass>();
+    AU.addPreserved<GlobalsAAWrapperPass>();
 
-    if (!IsUnrollMetadata) {
-      // Add runtime unroll disable metadata.
-      LLVMContext &Context = L->getHeader()->getContext();
-      SmallVector<Metadata *, 1> DisableOperands;
-      DisableOperands.push_back(
-          MDString::get(Context, "llvm.loop.unroll.runtime.disable"));
-      MDNode *DisableNode = MDNode::get(Context, DisableOperands);
-      MDs.push_back(DisableNode);
-      MDNode *NewLoopID = MDNode::get(Context, MDs);
-      // Set operand 0 to refer to the loop id itself.
-      NewLoopID->replaceOperandWith(0, NewLoopID);
-      L->setLoopID(NewLoopID);
-    }
-  }
+};
+
+} // end anonymous namespace
+
+LoopVectorization::LoopVectorization(ScalarEvolution *SE, LoopInfo *LI,
+                                     DominatorTree *DT,
+                                     const TargetTransformInfo *TTI,
+                                     LoopAccessAnalysis *LAA,
+                                     AliasAnalysis *AA,
+                                     AssumptionCache *AC,
+                                     BlockFrequencyInfo *BFI,
+                                     TargetLibraryInfo *TLI,
+                                     bool DisableUnrolling,
+                                     bool AlwaysVectorize) :
+  SE(SE), LI(LI), DT(DT), TTI(TTI), LAA(LAA), AA(AA), AC(AC), BFI(BFI),
+  TLI(TLI),
+  DisableUnrolling(DisableUnrolling), AlwaysVectorize(AlwaysVectorize) {
 
-  bool processLoop(Loop *L) {
-    assert(L->empty() && "Only process inner loops.");
+bool LoopVectorization::vectorize(Loop *L,
+                                  int VectorWidth, int UnrollFactor) {
+  assert(VectorWidth == -1 && "Overriding VF not yet implemented!");
+  assert(UnrollFactor == -1 && "Overriding UF not yet implemented!");
   
 #ifndef NDEBUG
-    const std::string DebugLocStr = getDebugLocString(L);
 #endif /* NDEBUG */
 
-    DEBUG(dbgs() << "\nLV: Checking a loop in \""
-                 << L->getHeader()->getParent()->getName() << "\" from "
-                 << DebugLocStr << "\n");
 
-    LoopVectorizeHints Hints(L, DisableUnrolling);
 
-    DEBUG(dbgs() << "LV: Loop hints:"
-                 << " force="
-                 << (Hints.getForce() == LoopVectorizeHints::FK_Disabled
-                         ? "disabled"
-                         : (Hints.getForce() == LoopVectorizeHints::FK_Enabled
-                                ? "enabled"
-                                : "?")) << " width=" << Hints.getWidth()
-                 << " unroll=" << Hints.getInterleave() << "\n");
 
-    // Function containing loop
-    Function *F = L->getHeader()->getParent();
 
-    // Looking at the diagnostic output is the only way to determine if a loop
-    // was vectorized (other than looking at the IR or machine code), so it
-    // is important to generate an optimization remark for each loop. Most of
-    // these messages are generated by emitOptimizationRemarkAnalysis. Remarks
-    // generated by emitOptimizationRemark and emitOptimizationRemarkMissed are
-    // less verbose reporting vectorized loops and unvectorized loops that may
-    // benefit from vectorization, respectively.
 
-    if (!Hints.allowVectorization(F, L, AlwaysVectorize)) {
-      DEBUG(dbgs() << "LV: Loop hints prevent vectorization.\n");
-      return false;
-    }
 
-    // Check the loop for a trip count threshold:
-    // do not vectorize loops with a tiny trip count.
-    const unsigned TC = SE->getSmallConstantTripCount(L);
-    if (TC > 0u && TC < TinyTripCountVectorThreshold) {
-      DEBUG(dbgs() << "LV: Found a loop with a very small trip count. "
-                   << "This loop is not worth vectorizing.");
-      if (Hints.getForce() == LoopVectorizeHints::FK_Enabled)
-        DEBUG(dbgs() << " But vectorizing was explicitly forced.\n");
-      else {
-        DEBUG(dbgs() << "\n");
-        emitAnalysisDiag(F, L, Hints, VectorizationReport()
-                                          << "vectorization is not beneficial "
-                                             "and is not explicitly forced");
-        return false;
-      }
-    }
 
-    // Check if it is legal to vectorize the loop.
-    LoopVectorizationRequirements Requirements;
-    LoopVectorizationLegality LVL(L, SE, DT, TLI, AA, F, TTI, LAA,
-                                  &Requirements, &Hints);
-    if (!LVL.canVectorize()) {
-      DEBUG(dbgs() << "LV: Not vectorizing: Cannot prove legality.\n");
-      emitMissedWarning(F, L, Hints);
-      return false;
-    }
 
-    // Collect values we want to ignore in the cost model. This includes
-    // type-promoting instructions we identified during reduction detection.
-    SmallPtrSet<const Value *, 32> ValuesToIgnore;
-    CodeMetrics::collectEphemeralValues(L, AC, ValuesToIgnore);
-    for (auto &Reduction : *LVL.getReductionVars()) {
-      RecurrenceDescriptor &RedDes = Reduction.second;
-      SmallPtrSetImpl<Instruction *> &Casts = RedDes.getCastInsts();
-      ValuesToIgnore.insert(Casts.begin(), Casts.end());
-    }
 
-    // Use the cost model.
-    LoopVectorizationCostModel CM(L, SE, LI, &LVL, *TTI, TLI, AC, F, &Hints,
-                                  ValuesToIgnore);
 
-    // Check the function attributes to find out if this function should be
-    // optimized for size.
-    bool OptForSize = Hints.getForce() != LoopVectorizeHints::FK_Enabled &&
-                      F->optForSize();
 
-    // Compute the weighted frequency of this loop being executed and see if it
-    // is less than 20% of the function entry baseline frequency. Note that we
-    // always have a canonical loop here because we think we *can* vectorize.
-    // FIXME: This is hidden behind a flag due to pervasive problems with
-    // exactly what block frequency models.
-    if (LoopVectorizeWithBlockFrequency) {
-      BlockFrequency LoopEntryFreq = BFI->getBlockFreq(L->getLoopPreheader());
-      if (Hints.getForce() != LoopVectorizeHints::FK_Enabled &&
-          LoopEntryFreq < ColdEntryFreq)
-        OptForSize = true;
-    }
+  if (LoopVectorizeWithBlockFrequency && BFI) {
+    // Compute some weights outside of the loop over the loops. Compute this
+    // using a BranchProbability to re-use its scaling math.
+    const BranchProbability ColdProb(1, 5); // 20%
+    auto ColdEntryFreq = BlockFrequency(BFI->getEntryFreq()) * ColdProb;
+
 
-    // Check the function attributes to see if implicit floats are allowed.
-    // FIXME: This check doesn't seem possibly correct -- what if the loop is
-    // an integer loop and the vector instructions selected are purely integer
-    // vector instructions?
-    if (F->hasFnAttribute(Attribute::NoImplicitFloat)) {
-      DEBUG(dbgs() << "LV: Can't vectorize when the NoImplicitFloat"
-            "attribute is used.\n");
-      emitAnalysisDiag(
-          F, L, Hints,
-          VectorizationReport()
-              << "loop not vectorized due to NoImplicitFloat attribute");
-      emitMissedWarning(F, L, Hints);
-      return false;
-    }
 
-    // Select the optimal vectorization factor.
-    const LoopVectorizationCostModel::VectorizationFactor VF =
-        CM.selectVectorizationFactor(OptForSize);
 
-    // Select the interleave count.
-    unsigned IC = CM.selectInterleaveCount(OptForSize, VF.Width, VF.Cost);
 
-    // Get user interleave count.
-    unsigned UserIC = Hints.getInterleave();
 
-    // Identify the diagnostic messages that should be produced.
-    std::string VecDiagMsg, IntDiagMsg;
-    bool VectorizeLoop = true, InterleaveLoop = true;
 
-    if (Requirements.doesNotMeet(F, L, Hints)) {
-      DEBUG(dbgs() << "LV: Not vectorizing: loop did not meet vectorization "
-                      "requirements.\n");
-      emitMissedWarning(F, L, Hints);
-      return false;
-    }
 
-    if (VF.Width == 1) {
-      DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
-      VecDiagMsg =
-          "the cost-model indicates that vectorization is not beneficial";
-      VectorizeLoop = false;
-    }
 
-    if (IC == 1 && UserIC <= 1) {
-      // Tell the user interleaving is not beneficial.
-      DEBUG(dbgs() << "LV: Interleaving is not beneficial.\n");
-      IntDiagMsg =
-          "the cost-model indicates that interleaving is not beneficial";
-      InterleaveLoop = false;
-      if (UserIC == 1)
-        IntDiagMsg +=
-            " and is explicitly disabled or interleave count is set to 1";
-    } else if (IC > 1 && UserIC == 1) {
-      // Tell the user interleaving is beneficial, but it explicitly disabled.
-      DEBUG(dbgs()
-            << "LV: Interleaving is beneficial but is explicitly disabled.");
-      IntDiagMsg = "the cost-model indicates that interleaving is beneficial "
-                   "but is explicitly disabled or interleave count is set to 1";
-      InterleaveLoop = false;
-    }
 
-    // Override IC if user provided an interleave count.
-    IC = UserIC > 0 ? UserIC : IC;
 
-    // Emit diagnostic messages, if any.
-    const char *VAPassName = Hints.vectorizeAnalysisPassName();
-    if (!VectorizeLoop && !InterleaveLoop) {
-      // Do not vectorize or interleaving the loop.
-      emitOptimizationRemarkAnalysis(F->getContext(), VAPassName, *F,
-                                     L->getStartLoc(), VecDiagMsg);
-      emitOptimizationRemarkAnalysis(F->getContext(), LV_NAME, *F,
-                                     L->getStartLoc(), IntDiagMsg);
-      return false;
-    } else if (!VectorizeLoop && InterleaveLoop) {
-      DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
-      emitOptimizationRemarkAnalysis(F->getContext(), VAPassName, *F,
-                                     L->getStartLoc(), VecDiagMsg);
-    } else if (VectorizeLoop && !InterleaveLoop) {
-      DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width << ") in "
-                   << DebugLocStr << '\n');
-      emitOptimizationRemarkAnalysis(F->getContext(), LV_NAME, *F,
-                                     L->getStartLoc(), IntDiagMsg);
-    } else if (VectorizeLoop && InterleaveLoop) {
-      DEBUG(dbgs() << "LV: Found a vectorizable loop (" << VF.Width << ") in "
-                   << DebugLocStr << '\n');
-      DEBUG(dbgs() << "LV: Interleave Count is " << IC << '\n');
-    }
 
-    if (!VectorizeLoop) {
-      assert(IC > 1 && "interleave count should not be 1 or 0");
-      // If we decided that it is not legal to vectorize the loop then
-      // interleave it.
-      InnerLoopUnroller Unroller(L, SE, LI, DT, TLI, TTI, IC);
-      Unroller.vectorize(&LVL);
 
-      emitOptimizationRemark(F->getContext(), LV_NAME, *F, L->getStartLoc(),
-                             Twine("interleaved loop (interleaved count: ") +
-                                 Twine(IC) + ")");
-    } else {
-      // If we decided that it is *legal* to vectorize the loop then do it.
-      InnerLoopVectorizer LB(L, SE, LI, DT, TLI, TTI, VF.Width, IC);
-      LB.vectorize(&LVL);
-      ++LoopsVectorized;
 
-      // Add metadata to disable runtime unrolling scalar loop when there's no
-      // runtime check about strides and memory. Because at this situation,
-      // scalar loop is rarely used not worthy to be unrolled.
-      if (!LB.IsSafetyChecksAdded())
-        AddRuntimeUnrollDisableMetaData(L);
 
-      // Report the vectorization decision.
-      emitOptimizationRemark(F->getContext(), LV_NAME, *F, L->getStartLoc(),
-                             Twine("vectorized loop (vectorization width: ") +
-                                 Twine(VF.Width) + ", interleaved count: " +
-                                 Twine(IC) + ")");
-    }
 
-    // Mark the loop as already vectorized to avoid vectorizing again.
-    Hints.setAlreadyVectorized();
 
-    DEBUG(verifyFunction(*L->getHeader()->getParent()));
-    return true;
-  }
+}  
 
-  void getAnalysisUsage(AnalysisUsage &AU) const override {
-    AU.addRequired<AssumptionCacheTracker>();
-    AU.addRequiredID(LoopSimplifyID);
-    AU.addRequiredID(LCSSAID);
-    AU.addRequired<BlockFrequencyInfoWrapperPass>();
-    AU.addRequired<DominatorTreeWrapperPass>();
-    AU.addRequired<LoopInfoWrapperPass>();
-    AU.addRequired<ScalarEvolutionWrapperPass>();
-    AU.addRequired<TargetTransformInfoWrapperPass>();
-    AU.addRequired<AAResultsWrapperPass>();
-    AU.addRequired<LoopAccessAnalysis>();
-    AU.addPreserved<LoopInfoWrapperPass>();
-    AU.addPreserved<DominatorTreeWrapperPass>();
-    AU.addPreserved<BasicAAWrapperPass>();
-    AU.addPreserved<AAResultsWrapperPass>();
-    AU.addPreserved<GlobalsAAWrapperPass>();
+void LoopVectorization::AddRuntimeUnrollDisableMetaData(Loop *L) {
+  SmallVector<Metadata *, 4> MDs;
+  // Reserve first location for self reference to the LoopID metadata node.
+  MDs.push_back(nullptr);
+  bool IsUnrollMetadata = false;
+  MDNode *LoopID = L->getLoopID();
+  if (LoopID) {
+    // First find existing loop unrolling disable metadata.
+    for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
+      MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
+      if (MD) {
+        const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
+        IsUnrollMetadata =
+          S && S->getString().startswith("llvm.loop.unroll.disable");
+      }
+      MDs.push_back(LoopID->getOperand(i));
+    }
   }
 
-};
-
-} // end anonymous namespace
+  if (!IsUnrollMetadata) {
+    // Add runtime unroll disable metadata.
+    LLVMContext &Context = L->getHeader()->getContext();
+    SmallVector<Metadata *, 1> DisableOperands;
+    DisableOperands.push_back(
+      MDString::get(Context, "llvm.loop.unroll.runtime.disable"));
+    MDNode *DisableNode = MDNode::get(Context, DisableOperands);
+    MDs.push_back(DisableNode);
+    MDNode *NewLoopID = MDNode::get(Context, MDs);
+    // Set operand 0 to refer to the loop id itself.
+    NewLoopID->replaceOperandWith(0, NewLoopID);
+    L->setLoopID(NewLoopID);
+  }
+}
 
 //===----------------------------------------------------------------------===//
 // Implementation of LoopVectorizationLegality, InnerLoopVectorizer and
 // LoopVectorizationCostModel.
 //===----------------------------------------------------------------------===//
 
 Value *InnerLoopVectorizer::getBroadcastInstrs(Value *V) {
   // We need to place the broadcast of invariant variables outside the loop.