[PM] Teach the loop PM to run LoopSimplify prior to the loop pipeline.

[chandlerc]

Summary

This adds the last remaining core feature of the loop pass pipeline in
the new PM and removes the last of the really egregious hacks in the
LICM tests.

Sadly, this requires really substantial changes in the unittests in
order to provide and maintain simplified loops. This is particularly
hard because for example LoopSimplify will try to fold undef branches to
an ideal direction and simplify the loop accordingly.

Depends on D28743.

[chandlerc]

Rebase.

[chandlerc]

Rebase.

[mzolotukhin]

The changes look good to me!

Michael

[chandlerc]

Thank so much, landing!

Also, I'll send you a patch with more docs about how the unittest works and we can see if that'll help going forward. I'm hoping to do it as part of factoring the unittest facilities into a library that we can use for other parts of the pass manager as well. Seems a good time to get some good docs in place.

llvm/trunk/include/llvm/Transforms/Scalar/LoopPassManager.h

 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/Transforms/Utils/LCSSA.h"
+#include "llvm/Transforms/Utils/LoopSimplify.h"
 
 namespace llvm {
 
 // Forward declarations of an update tracking API used in the pass manager.
 class LPMUpdater;
 
 // Explicit specialization and instantiation declarations for the pass manager.
 // See the comments on the definition of the specialization for details on how

 /// FunctionAnalysisManager it will run the \c LoopAnalysisManagerFunctionProxy
 /// analysis prior to running the loop passes over the function to enable a \c
 /// LoopAnalysisManager to be used within this run safely.
 template <typename LoopPassT>
 class FunctionToLoopPassAdaptor
     : public PassInfoMixin<FunctionToLoopPassAdaptor<LoopPassT>> {
 public:
   explicit FunctionToLoopPassAdaptor(LoopPassT Pass) : Pass(std::move(Pass)) {
+    LoopCanonicalizationFPM.addPass(LoopSimplifyPass());
     LoopCanonicalizationFPM.addPass(LCSSAPass());
   }
 
   /// \brief Runs the loop passes across every loop in the function.
   PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM) {
     // Before we even compute any loop analyses, first run a miniature function
     // pass pipeline to put loops into their canonical form. Note that we can
     // directly build up function analyses after this as the function pass

llvm/trunk/test/Other/new-pass-manager.ll

 ; RUN:     | FileCheck %s --check-prefix=CHECK-REPEAT-LOOP-PASS
 ; CHECK-REPEAT-LOOP-PASS: Starting llvm::Module pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: ModuleToFunctionPassAdaptor
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}>
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Starting llvm::Function pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: FunctionToLoopPassAdaptor
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: LoopAnalysis
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: DominatorTreeAnalysis
+; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: AssumptionAnalysis
+; CHECK-REPEAT-LOOP-PASS-NEXT: Invalidating all non-preserved analyses
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}>
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: AAManager
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: TargetLibraryAnalysis
-; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: AssumptionAnalysis
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running analysis: TargetIRAnalysis
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Starting Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: RepeatedPass
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Starting Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: NoOpLoopPass
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Starting Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: NoOpLoopPass
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Starting Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Running pass: NoOpLoopPass
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished Loop pass manager run
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished llvm::Function pass manager run
+; CHECK-REPEAT-LOOP-PASS-NEXT: Invalidating all non-preserved analyses
 ; CHECK-REPEAT-LOOP-PASS-NEXT: Finished llvm::Module pass manager run
 
 define void @foo(i1 %x, i8* %p1, i8* %p2) {
 entry:
   store i8 42, i8* %p1
   br i1 %x, label %loop, label %exit
 
 loop:
   %tmp1 = load i8, i8* %p2
   br label %loop
 
 exit:
   ret void
 }
 
 declare void @bar()

llvm/trunk/test/Other/new-pm-defaults.ll

 ; CHECK-O-NEXT: Running analysis: LoopAnalysis
 ; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
 ; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-O-NEXT: Starting Loop pass manager run.
 ; CHECK-O-NEXT: Finished Loop pass manager run.
 ; CHECK-O-NEXT: Running pass: SimplifyCFGPass
 ; CHECK-O-NEXT: Running pass: InstCombinePass
 ; CHECK-O-NEXT: Running pass: FunctionToLoopPassAdaptor<{{.*}}LoopStandardAnalysisResults{{.*}}>
+; CHECK-O-NEXT: Invalidating analysis: ScalarEvolutionAnalysis
+; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
 ; CHECK-O-NEXT: Starting Loop pass manager run.
 ; CHECK-O-NEXT: Finished Loop pass manager run.
 ; CHECK-Os-NEXT: Running pass: MergedLoadStoreMotionPass
 ; CHECK-Os-NEXT: Running pass: GVN
 ; CHECK-Oz-NEXT: Running pass: MergedLoadStoreMotionPass
 ; CHECK-Oz-NEXT: Running pass: GVN
 ; CHECK-O2-NEXT: Running pass: MergedLoadStoreMotionPass
 ; CHECK-O2-NEXT: Running pass: GVN

llvm/trunk/test/Transforms/LICM/hoist-bitcast-load.ll

 ; RUN: opt -S -basicaa -licm < %s | FileCheck %s
-; RUN: opt -aa-pipeline=basic-aa -passes='loop-simplify,require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; Make sure the basic alloca pointer hoisting works:
 ; CHECK-LABEL: @test1
 ; CHECK: load i32, i32* %c, align 4
 ; CHECK: for.body:
 

llvm/trunk/test/Transforms/LICM/hoist-deref-load.ll

 ; RUN: opt -S -basicaa -licm < %s | FileCheck %s
-; RUN: opt -aa-pipeline=basic-aa -passes='loop-simplify,require<aa>,require<targetir>,require<scalar-evolution>,require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
+; RUN: opt -aa-pipeline=basic-aa -passes='require<opt-remark-emit>,loop(simplify-cfg,licm)' -S < %s | FileCheck %s
+
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
 ; This test represents the following function:
 ; void test1(int * __restrict__ a, int * __restrict__ b, int &c, int n) {
 ;   for (int i = 0; i < n; ++i)
 ;     if (a[i] > 0)
 ;       a[i] = c*b[i];

llvm/trunk/unittests/Transforms/Scalar/LoopPassManagerTest.cpp

protected:
   getLoopAnalysisResult(Loop &L, LoopAnalysisManager &AM,
                         LoopStandardAnalysisResults &AR, LPMUpdater &) {
     (void)AM.getResult<MockLoopAnalysisHandle::Analysis>(L, AR);
     return PreservedAnalyses::all();
   };
 
 public:
   LoopPassManagerTest()
-      : M(parseIR(Context, "define void @f() {\n"
-                           "entry:\n"
-                           "  br label %loop.0\n"
-                           "loop.0:\n"
-                           "  br i1 undef, label %loop.0.0, label %end\n"
-                           "loop.0.0:\n"
-                           "  br i1 undef, label %loop.0.0, label %loop.0.1\n"
-                           "loop.0.1:\n"
-                           "  br i1 undef, label %loop.0.1, label %loop.0\n"
-                           "end:\n"
-                           "  ret void\n"
-                           "}\n"
-                           "\n"
-                           "define void @g() {\n"
-                           "entry:\n"
-                           "  br label %loop.g.0\n"
-                           "loop.g.0:\n"
-                           "  br i1 undef, label %loop.g.0, label %end\n"
-                           "end:\n"
-                           "  ret void\n"
-                           "}\n")),
+      : M(parseIR(Context,
+                  "define void @f(i1* %ptr) {\n"
+                  "  %cond.0 = load volatile i1, i1* %ptr\n"
+                  "  br i1 %cond.0, label %loop.0.0.ph, label %end\n"
+                  "loop.0.0.ph:\n"
+                  "  br label %loop.0.0\n"
+                  "  %cond.0.0 = load volatile i1, i1* %ptr\n"
+                  "  br i1 %cond.0.0, label %loop.0.0, label %loop.0.1.ph\n"
+                  "loop.0.1.ph:\n"
+                  "  br label %loop.0.1\n"
+                  "  %cond.0.1 = load volatile i1, i1* %ptr\n"
+                  "  br i1 %cond.0.1, label %loop.0.1, label %loop.0.latch\n"
+                  "loop.0.latch:\n"
+                  "  br label %loop.0\n"
+                  "define void @g(i1* %ptr) {\n"
+                  "  %cond.0 = load volatile i1, i1* %ptr\n"
+                  "  br i1 %cond.0, label %loop.g.0, label %end\n"
         LAM(true), FAM(true), MAM(true) {
     // Register our mock analysis.
     LAM.registerPass([&] { return MLAHandle.getAnalysis(); });
 
     // We need DominatorTreeAnalysis for LoopAnalysis.
     FAM.registerPass([&] { return DominatorTreeAnalysis(); });
     FAM.registerPass([&] { return LoopAnalysis(); });
     // We also allow loop passes to assume a set of other analyses and so need
TEST_F(LoopPassManagerTest, IndirectOuterPassInvalidation) {
   FPM.addPass(
       createFunctionToLoopPassAdaptor(RequireAnalysisLoopPass<LoopAnalysis>()));
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
   MPM.run(*M, MAM);
 }
 
 TEST_F(LoopPassManagerTest, LoopChildInsertion) {
   // Super boring module with three loops in a single loop nest.
-  M = parseIR(Context, "define void @f() {\n"
+  M = parseIR(Context, "define void @f(i1* %ptr) {\n"
                        "entry:\n"
                        "  br label %loop.0\n"
                        "loop.0:\n"
-                       "  br i1 undef, label %loop.0.0, label %end\n"
+                       "  %cond.0 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0, label %loop.0.0.ph, label %end\n"
+                       "loop.0.0.ph:\n"
+                       "  br label %loop.0.0\n"
                        "loop.0.0:\n"
-                       "  br i1 undef, label %loop.0.0, label %loop.0.1\n"
+                       "  %cond.0.0 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0.0, label %loop.0.0, label %loop.0.1.ph\n"
+                       "loop.0.1.ph:\n"
+                       "  br label %loop.0.1\n"
                        "loop.0.1:\n"
-                       "  br i1 undef, label %loop.0.1, label %loop.0.2\n"
+                       "  %cond.0.1 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0.1, label %loop.0.1, label %loop.0.2.ph\n"
+                       "loop.0.2.ph:\n"
+                       "  br label %loop.0.2\n"
                        "loop.0.2:\n"
-                       "  br i1 undef, label %loop.0.2, label %loop.0\n"
+                       "  %cond.0.2 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0.2, label %loop.0.2, label %loop.0.latch\n"
+                       "loop.0.latch:\n"
+                       "  br label %loop.0\n"
                        "end:\n"
                        "  ret void\n"
                        "}\n");
 
   // Build up variables referring into the IR so we can rewrite it below
   // easily.
   Function &F = *M->begin();
   ASSERT_THAT(F, HasName("f"));
+  Argument &Ptr = *F.arg_begin();
   auto BBI = F.begin();
   BasicBlock &EntryBB = *BBI++;
   ASSERT_THAT(EntryBB, HasName("entry"));
   BasicBlock &Loop0BB = *BBI++;
   ASSERT_THAT(Loop0BB, HasName("loop.0"));
+  BasicBlock &Loop00PHBB = *BBI++;
+  ASSERT_THAT(Loop00PHBB, HasName("loop.0.0.ph"));
   BasicBlock &Loop00BB = *BBI++;
   ASSERT_THAT(Loop00BB, HasName("loop.0.0"));
+  BasicBlock &Loop01PHBB = *BBI++;
+  ASSERT_THAT(Loop01PHBB, HasName("loop.0.1.ph"));
   BasicBlock &Loop01BB = *BBI++;
   ASSERT_THAT(Loop01BB, HasName("loop.0.1"));
+  BasicBlock &Loop02PHBB = *BBI++;
+  ASSERT_THAT(Loop02PHBB, HasName("loop.0.2.ph"));
   BasicBlock &Loop02BB = *BBI++;
   ASSERT_THAT(Loop02BB, HasName("loop.0.2"));
+  BasicBlock &Loop0LatchBB = *BBI++;
+  ASSERT_THAT(Loop0LatchBB, HasName("loop.0.latch"));
   BasicBlock &EndBB = *BBI++;
   ASSERT_THAT(EndBB, HasName("end"));
   ASSERT_THAT(BBI, F.end());
+  auto CreateCondBr = [&](BasicBlock *TrueBB, BasicBlock *FalseBB,
+                          const char *Name, BasicBlock *BB) {
+    auto *Cond = new LoadInst(&Ptr, Name, /*isVolatile*/ true, BB);
+    BranchInst::Create(TrueBB, FalseBB, Cond, BB);
+  };
 
   // Build the pass managers and register our pipeline. We build a single loop
   // pass pipeline consisting of three mock pass runs over each loop. After
   // this we run both domtree and loop verification passes to make sure that
   // the IR remained valid during our mutations.
   ModulePassManager MPM(true);
   FunctionPassManager FPM(true);
   LoopPassManager LPM(true);
EXPECT_CALL(MLPHandle, run(HasName("loop.0.0"), _, _, _))
       .WillRepeatedly(Invoke(getLoopAnalysisResult));
 
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.1"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
   EXPECT_CALL(MLAHandle, run(HasName("loop.0.1"), _, _));
 
   // When running over the middle loop, the second run inserts two new child
   // loops, inserting them and itself into the worklist.
-  BasicBlock *NewLoop010BB;
+  BasicBlock *NewLoop010BB, *NewLoop01LatchBB;
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.1"), _, _, _))
       .WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
                            LoopStandardAnalysisResults &AR,
                            LPMUpdater &Updater) {
         auto *NewLoop = new Loop();
         L.addChildLoop(NewLoop);
-        NewLoop010BB = BasicBlock::Create(Context, "loop.0.1.0", &F, &Loop02BB);
-        BranchInst::Create(&Loop01BB, NewLoop010BB,
-                           UndefValue::get(Type::getInt1Ty(Context)),
-                           NewLoop010BB);
-        Loop01BB.getTerminator()->replaceUsesOfWith(&Loop01BB, NewLoop010BB);
-        AR.DT.addNewBlock(NewLoop010BB, &Loop01BB);
+        auto *NewLoop010PHBB =
+            BasicBlock::Create(Context, "loop.0.1.0.ph", &F, &Loop02PHBB);
+        NewLoop010BB =
+            BasicBlock::Create(Context, "loop.0.1.0", &F, &Loop02PHBB);
+        NewLoop01LatchBB =
+            BasicBlock::Create(Context, "loop.0.1.latch", &F, &Loop02PHBB);
+        Loop01BB.getTerminator()->replaceUsesOfWith(&Loop01BB, NewLoop010PHBB);
+        BranchInst::Create(NewLoop010BB, NewLoop010PHBB);
+        CreateCondBr(NewLoop01LatchBB, NewLoop010BB, "cond.0.1.0",
+        BranchInst::Create(&Loop01BB, NewLoop01LatchBB);
+        AR.DT.addNewBlock(NewLoop010PHBB, &Loop01BB);
+        AR.DT.addNewBlock(NewLoop010BB, NewLoop010PHBB);
+        AR.DT.addNewBlock(NewLoop01LatchBB, NewLoop010BB);
+        AR.DT.verifyDomTree();
+        L.addBasicBlockToLoop(NewLoop010PHBB, AR.LI);
         NewLoop->addBasicBlockToLoop(NewLoop010BB, AR.LI);
+        L.addBasicBlockToLoop(NewLoop01LatchBB, AR.LI);
+        NewLoop->verifyLoop();
+        L.verifyLoop();
         Updater.addChildLoops({NewLoop});
         return PreservedAnalyses::all();
       }));
 
   // We should immediately drop down to fully visit the new inner loop.
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.1.0"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
   EXPECT_CALL(MLAHandle, run(HasName("loop.0.1.0"), _, _));
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.1.0"), _, _, _))
       .Times(2)
       .WillRepeatedly(Invoke(getLoopAnalysisResult));
 
   // After visiting the inner loop, we should re-visit the second loop
   // reflecting its new loop nest structure.
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.1"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
 
   // In the second run over the middle loop after we've visited the new child,
   // we add another child to check that we can repeatedly add children, and add
   // children to a loop that already has children.
-  BasicBlock *NewLoop011BB;
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.1"), _, _, _))
       .WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
                            LoopStandardAnalysisResults &AR,
                            LPMUpdater &Updater) {
         auto *NewLoop = new Loop();
         L.addChildLoop(NewLoop);
-        NewLoop011BB = BasicBlock::Create(Context, "loop.0.1.1", &F, &Loop02BB);
-        BranchInst::Create(&Loop01BB, NewLoop011BB,
-                           UndefValue::get(Type::getInt1Ty(Context)),
-                           NewLoop011BB);
-        NewLoop010BB->getTerminator()->replaceUsesOfWith(&Loop01BB,
-                                                         NewLoop011BB);
-        AR.DT.addNewBlock(NewLoop011BB, NewLoop010BB);
+        auto *NewLoop011PHBB = BasicBlock::Create(Context, "loop.0.1.1.ph", &F, NewLoop01LatchBB);
+        auto *NewLoop011BB = BasicBlock::Create(Context, "loop.0.1.1", &F, NewLoop01LatchBB);
+        NewLoop010BB->getTerminator()->replaceUsesOfWith(NewLoop01LatchBB,
+                                                         NewLoop011PHBB);
+        BranchInst::Create(NewLoop011BB, NewLoop011PHBB);
+        CreateCondBr(NewLoop01LatchBB, NewLoop011BB, "cond.0.1.1",
+        AR.DT.addNewBlock(NewLoop011PHBB, NewLoop010BB);
+        auto *NewDTNode = AR.DT.addNewBlock(NewLoop011BB, NewLoop011PHBB);
+        AR.DT.changeImmediateDominator(AR.DT[NewLoop01LatchBB], NewDTNode);
+        AR.DT.verifyDomTree();
+        L.addBasicBlockToLoop(NewLoop011PHBB, AR.LI);
         NewLoop->addBasicBlockToLoop(NewLoop011BB, AR.LI);
+        NewLoop->verifyLoop();
+        L.verifyLoop();
         Updater.addChildLoops({NewLoop});
         return PreservedAnalyses::all();
       }));
 
   // Again, we should immediately drop down to visit the new, unvisited child
   // loop. We don't need to revisit the other child though.
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.1.1"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
TEST_F(LoopPassManagerTest, LoopChildInsertion) {
   // Now that all the expected actions are registered, run the pipeline over
   // our module. All of our expectations are verified when the test finishes.
   MPM.run(*M, MAM);
 }
 
 TEST_F(LoopPassManagerTest, LoopPeerInsertion) {
   // Super boring module with two loop nests and loop nest with two child
   // loops.
-  M = parseIR(Context, "define void @f() {\n"
+  M = parseIR(Context, "define void @f(i1* %ptr) {\n"
                        "entry:\n"
                        "  br label %loop.0\n"
                        "loop.0:\n"
-                       "  br i1 undef, label %loop.0.0, label %loop.2\n"
+                       "  %cond.0 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0, label %loop.0.0.ph, label %loop.2.ph\n"
+                       "loop.0.0.ph:\n"
+                       "  br label %loop.0.0\n"
                        "loop.0.0:\n"
-                       "  br i1 undef, label %loop.0.0, label %loop.0.2\n"
+                       "  %cond.0.0 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0.0, label %loop.0.0, label %loop.0.2.ph\n"
+                       "loop.0.2.ph:\n"
+                       "  br label %loop.0.2\n"
                        "loop.0.2:\n"
-                       "  br i1 undef, label %loop.0.2, label %loop.0\n"
+                       "  %cond.0.2 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0.2, label %loop.0.2, label %loop.0.latch\n"
+                       "loop.0.latch:\n"
+                       "  br label %loop.0\n"
+                       "loop.2.ph:\n"
+                       "  br label %loop.2\n"
                        "loop.2:\n"
-                       "  br i1 undef, label %loop.2, label %end\n"
+                       "  %cond.2 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.2, label %loop.2, label %end\n"
                        "end:\n"
                        "  ret void\n"
                        "}\n");
 
   // Build up variables referring into the IR so we can rewrite it below
   // easily.
   Function &F = *M->begin();
   ASSERT_THAT(F, HasName("f"));
+  Argument &Ptr = *F.arg_begin();
   auto BBI = F.begin();
   BasicBlock &EntryBB = *BBI++;
   ASSERT_THAT(EntryBB, HasName("entry"));
   BasicBlock &Loop0BB = *BBI++;
   ASSERT_THAT(Loop0BB, HasName("loop.0"));
+  BasicBlock &Loop00PHBB = *BBI++;
+  ASSERT_THAT(Loop00PHBB, HasName("loop.0.0.ph"));
   BasicBlock &Loop00BB = *BBI++;
   ASSERT_THAT(Loop00BB, HasName("loop.0.0"));
+  BasicBlock &Loop02PHBB = *BBI++;
+  ASSERT_THAT(Loop02PHBB, HasName("loop.0.2.ph"));
   BasicBlock &Loop02BB = *BBI++;
   ASSERT_THAT(Loop02BB, HasName("loop.0.2"));
+  BasicBlock &Loop0LatchBB = *BBI++;
+  ASSERT_THAT(Loop0LatchBB, HasName("loop.0.latch"));
+  BasicBlock &Loop2PHBB = *BBI++;
+  ASSERT_THAT(Loop2PHBB, HasName("loop.2.ph"));
   BasicBlock &Loop2BB = *BBI++;
   ASSERT_THAT(Loop2BB, HasName("loop.2"));
   BasicBlock &EndBB = *BBI++;
   ASSERT_THAT(EndBB, HasName("end"));
   ASSERT_THAT(BBI, F.end());
-  Constant *Undefi1 = UndefValue::get(Type::getInt1Ty(Context));
+  auto CreateCondBr = [&](BasicBlock *TrueBB, BasicBlock *FalseBB,
+                          const char *Name, BasicBlock *BB) {
+    auto *Cond = new LoadInst(&Ptr, Name, /*isVolatile*/ true, BB);
+    BranchInst::Create(TrueBB, FalseBB, Cond, BB);
+  };
 
   // Build the pass managers and register our pipeline. We build a single loop
   // pass pipeline consisting of three mock pass runs over each loop. After
   // this we run both domtree and loop verification passes to make sure that
   // the IR remained valid during our mutations.
   ModulePassManager MPM(true);
   FunctionPassManager FPM(true);
   LoopPassManager LPM(true);
TEST_F(LoopPassManagerTest, LoopPeerInsertion) {
   ::testing::InSequence MakeExpectationsSequenced;
 
   // We run loop passes three times over each of the loops.
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.0"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
   EXPECT_CALL(MLAHandle, run(HasName("loop.0.0"), _, _));
 
   // On the second run, we insert a sibling loop.
-  BasicBlock *NewLoop01BB;
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.0"), _, _, _))
       .WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
                            LoopStandardAnalysisResults &AR,
                            LPMUpdater &Updater) {
         auto *NewLoop = new Loop();
         L.getParentLoop()->addChildLoop(NewLoop);
-        NewLoop01BB = BasicBlock::Create(Context, "loop.0.1", &F, &Loop02BB);
-        BranchInst::Create(&Loop02BB, NewLoop01BB, Undefi1, NewLoop01BB);
-        Loop00BB.getTerminator()->replaceUsesOfWith(&Loop02BB, NewLoop01BB);
-        auto *NewDTNode = AR.DT.addNewBlock(NewLoop01BB, &Loop00BB);
-        AR.DT.changeImmediateDominator(AR.DT[&Loop02BB], NewDTNode);
+        auto *NewLoop01PHBB = BasicBlock::Create(Context, "loop.0.1.ph", &F, &Loop02PHBB);
+        auto *NewLoop01BB = BasicBlock::Create(Context, "loop.0.1", &F, &Loop02PHBB);
+        BranchInst::Create(NewLoop01BB, NewLoop01PHBB);
+        CreateCondBr(&Loop02PHBB, NewLoop01BB, "cond.0.1", NewLoop01BB);
+        Loop00BB.getTerminator()->replaceUsesOfWith(&Loop02PHBB, NewLoop01PHBB);
+        AR.DT.addNewBlock(NewLoop01PHBB, &Loop00BB);
+        auto *NewDTNode = AR.DT.addNewBlock(NewLoop01BB, NewLoop01PHBB);
+        AR.DT.changeImmediateDominator(AR.DT[&Loop02PHBB], NewDTNode);
+        AR.DT.verifyDomTree();
+        L.getParentLoop()->addBasicBlockToLoop(NewLoop01PHBB, AR.LI);
         NewLoop->addBasicBlockToLoop(NewLoop01BB, AR.LI);
+        L.getParentLoop()->verifyLoop();
         Updater.addSiblingLoops({NewLoop});
         return PreservedAnalyses::all();
       }));
   // We finish processing this loop as sibling loops don't perturb the
   // postorder walk.
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.0"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
 
TEST_F(LoopPassManagerTest, LoopPeerInsertion) {
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.2"), _, _, _))
       .WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
                            LoopStandardAnalysisResults &AR,
                            LPMUpdater &Updater) {
         Loop *NewLoops[] = {new Loop(), new Loop(), new Loop()};
         L.getParentLoop()->addChildLoop(NewLoops[0]);
         L.getParentLoop()->addChildLoop(NewLoops[1]);
         NewLoops[1]->addChildLoop(NewLoops[2]);
+        auto *NewLoop03PHBB =
+            BasicBlock::Create(Context, "loop.0.3.ph", &F, &Loop0LatchBB);
         auto *NewLoop03BB =
-            BasicBlock::Create(Context, "loop.0.3", &F, &Loop2BB);
+            BasicBlock::Create(Context, "loop.0.3", &F, &Loop0LatchBB);
+        auto *NewLoop04PHBB =
+            BasicBlock::Create(Context, "loop.0.4.ph", &F, &Loop0LatchBB);
         auto *NewLoop04BB =
-            BasicBlock::Create(Context, "loop.0.4", &F, &Loop2BB);
+            BasicBlock::Create(Context, "loop.0.4", &F, &Loop0LatchBB);
+        auto *NewLoop040PHBB =
+            BasicBlock::Create(Context, "loop.0.4.0.ph", &F, &Loop0LatchBB);
         auto *NewLoop040BB =
-            BasicBlock::Create(Context, "loop.0.4.0", &F, &Loop2BB);
-        Loop02BB.getTerminator()->replaceUsesOfWith(&Loop0BB, NewLoop03BB);
-        BranchInst::Create(NewLoop04BB, NewLoop03BB, Undefi1, NewLoop03BB);
-        BranchInst::Create(&Loop0BB, NewLoop040BB, Undefi1, NewLoop04BB);
-        BranchInst::Create(NewLoop04BB, NewLoop040BB, Undefi1, NewLoop040BB);
-        AR.DT.addNewBlock(NewLoop03BB, &Loop02BB);
-        AR.DT.addNewBlock(NewLoop04BB, NewLoop03BB);
-        AR.DT.addNewBlock(NewLoop040BB, NewLoop04BB);
+            BasicBlock::Create(Context, "loop.0.4.0", &F, &Loop0LatchBB);
+        auto *NewLoop04LatchBB =
+            BasicBlock::Create(Context, "loop.0.4.latch", &F, &Loop0LatchBB);
+        Loop02BB.getTerminator()->replaceUsesOfWith(&Loop0LatchBB, NewLoop03PHBB);
+        BranchInst::Create(NewLoop03BB, NewLoop03PHBB);
+        CreateCondBr(NewLoop04PHBB, NewLoop03BB, "cond.0.3", NewLoop03BB);
+        BranchInst::Create(NewLoop04BB, NewLoop04PHBB);
+        CreateCondBr(&Loop0LatchBB, NewLoop040PHBB, "cond.0.4", NewLoop04BB);
+        BranchInst::Create(NewLoop040BB, NewLoop040PHBB);
+        CreateCondBr(NewLoop04LatchBB, NewLoop040BB, "cond.0.4.0", NewLoop040BB);
+        BranchInst::Create(NewLoop04BB, NewLoop04LatchBB);
+        AR.DT.addNewBlock(NewLoop03PHBB, &Loop02BB);
+        AR.DT.addNewBlock(NewLoop03BB, NewLoop03PHBB);
+        AR.DT.addNewBlock(NewLoop04PHBB, NewLoop03BB);
+        auto *NewDTNode = AR.DT.addNewBlock(NewLoop04BB, NewLoop04PHBB);
+        AR.DT.changeImmediateDominator(AR.DT[&Loop0LatchBB], NewDTNode);
+        AR.DT.addNewBlock(NewLoop040PHBB, NewLoop04BB);
+        AR.DT.addNewBlock(NewLoop040BB, NewLoop040PHBB);
+        AR.DT.addNewBlock(NewLoop04LatchBB, NewLoop040BB);
+        AR.DT.verifyDomTree();
+        L.getParentLoop()->addBasicBlockToLoop(NewLoop03PHBB, AR.LI);
         NewLoops[0]->addBasicBlockToLoop(NewLoop03BB, AR.LI);
+        L.getParentLoop()->addBasicBlockToLoop(NewLoop04PHBB, AR.LI);
         NewLoops[1]->addBasicBlockToLoop(NewLoop04BB, AR.LI);
+        NewLoops[1]->addBasicBlockToLoop(NewLoop040PHBB, AR.LI);
         NewLoops[2]->addBasicBlockToLoop(NewLoop040BB, AR.LI);
+        NewLoops[1]->addBasicBlockToLoop(NewLoop04LatchBB, AR.LI);
+        L.getParentLoop()->verifyLoop();
         Updater.addSiblingLoops({NewLoops[0], NewLoops[1]});
         return PreservedAnalyses::all();
       }));
 
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.3"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
   EXPECT_CALL(MLAHandle, run(HasName("loop.0.3"), _, _));
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.3"), _, _, _))
TEST_F(LoopPassManagerTest, LoopPeerInsertion) {
   EXPECT_CALL(MLAHandle, run(HasName("loop.0"), _, _));
   // On the second pass, we add sibling loops which become new top-level loops.
   EXPECT_CALL(MLPHandle, run(HasName("loop.0"), _, _, _))
       .WillOnce(Invoke([&](Loop &L, LoopAnalysisManager &AM,
                            LoopStandardAnalysisResults &AR,
                            LPMUpdater &Updater) {
         auto *NewLoop = new Loop();
         AR.LI.addTopLevelLoop(NewLoop);
+        auto *NewLoop1PHBB = BasicBlock::Create(Context, "loop.1.ph", &F, &Loop2BB);
         auto *NewLoop1BB = BasicBlock::Create(Context, "loop.1", &F, &Loop2BB);
-        BranchInst::Create(&Loop2BB, NewLoop1BB, Undefi1, NewLoop1BB);
-        Loop0BB.getTerminator()->replaceUsesOfWith(&Loop2BB, NewLoop1BB);
-        auto *NewDTNode = AR.DT.addNewBlock(NewLoop1BB, &Loop0BB);
-        AR.DT.changeImmediateDominator(AR.DT[&Loop2BB], NewDTNode);
+        BranchInst::Create(NewLoop1BB, NewLoop1PHBB);
+        CreateCondBr(&Loop2PHBB, NewLoop1BB, "cond.1", NewLoop1BB);
+        Loop0BB.getTerminator()->replaceUsesOfWith(&Loop2PHBB, NewLoop1PHBB);
+        AR.DT.addNewBlock(NewLoop1PHBB, &Loop0BB);
+        auto *NewDTNode = AR.DT.addNewBlock(NewLoop1BB, NewLoop1PHBB);
+        AR.DT.changeImmediateDominator(AR.DT[&Loop2PHBB], NewDTNode);
+        AR.DT.verifyDomTree();
         NewLoop->addBasicBlockToLoop(NewLoop1BB, AR.LI);
+        NewLoop->verifyLoop();
         Updater.addSiblingLoops({NewLoop});
         return PreservedAnalyses::all();
       }));
   EXPECT_CALL(MLPHandle, run(HasName("loop.0"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
 
   EXPECT_CALL(MLPHandle, run(HasName("loop.1"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
TEST_F(LoopPassManagerTest, LoopPeerInsertion) {
   // our module. All of our expectations are verified when the test finishes.
   MPM.run(*M, MAM);
 }
 
 TEST_F(LoopPassManagerTest, LoopDeletion) {
   // Build a module with a single loop nest that contains one outer loop with
   // three subloops, and one of those with its own subloop. We will
   // incrementally delete all of these to test different deletion scenarios.
-  M = parseIR(Context, "define void @f() {\n"
+  M = parseIR(Context, "define void @f(i1* %ptr) {\n"
                        "entry:\n"
                        "  br label %loop.0\n"
                        "loop.0:\n"
-                       "  br i1 undef, label %loop.0.0, label %end\n"
+                       "  %cond.0 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0, label %loop.0.0.ph, label %end\n"
+                       "loop.0.0.ph:\n"
+                       "  br label %loop.0.0\n"
                        "loop.0.0:\n"
-                       "  br i1 undef, label %loop.0.0, label %loop.0.1\n"
+                       "  %cond.0.0 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0.0, label %loop.0.0, label %loop.0.1.ph\n"
+                       "loop.0.1.ph:\n"
+                       "  br label %loop.0.1\n"
                        "loop.0.1:\n"
-                       "  br i1 undef, label %loop.0.1, label %loop.0.2\n"
+                       "  %cond.0.1 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0.1, label %loop.0.1, label %loop.0.2.ph\n"
+                       "loop.0.2.ph:\n"
+                       "  br label %loop.0.2\n"
                        "loop.0.2:\n"
-                       "  br i1 undef, label %loop.0.2.0, label %loop.0\n"
+                       "  %cond.0.2 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0.2, label %loop.0.2.0.ph, label %loop.0.latch\n"
+                       "loop.0.2.0.ph:\n"
+                       "  br label %loop.0.2.0\n"
                        "loop.0.2.0:\n"
-                       "  br i1 undef, label %loop.0.2.0, label %loop.0.2\n"
+                       "  %cond.0.2.0 = load volatile i1, i1* %ptr\n"
+                       "  br i1 %cond.0.2.0, label %loop.0.2.0, label %loop.0.2.latch\n"
+                       "loop.0.2.latch:\n"
+                       "  br label %loop.0.2\n"
+                       "loop.0.latch:\n"
+                       "  br label %loop.0\n"
                        "end:\n"
                        "  ret void\n"
                        "}\n");
 
   // Build up variables referring into the IR so we can rewrite it below
   // easily.
   Function &F = *M->begin();
   ASSERT_THAT(F, HasName("f"));
+  Argument &Ptr = *F.arg_begin();
   auto BBI = F.begin();
   BasicBlock &EntryBB = *BBI++;
   ASSERT_THAT(EntryBB, HasName("entry"));
   BasicBlock &Loop0BB = *BBI++;
   ASSERT_THAT(Loop0BB, HasName("loop.0"));
+  BasicBlock &Loop00PHBB = *BBI++;
+  ASSERT_THAT(Loop00PHBB, HasName("loop.0.0.ph"));
   BasicBlock &Loop00BB = *BBI++;
   ASSERT_THAT(Loop00BB, HasName("loop.0.0"));
+  BasicBlock &Loop01PHBB = *BBI++;
+  ASSERT_THAT(Loop01PHBB, HasName("loop.0.1.ph"));
   BasicBlock &Loop01BB = *BBI++;
   ASSERT_THAT(Loop01BB, HasName("loop.0.1"));
+  BasicBlock &Loop02PHBB = *BBI++;
+  ASSERT_THAT(Loop02PHBB, HasName("loop.0.2.ph"));
   BasicBlock &Loop02BB = *BBI++;
   ASSERT_THAT(Loop02BB, HasName("loop.0.2"));
+  BasicBlock &Loop020PHBB = *BBI++;
+  ASSERT_THAT(Loop020PHBB, HasName("loop.0.2.0.ph"));
   BasicBlock &Loop020BB = *BBI++;
   ASSERT_THAT(Loop020BB, HasName("loop.0.2.0"));
+  BasicBlock &Loop02LatchBB = *BBI++;
+  ASSERT_THAT(Loop02LatchBB, HasName("loop.0.2.latch"));
+  BasicBlock &Loop0LatchBB = *BBI++;
+  ASSERT_THAT(Loop0LatchBB, HasName("loop.0.latch"));
   BasicBlock &EndBB = *BBI++;
   ASSERT_THAT(EndBB, HasName("end"));
   ASSERT_THAT(BBI, F.end());
-  Constant *Undefi1 = UndefValue::get(Type::getInt1Ty(Context));
 
   // Helper to do the actual deletion of a loop. We directly encode this here
   // to isolate ourselves from the rest of LLVM and for simplicity. Here we can
   // egregiously cheat based on knowledge of the test case. For example, we
   // have no PHI nodes and there is always a single i-dom.
-  auto DeleteLoopBlocks = [](Loop &L, BasicBlock &IDomBB,
+  auto RemoveLoop = [](Loop &L, BasicBlock &IDomBB,
                              LoopStandardAnalysisResults &AR,
                              LPMUpdater &Updater) {
-    for (BasicBlock *LoopBB : L.blocks()) {
+    assert(L.empty() && "Can only delete leaf loops with this routine!");
+    SmallVector<BasicBlock *, 4> LoopBBs(L.block_begin(), L.block_end());
+    Updater.markLoopAsDeleted(L);
+    IDomBB.getTerminator()->replaceUsesOfWith(L.getHeader(),
+                                              L.getUniqueExitBlock());
+    for (BasicBlock *LoopBB : LoopBBs) {
       SmallVector<DomTreeNode *, 4> ChildNodes(AR.DT[LoopBB]->begin(),
                                                AR.DT[LoopBB]->end());
       for (DomTreeNode *ChildNode : ChildNodes)
         AR.DT.changeImmediateDominator(ChildNode, AR.DT[&IDomBB]);
       AR.DT.eraseNode(LoopBB);
+      AR.LI.removeBlock(LoopBB);
       LoopBB->dropAllReferences();
     }
-    SmallVector<BasicBlock *, 4> LoopBBs(L.block_begin(), L.block_end());
-    Updater.markLoopAsDeleted(L);
-    AR.LI.markAsRemoved(&L);
     for (BasicBlock *LoopBB : LoopBBs)
       LoopBB->eraseFromParent();
+
+    if (Loop *ParentL = L.getParentLoop())
+      return ParentL->removeChildLoop(find(*ParentL, &L));
+
+    return AR.LI.removeLoop(find(AR.LI, &L));
   };
 
   // Build up the pass managers.
   ModulePassManager MPM(true);
   FunctionPassManager FPM(true);
   // We run several loop pass pipelines across the loop nest, but they all take
   // the same form of three mock pass runs in a loop pipeline followed by
   // domtree and loop verification. We use a lambda to stamp this out each
TEST_F(LoopPassManagerTest, LoopDeletion) {
 
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.1"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
   EXPECT_CALL(MLAHandle, run(HasName("loop.0.1"), _, _));
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.1"), _, _, _))
       .WillOnce(
           Invoke([&](Loop &L, LoopAnalysisManager &AM,
                      LoopStandardAnalysisResults &AR, LPMUpdater &Updater) {
+            Loop *ParentL = L.getParentLoop();
             AR.SE.forgetLoop(&L);
-            Loop00BB.getTerminator()->replaceUsesOfWith(&Loop01BB, &Loop02BB);
-            DeleteLoopBlocks(L, Loop00BB, AR, Updater);
+            delete RemoveLoop(L, Loop01PHBB, AR, Updater);
+            ParentL->verifyLoop();
             return PreservedAnalyses::all();
           }));
 
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.2.0"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
   EXPECT_CALL(MLAHandle, run(HasName("loop.0.2.0"), _, _));
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.2.0"), _, _, _))
       .Times(2)
EXPECT_CALL(MLPHandle, run(HasName("loop.0.0"), _, _, _))
       .WillRepeatedly(Invoke(getLoopAnalysisResult));
 
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.2.0"), _, _, _))
       .Times(3)
       .WillRepeatedly(Invoke(getLoopAnalysisResult));
 
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.2"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
-  BasicBlock *NewLoop03BB;
+  BasicBlock *NewLoop03PHBB;
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.2"), _, _, _))
       .WillOnce(
           Invoke([&](Loop &L, LoopAnalysisManager &AM,
                      LoopStandardAnalysisResults &AR, LPMUpdater &Updater) {
-            // Delete the inner loop first. we also do this manually because we
-            // want to preserve the loop object and reuse it.
+            // Remove the inner loop first but retain it to reuse later.
             AR.SE.forgetLoop(*L.begin());
-            Loop02BB.getTerminator()->replaceUsesOfWith(&Loop020BB, &Loop02BB);
-            assert(std::next((*L.begin())->block_begin()) ==
-                       (*L.begin())->block_end() &&
-                   "There should only be one block.");
-            assert(AR.DT[&Loop020BB]->getNumChildren() == 0 &&
-                   "Cannot have children in the domtree!");
-            AR.DT.eraseNode(&Loop020BB);
-            Updater.markLoopAsDeleted(**L.begin());
-            AR.LI.removeBlock(&Loop020BB);
-            auto *OldL = L.removeChildLoop(L.begin());
-            Loop020BB.eraseFromParent();
+            auto *OldL = RemoveLoop(**L.begin(), Loop020PHBB, AR, Updater);
 
             auto *ParentL = L.getParentLoop();
             AR.SE.forgetLoop(&L);
-            Loop00BB.getTerminator()->replaceUsesOfWith(&Loop02BB, &Loop0BB);
-            DeleteLoopBlocks(L, Loop00BB, AR, Updater);
+            delete RemoveLoop(L, Loop02PHBB, AR, Updater);
 
             // Now insert a new sibling loop, reusing a loop pointer.
             ParentL->addChildLoop(OldL);
-            NewLoop03BB = BasicBlock::Create(Context, "loop.0.3", &F, &EndBB);
-            BranchInst::Create(&Loop0BB, NewLoop03BB, Undefi1, NewLoop03BB);
-            Loop00BB.getTerminator()->replaceUsesOfWith(&Loop0BB, NewLoop03BB);
-            AR.DT.addNewBlock(NewLoop03BB, &Loop00BB);
+            NewLoop03PHBB =
+                BasicBlock::Create(Context, "loop.0.3.ph", &F, &Loop0LatchBB);
+            auto *NewLoop03BB =
+                BasicBlock::Create(Context, "loop.0.3", &F, &Loop0LatchBB);
+            BranchInst::Create(NewLoop03BB, NewLoop03PHBB);
+            auto *Cond = new LoadInst(&Ptr, "cond.0.3", /*isVolatile*/ true,
+                                      NewLoop03BB);
+            BranchInst::Create(&Loop0LatchBB, NewLoop03BB, Cond, NewLoop03BB);
+            Loop02PHBB.getTerminator()->replaceUsesOfWith(&Loop0LatchBB,
+                                                          NewLoop03PHBB);
+            AR.DT.addNewBlock(NewLoop03PHBB, &Loop02PHBB);
+            AR.DT.addNewBlock(NewLoop03BB, NewLoop03PHBB);
+            AR.DT.changeImmediateDominator(AR.DT[&Loop0LatchBB],
+                                           AR.DT[NewLoop03BB]);
+            AR.DT.verifyDomTree();
+            ParentL->addBasicBlockToLoop(NewLoop03PHBB, AR.LI);
             OldL->addBasicBlockToLoop(NewLoop03BB, AR.LI);
+            OldL->verifyLoop();
+            ParentL->verifyLoop();
             Updater.addSiblingLoops({OldL});
             return PreservedAnalyses::all();
           }));
 
   // To respect our inner-to-outer traversal order, we must visit the
   // newly-inserted sibling of the loop we just deleted before we visit the
   // outer loop. When we do so, this must compute a fresh analysis result, even
   // though our new loop has the same pointer value as the loop we deleted.
TEST_F(LoopPassManagerTest, LoopDeletion) {
   AddLoopPipelineAndVerificationPasses();
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.0"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.0"), _, _, _))
       .WillOnce(
           Invoke([&](Loop &L, LoopAnalysisManager &AM,
                      LoopStandardAnalysisResults &AR, LPMUpdater &Updater) {
             AR.SE.forgetLoop(&L);
-            Loop0BB.getTerminator()->replaceUsesOfWith(&Loop00BB, NewLoop03BB);
-            DeleteLoopBlocks(L, Loop0BB, AR, Updater);
+            delete RemoveLoop(L, Loop00PHBB, AR, Updater);
             return PreservedAnalyses::all();
           }));
 
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.3"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
   EXPECT_CALL(MLPHandle, run(HasName("loop.0.3"), _, _, _))
       .WillOnce(
           Invoke([&](Loop &L, LoopAnalysisManager &AM,
                      LoopStandardAnalysisResults &AR, LPMUpdater &Updater) {
             AR.SE.forgetLoop(&L);
-            Loop0BB.getTerminator()->replaceUsesOfWith(NewLoop03BB, &Loop0BB);
-            DeleteLoopBlocks(L, Loop0BB, AR, Updater);
+            delete RemoveLoop(L, *NewLoop03PHBB, AR, Updater);
             return PreservedAnalyses::all();
           }));
 
   EXPECT_CALL(MLPHandle, run(HasName("loop.0"), _, _, _))
       .WillOnce(Invoke(getLoopAnalysisResult));
   EXPECT_CALL(MLPHandle, run(HasName("loop.0"), _, _, _))
       .WillOnce(
           Invoke([&](Loop &L, LoopAnalysisManager &AM,
                      LoopStandardAnalysisResults &AR, LPMUpdater &Updater) {
             AR.SE.forgetLoop(&L);
-            EntryBB.getTerminator()->replaceUsesOfWith(&Loop0BB, &EndBB);
-            DeleteLoopBlocks(L, EntryBB, AR, Updater);
+            delete RemoveLoop(L, EntryBB, AR, Updater);
             return PreservedAnalyses::all();
           }));
 
   // Add the function pass pipeline now that it is fully built up and run it
   // over the module's one function.
   MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
   MPM.run(*M, MAM);
 }
 }