diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -1114,6 +1114,7 @@ FullyAvailableBlocks[UnavailableBB] = false; SmallVector CriticalEdgePred; + for (BasicBlock *Pred : predecessors(LoadBB)) { // If any predecessor block is an EH pad that does not allow non-PHI // instructions before the terminator, we can't PRE the load. @@ -2211,36 +2212,47 @@ ReplaceOperandsWithMap.clear(); bool ChangedFunction = false; + auto eraseInstrs = [this, BB](bool BBContentsMoved) { + // If we need some instructions deleted, do it now. + NumGVNInstr += InstrsToErase.size(); + + for (auto *I : InstrsToErase) { + assert((BBContentsMoved || I->getParent() == BB) && + "Removing instruction from wrong block?"); + LLVM_DEBUG(dbgs() << "GVN removed: " << *I << '\n'); + salvageKnowledge(I, AC); + salvageDebugInfo(*I); + if (MD) + MD->removeInstruction(I); + LLVM_DEBUG(verifyRemoved(I)); + ICF->removeInstruction(I); + I->eraseFromParent(); + } + InstrsToErase.clear(); + }; + for (BasicBlock::iterator BI = BB->begin(), BE = BB->end(); BI != BE;) { if (!ReplaceOperandsWithMap.empty()) ChangedFunction |= replaceOperandsForInBlockEquality(&*BI); ChangedFunction |= processInstruction(&*BI); + if (BB->getTerminator()->getIterator() == BB->begin()) { + eraseInstrs(true); + return ChangedFunction; + } + if (InstrsToErase.empty()) { ++BI; continue; } - // If we need some instructions deleted, do it now. - NumGVNInstr += InstrsToErase.size(); - // Avoid iterator invalidation. bool AtStart = BI == BB->begin(); if (!AtStart) --BI; - for (auto *I : InstrsToErase) { - assert(I->getParent() == BB && "Removing instruction from wrong block?"); - LLVM_DEBUG(dbgs() << "GVN removed: " << *I << '\n'); - salvageKnowledge(I, AC); - salvageDebugInfo(*I); - if (MD) MD->removeInstruction(I); - LLVM_DEBUG(verifyRemoved(I)); - ICF->removeInstruction(I); - I->eraseFromParent(); - } - InstrsToErase.clear(); + eraseInstrs(false); if (AtStart) BI = BB->begin(); diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp --- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -5341,6 +5341,18 @@ // phi predecessors are identical. The simple thing to do is skip // splitting in this case rather than complicate the API. if (NewBB) { + auto *MergeBlock = NewBB->getSingleSuccessor(); + for (auto &MergePN : MergeBlock->phis()) { + if (&MergePN == PN) { + PN = &MergePN; + break; + } + if (any_of(MergePN.incoming_values(), + [PN](Value *Inc) { return Inc == PN; })) { + PN = &MergePN; + break; + } + } // If PN is outside of the loop and BB is in the loop, we want to // move the block to be immediately before the PHI block, not // immediately after BB. diff --git a/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp b/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp --- a/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp +++ b/llvm/lib/Transforms/Utils/BreakCriticalEdges.cpp @@ -305,10 +305,23 @@ } if (!LoopPreds.empty()) { assert(!DestBB->isEHPad() && "We don't split edges to EH pads!"); - BasicBlock *NewExitBB = SplitBlockPredecessors( - DestBB, LoopPreds, "split", DT, LI, MSSAU, Options.PreserveLCSSA); if (Options.PreserveLCSSA) - createPHIsForSplitLoopExit(LoopPreds, NewExitBB, DestBB); + createPHIsForSplitLoopExit(LoopPreds, NewBB, DestBB); + auto *ExitCont = SplitBlock(DestBB, DestBB->getFirstNonPHI(), DT, LI, + MSSAU, DestBB->getName() + ".cont"); + NewBB->getTerminator()->setSuccessor(0, ExitCont); + for (auto &PN : DestBB->phis()) { + PHINode *NewExitContPhi = PHINode::Create( + PN.getType(), 2, PN.getName(), &*ExitCont->begin()); + PN.replaceAllUsesWith(NewExitContPhi); + NewExitContPhi->addIncoming(PN.getIncomingValueForBlock(NewBB), + NewBB); + NewExitContPhi->addIncoming(&PN, DestBB); + PN.removeIncomingValue(NewBB); + } + + DT->applyUpdates({{DominatorTree::Insert, NewBB, ExitCont}, + {DominatorTree::Delete, NewBB, DestBB}}); } } } diff --git a/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll b/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll --- a/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll +++ b/llvm/test/Transforms/GVN/PRE/2011-06-01-NonLocalMemdepMiscompile.ll @@ -51,13 +51,13 @@ br label %bb19 ; CHECK-LABEL: bb6: -; CHECK: br i1 undef, label %bb15split, label %bb10 +; CHECK: br i1 undef, label %bb15, label %bb10 -; CHECK-LABEL: bb15split: ; preds = %bb6 -; CHECK-NEXT: br label %bb15 +; CHECK-LABEL: bb15: ; preds = %bb6 +; CHECK-NEXT: br label %bb15.cont -; CHECK-LABEL: bb15: -; CHECK: %tmp17 = phi i8 [ %tmp8, %bb15split ], [ %tmp17.pre, %bb1.bb15_crit_edge ] +; CHECK-LABEL: bb15.cont: +; CHECK: %tmp17 = phi i8 [ %tmp17.pre, %bb1.bb15_crit_edge ], [ %tmp8, %bb15 ] bb19: ; preds = %bb15 ret i1 %tmp18 diff --git a/llvm/test/Transforms/GVN/critical-edge-split-indbr-pred-in-loop.ll b/llvm/test/Transforms/GVN/critical-edge-split-indbr-pred-in-loop.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/GVN/critical-edge-split-indbr-pred-in-loop.ll @@ -0,0 +1,43 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -gvn -S -verify-loop-info %s | FileCheck %s + +; Make sure we do not try to split blocks with indirectbr terminators in a +; loop while trying to preserve loop info. + +declare i1 @cond() + +define i32 @foo(i8* %p1, i8* %p2, i32* %p3) { +; CHECK-LABEL: @foo( +; CHECK-NEXT: bb: +; CHECK-NEXT: br label [[LOOP_HEADER:%.*]] +; CHECK: loop.header: +; CHECK-NEXT: store i32 0, i32* [[P3:%.*]], align 4 +; CHECK-NEXT: indirectbr i8* [[P1:%.*]], [label [[LOOP_LATCH1:%.*]], label %loop.latch2] +; CHECK: loop.latch1: +; CHECK-NEXT: [[C:%.*]] = call i1 @cond() +; CHECK-NEXT: br i1 [[C]], label [[LOOP_HEADER]], label [[EXIT:%.*]] +; CHECK: loop.latch2: +; CHECK-NEXT: indirectbr i8* [[P2:%.*]], [label [[LOOP_HEADER]], label %exit] +; CHECK: exit: +; CHECK-NEXT: [[X:%.*]] = load i32, i32* [[P3]], align 4 +; CHECK-NEXT: ret i32 [[X]] +; +bb: + br label %loop.header + +loop.header: ; preds = %loop.latch2, %loop.latch1, %bb + store i32 0, i32* %p3, align 4 + indirectbr i8* %p1, [label %loop.latch1, label %loop.latch2] + +loop.latch1: ; preds = %loop.header + %c = call i1 @cond() + br i1 %c, label %loop.header, label %exit + +loop.latch2: ; preds = %loop.header + indirectbr i8* %p2, [label %loop.header, label %exit] + +exit: ; preds = %loop.latch2, %loop.latch1 + %x = load i32, i32* %p3, align 4 + %y = add i32 %x, 0 + ret i32 %y +} diff --git a/llvm/test/Transforms/Util/PR37334-break-crit-edges-require-dt.ll b/llvm/test/Transforms/Util/PR37334-break-crit-edges-require-dt.ll --- a/llvm/test/Transforms/Util/PR37334-break-crit-edges-require-dt.ll +++ b/llvm/test/Transforms/Util/PR37334-break-crit-edges-require-dt.ll @@ -17,14 +17,14 @@ ; CHECK: for.cond: ; CHECK-NEXT: br i1 false, label [[FOR_BODY:%.*]], label [[FOR_COND_FOR_END_CRIT_EDGE:%.*]] ; CHECK: for.cond.for.end_crit_edge: -; CHECK-NEXT: br label [[FOR_END:%.*]] +; CHECK-NEXT: br label [[FOR_END_CONT:%.*]] ; CHECK: for.body: -; CHECK-NEXT: br i1 true, label [[FOR_ENDSPLIT:%.*]], label [[FOR_INC:%.*]] +; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[FOR_INC:%.*]] ; CHECK: for.inc: ; CHECK-NEXT: br label [[FOR_COND]] -; CHECK: for.endsplit: -; CHECK-NEXT: br label [[FOR_END]] ; CHECK: for.end: +; CHECK-NEXT: br label [[FOR_END_CONT]] +; CHECK: for.end.cont: ; CHECK-NEXT: ret void ; entry: