diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -192,6 +192,8 @@ STATISTIC(NumSpeculations, "Number of speculative executed instructions"); STATISTIC(NumInvokes, "Number of invokes with empty resume blocks simplified into calls"); +STATISTIC(NumInvokesMerged, "Number of invokes that were merged together"); +STATISTIC(NumInvokeSetsFormed, "Number of invoke sets that were formed"); namespace { @@ -2212,6 +2214,265 @@ return Changed; } +namespace { + +struct CompatibleSets { + using SetTy = SmallVector; + + SmallVector Sets; + + static bool shouldBelongToSameSet(ArrayRef Invokes); + + SetTy &getCompatibleSet(InvokeInst *II); + + void insert(InvokeInst *II); +}; + +CompatibleSets::SetTy &CompatibleSets::getCompatibleSet(InvokeInst *II) { + // Perform a linear scan over all the existing sets, see if the new `invoke` + // is compatible with any particular set. Since we know that all the `invokes` + // within a set are compatible, only check the first `invoke` in each set. + // WARNING: at worst, this has quadratic complexity. + for (CompatibleSets::SetTy &Set : Sets) { + if (CompatibleSets::shouldBelongToSameSet({Set.front(), II})) + return Set; + } + + // Otherwise, we either had no sets yet, or this invoke forms a new set. + return Sets.emplace_back(); +} + +void CompatibleSets::insert(InvokeInst *II) { + getCompatibleSet(II).emplace_back(II); +} + +bool CompatibleSets::shouldBelongToSameSet(ArrayRef Invokes) { + assert(Invokes.size() == 2 && "Always called with exactly two candidates."); + + // Can we theoretically merge these `invoke`s? + auto IsIllegalToMerge = [](InvokeInst *II) { + return II->cannotMerge() || II->isInlineAsm(); + }; + if (any_of(Invokes, IsIllegalToMerge)) + return false; + + // All callees must be identical. + // FIXME: support indirect callees? + Value *Callee = nullptr; + for (InvokeInst *II : Invokes) { + Value *CurrCallee = II->getCalledOperand(); + assert(CurrCallee && "There is always a called operand."); + if (!Callee) + Callee = CurrCallee; + else if (Callee != CurrCallee) + return false; + } + + // Both `invoke`s must not have a normal destination. + // FIXME: them sharing the normal destination should be fine? + auto HasNormalDest = [](InvokeInst *II) { + return !isa(II->getNormalDest()->getFirstNonPHIOrDbg()); + }; + if (any_of(Invokes, HasNormalDest)) + return false; + +#ifndef NDEBUG + // All unwind destinations must be identical. + // We know that because we have started from said unwind destination. + BasicBlock *UnwindBB = nullptr; + for (InvokeInst *II : Invokes) { + BasicBlock *CurrUnwindBB = II->getUnwindDest(); + assert(CurrUnwindBB && "There is always an 'unwind to' basic block."); + if (!UnwindBB) + UnwindBB = CurrUnwindBB; + else + assert(UnwindBB == CurrUnwindBB && "Unexpected unwind destination."); + } +#endif + + // The successor blocks must not have any PHI nodes. + // We know we don't have the normal destination, so we don't check it. + // FIXME: instead check that the incoming values are compatible? + auto HasPHIsInUnwindDest = [](InvokeInst *II) { + return !empty(II->getUnwindDest()->phis()); + }; + if (any_of(Invokes, HasPHIsInUnwindDest)) + return false; + + // Ignoring arguments, these `invoke`s must be identical, + // including operand bundles. + const InvokeInst *II0 = Invokes.front(); + for (auto *II : Invokes.drop_front()) + if (!II->isSameOperationAs(II0)) + return false; + + // Can we theoretically form the data operands for the merged `invoke`? + auto IsIllegalToMergeArguments = [](auto Ops) { + Type *Ty = std::get<0>(Ops)->getType(); + assert(Ty == std::get<1>(Ops)->getType() && "Incompatible types?"); + return Ty->isTokenTy() && std::get<0>(Ops) != std::get<1>(Ops); + }; + assert(Invokes.size() == 2 && "Always called with exactly two candidates."); + if (any_of(zip(Invokes[0]->data_ops(), Invokes[1]->data_ops()), + IsIllegalToMergeArguments)) + return false; + + return true; +} + +} // namespace + +// Merge all invokes in the provided set, all of which are compatible +// as per the `CompatibleSets::shouldBelongToSameSet()`. +static void MergeCompatibleInvokesImpl(ArrayRef Invokes, + DomTreeUpdater *DTU) { + assert(Invokes.size() >= 2 && "Must have at least two invokes to merge."); + + SmallVector Updates; + if (DTU) + Updates.reserve(2 + 3 * Invokes.size()); + + // Clone one of the invokes into a new basic block. + // Since they are all compatible, it doesn't matter which invoke is cloned. + InvokeInst *MergedInvoke = [&Invokes]() { + InvokeInst *II0 = Invokes.front(); + BasicBlock *II0BB = II0->getParent(); + BasicBlock *InsertBeforeBlock = + II0->getParent()->getIterator()->getNextNode(); + Function *Func = II0BB->getParent(); + LLVMContext &Ctx = II0->getContext(); + + BasicBlock *MergedInvokeBB = BasicBlock::Create( + Ctx, II0BB->getName() + ".invoke", Func, InsertBeforeBlock); + + auto *MergedInvoke = cast(II0->clone()); + // NOTE: all invokes have the same attributes, so no handling needed. + MergedInvokeBB->getInstList().push_back(MergedInvoke); + + // For now, we've required that the normal destination is unreachable, + // so just form a new block with unreachable terminator. + BasicBlock *MergedNormalDest = BasicBlock::Create( + Ctx, II0BB->getName() + ".cont", Func, InsertBeforeBlock); + new UnreachableInst(Ctx, MergedNormalDest); + MergedInvoke->setNormalDest(MergedNormalDest); + + // The unwind destination, however, remainds identical for all invokes here. + + return MergedInvoke; + }(); + + if (DTU) { + // Predecessor blocks that contained these invokes will now branch to + // the new block that contains the merged invoke, ... + for (InvokeInst *II : Invokes) + Updates.push_back( + {DominatorTree::Insert, II->getParent(), MergedInvoke->getParent()}); + + // ... which has the new `unreachable` block as normal destination, + // or unwinds to the (same for all `invoke`s in this set) `landingpad`, + for (BasicBlock *SuccBBOfMergedInvoke : successors(MergedInvoke)) + Updates.push_back({DominatorTree::Insert, MergedInvoke->getParent(), + SuccBBOfMergedInvoke}); + + // Since predecessor blocks now unconditionally branch to a new block, + // they no longer branch to their original successors. + for (InvokeInst *II : Invokes) + for (BasicBlock *SuccOfPredBB : successors(II->getParent())) + Updates.push_back( + {DominatorTree::Delete, II->getParent(), SuccOfPredBB}); + } + + // Form the merged data operands for the merged invoke. + for (Use &U : MergedInvoke->data_ops()) { + Type *Ty = U->getType(); + if (Ty->isTokenTy()) + continue; // Keep this arg as-is, we've checked that all the invokes + // recieve the *same* token value. + + // Otherwise, simply form a PHI out of all the data ops under this index. + PHINode *PN = PHINode::Create(Ty, /*NumReservedValues=*/Invokes.size(), "", + MergedInvoke); + for (InvokeInst *II : Invokes) { + Use *IVU = II->data_operands_begin() + MergedInvoke->getDataOperandNo(&U); + PN->addIncoming(IVU->get(), II->getParent()); + } + + U.set(PN); + } + + // And finally, replace the original `invoke`s with an unconditional branch + // to the block with the merged `invoke`. Also, give that merged `invoke` + // the merged debugloc of all the original `invoke`s. + const DILocation *MergedDebugLoc = nullptr; + for (InvokeInst *II : Invokes) { + // Compute the debug location common to all the original `invoke`s. + if (!MergedDebugLoc) + MergedDebugLoc = II->getDebugLoc(); + else + MergedDebugLoc = + DILocation::getMergedLocation(MergedDebugLoc, II->getDebugLoc()); + + // And replace the old `invoke` with an unconditionally branch + // to the block with the merged `invoke`. + BranchInst::Create(MergedInvoke->getParent(), II->getParent()); + // Since the normal destination was unreachable, there are no live uses. + II->replaceAllUsesWith(UndefValue::get(II->getType())); + II->eraseFromParent(); + ++NumInvokesMerged; + } + MergedInvoke->setDebugLoc(MergedDebugLoc); + ++NumInvokeSetsFormed; + + if (DTU) + DTU->applyUpdates(Updates); +} + +/// If this block is a `landingpad` exception handling block, categorize all +/// the predecessor `invoke`s into sets, with all `invoke`s in each set +/// being "mergeable" together, and then merge invokes in each set together. +/// +/// This is a weird mix of hoisting and sinking. Visually, it goes from: +/// [...] [...] +/// | | +/// [invoke0] [invoke1] +/// / \ / \ +/// [cont0] [landingpad] [cont1] +/// to: +/// [...] [...] +/// \ / +/// [invoke] +/// / \ +/// [cont] [landingpad] +/// +/// But of course we can only do that if the invokes share the `landingpad`, +/// edges invoke0->cont0 and invoke1->cont1 are "compatible", +/// and the invoked functions are "compatible". +static bool MergeCompatibleInvokes(BasicBlock *BB, DomTreeUpdater *DTU) { + bool Changed = false; + + // FIXME: generalize to all exception handling blocks? + if (!BB->isLandingPad()) + return Changed; + + CompatibleSets Grouper; + + // Record all the predecessors of this `landingpad`. As per verifier, + // the only allowed predecessor is the unwind edge of an `invoke`. + // We want to group "compatible" `invokes` into the same set to be merged. + for (BasicBlock *PredBB : predecessors(BB)) + Grouper.insert(cast(PredBB->getTerminator())); + + // And now, merge `invoke`s that were grouped togeter. + for (ArrayRef Invokes : Grouper.Sets) { + if (Invokes.size() < 2) + continue; + Changed = true; + MergeCompatibleInvokesImpl(Invokes, DTU); + } + + return Changed; +} + /// Determine if we can hoist sink a sole store instruction out of a /// conditional block. /// @@ -6725,7 +6986,8 @@ return true; if (SinkCommon && Options.SinkCommonInsts) - if (SinkCommonCodeFromPredecessors(BB, DTU)) { + if (SinkCommonCodeFromPredecessors(BB, DTU) || + MergeCompatibleInvokes(BB, DTU)) { // SinkCommonCodeFromPredecessors() does not automatically CSE PHI's, // so we may now how duplicate PHI's. // Let's rerun EliminateDuplicatePHINodes() first, diff --git a/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad-debuginfo.ll b/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad-debuginfo.ll --- a/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad-debuginfo.ll +++ b/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad-debuginfo.ll @@ -11,29 +11,24 @@ ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond(), !dbg [[DBG12:![0-9]+]] ; CHECK-NEXT: call void @llvm.dbg.value(metadata i1 [[C0]], metadata [[META9:![0-9]+]], metadata !DIExpression()), !dbg [[DBG12]] -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE:%.*]], !dbg [[DBG13:![0-9]+]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]], !dbg [[DBG14:![0-9]+]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable, !dbg [[DBG15:![0-9]+]] +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE:%.*]], !dbg [[DBG13:![0-9]+]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } -; CHECK-NEXT: cleanup, !dbg [[DBG16:![0-9]+]] -; CHECK-NEXT: call void @destructor(), !dbg [[DBG17:![0-9]+]] -; CHECK-NEXT: resume { i8*, i32 } [[EH]], !dbg [[DBG18:![0-9]+]] +; CHECK-NEXT: cleanup, !dbg [[DBG14:![0-9]+]] +; CHECK-NEXT: call void @destructor(), !dbg [[DBG15:![0-9]+]] +; CHECK-NEXT: resume { i8*, i32 } [[EH]], !dbg [[DBG16:![0-9]+]] ; CHECK: if.else: -; CHECK-NEXT: [[C1:%.*]] = call i1 @cond(), !dbg [[DBG19:![0-9]+]] -; CHECK-NEXT: call void @llvm.dbg.value(metadata i1 [[C1]], metadata [[META11:![0-9]+]], metadata !DIExpression()), !dbg [[DBG19]] -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]], !dbg [[DBG20:![0-9]+]] -; CHECK: if.then1: +; CHECK-NEXT: [[C1:%.*]] = call i1 @cond(), !dbg [[DBG17:![0-9]+]] +; CHECK-NEXT: call void @llvm.dbg.value(metadata i1 [[C1]], metadata [[META11:![0-9]+]], metadata !DIExpression()), !dbg [[DBG17]] +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_END:%.*]], !dbg [[DBG18:![0-9]+]] +; CHECK: if.then1.invoke: ; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]], !dbg [[DBG21:![0-9]+]] -; CHECK: invoke.cont2: -; CHECK-NEXT: unreachable, !dbg [[DBG22:![0-9]+]] +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]], !dbg [[DBG19:![0-9]+]] +; CHECK: if.then1.cont: +; CHECK-NEXT: unreachable ; CHECK: if.end: -; CHECK-NEXT: call void @sideeffect(), !dbg [[DBG23:![0-9]+]] -; CHECK-NEXT: ret void, !dbg [[DBG24:![0-9]+]] +; CHECK-NEXT: call void @sideeffect(), !dbg [[DBG20:![0-9]+]] +; CHECK-NEXT: ret void, !dbg [[DBG21:![0-9]+]] ; entry: %c0 = call i1 @cond() @@ -89,15 +84,12 @@ ; CHECK: [[META11]] = !DILocalVariable(name: "2", scope: !5, file: !1, line: 8, type: !10) ; CHECK: [[DBG12]] = !DILocation(line: 1, column: 1, scope: !5) ; CHECK: [[DBG13]] = !DILocation(line: 2, column: 1, scope: !5) -; CHECK: [[DBG14]] = !DILocation(line: 3, column: 1, scope: !5) -; CHECK: [[DBG15]] = !DILocation(line: 4, column: 1, scope: !5) -; CHECK: [[DBG16]] = !DILocation(line: 5, column: 1, scope: !5) -; CHECK: [[DBG17]] = !DILocation(line: 6, column: 1, scope: !5) -; CHECK: [[DBG18]] = !DILocation(line: 7, column: 1, scope: !5) -; CHECK: [[DBG19]] = !DILocation(line: 8, column: 1, scope: !5) -; CHECK: [[DBG20]] = !DILocation(line: 9, column: 1, scope: !5) -; CHECK: [[DBG21]] = !DILocation(line: 10, column: 1, scope: !5) -; CHECK: [[DBG22]] = !DILocation(line: 11, column: 1, scope: !5) -; CHECK: [[DBG23]] = !DILocation(line: 12, column: 1, scope: !5) -; CHECK: [[DBG24]] = !DILocation(line: 13, column: 1, scope: !5) +; CHECK: [[DBG14]] = !DILocation(line: 5, column: 1, scope: !5) +; CHECK: [[DBG15]] = !DILocation(line: 6, column: 1, scope: !5) +; CHECK: [[DBG16]] = !DILocation(line: 7, column: 1, scope: !5) +; CHECK: [[DBG17]] = !DILocation(line: 8, column: 1, scope: !5) +; CHECK: [[DBG18]] = !DILocation(line: 9, column: 1, scope: !5) +; CHECK: [[DBG19]] = !DILocation(line: 0, scope: !5) +; CHECK: [[DBG20]] = !DILocation(line: 12, column: 1, scope: !5) +; CHECK: [[DBG21]] = !DILocation(line: 13, column: 1, scope: !5) ;. diff --git a/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad.ll b/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad.ll --- a/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad.ll +++ b/llvm/test/Transforms/SimplifyCFG/X86/merge-compatible-invokes-of-landingpad.ll @@ -50,12 +50,7 @@ ; CHECK-LABEL: @t1_mergeable_invoke( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -63,11 +58,11 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] -; CHECK: if.then1: +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_END:%.*]] +; CHECK: if.then1.invoke: ; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont2: +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then1.cont: ; CHECK-NEXT: unreachable ; CHECK: if.end: ; CHECK-NEXT: call void @sideeffect() @@ -108,12 +103,7 @@ ; CHECK-LABEL: @t2_shared_normal_dest( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -121,10 +111,12 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] -; CHECK: if.then1: +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_END:%.*]] +; CHECK: if.then1.invoke: ; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT]] unwind label [[LPAD]] +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then1.cont: +; CHECK-NEXT: unreachable ; CHECK: if.end: ; CHECK-NEXT: call void @sideeffect() ; CHECK-NEXT: ret void @@ -697,12 +689,7 @@ ; CHECK-LABEL: @t12_arguments_are_fine( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw_taking_argument(i32 42) -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -710,11 +697,12 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] -; CHECK: if.then1: -; CHECK-NEXT: invoke void @simple_throw_taking_argument(i32 42) -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont2: +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_END:%.*]] +; CHECK: if.then1.invoke: +; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ 42, [[IF_ELSE]] ], [ 42, [[ENTRY:%.*]] ] +; CHECK-NEXT: invoke void @simple_throw_taking_argument(i32 [[TMP0]]) +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then1.cont: ; CHECK-NEXT: unreachable ; CHECK: if.end: ; CHECK-NEXT: call void @sideeffect() @@ -755,12 +743,7 @@ ; CHECK-LABEL: @t13_different_arguments_are_fine( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw_taking_argument(i32 0) -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -768,11 +751,12 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] -; CHECK: if.then1: -; CHECK-NEXT: invoke void @simple_throw_taking_argument(i32 42) -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont2: +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_END:%.*]] +; CHECK: if.then1.invoke: +; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ 42, [[IF_ELSE]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: invoke void @simple_throw_taking_argument(i32 [[TMP0]]) +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then1.cont: ; CHECK-NEXT: unreachable ; CHECK: if.end: ; CHECK-NEXT: call void @sideeffect() @@ -813,12 +797,7 @@ ; CHECK-LABEL: @t14_three_invokes_only_two_compatible( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE0:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN2_INVOKE:%.*]], label [[IF_ELSE0:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -826,19 +805,14 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else0: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]] -; CHECK: if.then1: -; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont2: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN2_INVOKE]], label [[IF_ELSE1:%.*]] ; CHECK: if.else1: ; CHECK-NEXT: [[C2:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C2]], label [[IF_THEN2:%.*]], label [[IF_END:%.*]] -; CHECK: if.then2: +; CHECK-NEXT: br i1 [[C2]], label [[IF_THEN2_INVOKE]], label [[IF_END:%.*]] +; CHECK: if.then2.invoke: ; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont3: +; CHECK-NEXT: to label [[IF_THEN2_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then2.cont: ; CHECK-NEXT: unreachable ; CHECK: if.end: ; CHECK-NEXT: call void @sideeffect() @@ -889,12 +863,7 @@ ; CHECK-LABEL: @t15_three_invokes_only_two_compatible( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE0:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE0:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -902,11 +871,11 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else0: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]] -; CHECK: if.then1: +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_ELSE1:%.*]] +; CHECK: if.then1.invoke: ; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont2: +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then1.cont: ; CHECK-NEXT: unreachable ; CHECK: if.else1: ; CHECK-NEXT: [[C2:%.*]] = call i1 @cond() @@ -965,12 +934,7 @@ ; CHECK-LABEL: @t16_four_invokes_forming_two_sets( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE0:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE0:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -978,27 +942,22 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else0: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_ELSE1:%.*]] -; CHECK: if.then1: +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_ELSE1:%.*]] +; CHECK: if.then1.invoke: ; CHECK-NEXT: invoke void @simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont2: +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then1.cont: ; CHECK-NEXT: unreachable ; CHECK: if.else1: ; CHECK-NEXT: [[C2:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C2]], label [[IF_THEN2:%.*]], label [[IF_ELSE2:%.*]] -; CHECK: if.then2: -; CHECK-NEXT: invoke void @another_simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT3:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont3: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C2]], label [[IF_THEN3_INVOKE:%.*]], label [[IF_ELSE2:%.*]] ; CHECK: if.else2: ; CHECK-NEXT: [[C3:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C3]], label [[IF_THEN3:%.*]], label [[IF_END:%.*]] -; CHECK: if.then3: +; CHECK-NEXT: br i1 [[C3]], label [[IF_THEN3_INVOKE]], label [[IF_END:%.*]] +; CHECK: if.then3.invoke: ; CHECK-NEXT: invoke void @another_simple_throw() -; CHECK-NEXT: to label [[INVOKE_CONT4:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont4: +; CHECK-NEXT: to label [[IF_THEN3_CONT:%.*]] unwind label [[LPAD]] +; CHECK: if.then3.cont: ; CHECK-NEXT: unreachable ; CHECK: if.end: ; CHECK-NEXT: call void @sideeffect() @@ -1117,12 +1076,7 @@ ; CHECK-LABEL: @t18_attributes_are_preserved( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw() #[[ATTR2]] -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -1130,11 +1084,11 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] -; CHECK: if.then1: +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_END:%.*]] +; CHECK: if.then1.invoke: ; CHECK-NEXT: invoke void @simple_throw() #[[ATTR2]] -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont2: +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then1.cont: ; CHECK-NEXT: unreachable ; CHECK: if.end: ; CHECK-NEXT: call void @sideeffect() @@ -1175,12 +1129,7 @@ ; CHECK-LABEL: @t19_compatible_operand_bundle( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw() [ "abc"(i32 42) ] -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -1188,11 +1137,12 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] -; CHECK: if.then1: -; CHECK-NEXT: invoke void @simple_throw() [ "abc"(i32 42) ] -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont2: +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_END:%.*]] +; CHECK: if.then1.invoke: +; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ 42, [[IF_ELSE]] ], [ 42, [[ENTRY:%.*]] ] +; CHECK-NEXT: invoke void @simple_throw() [ "abc"(i32 [[TMP0]]) ] +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then1.cont: ; CHECK-NEXT: unreachable ; CHECK: if.end: ; CHECK-NEXT: call void @sideeffect() @@ -1291,12 +1241,7 @@ ; CHECK-LABEL: @t21_semicompatible_operand_bundle( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[C0:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN0:%.*]], label [[IF_ELSE:%.*]] -; CHECK: if.then0: -; CHECK-NEXT: invoke void @simple_throw() [ "abc"(i32 42) ] -; CHECK-NEXT: to label [[INVOKE_CONT0:%.*]] unwind label [[LPAD:%.*]] -; CHECK: invoke.cont0: -; CHECK-NEXT: unreachable +; CHECK-NEXT: br i1 [[C0]], label [[IF_THEN1_INVOKE:%.*]], label [[IF_ELSE:%.*]] ; CHECK: lpad: ; CHECK-NEXT: [[EH:%.*]] = landingpad { i8*, i32 } ; CHECK-NEXT: cleanup @@ -1304,11 +1249,12 @@ ; CHECK-NEXT: resume { i8*, i32 } [[EH]] ; CHECK: if.else: ; CHECK-NEXT: [[C1:%.*]] = call i1 @cond() -; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1:%.*]], label [[IF_END:%.*]] -; CHECK: if.then1: -; CHECK-NEXT: invoke void @simple_throw() [ "abc"(i32 0) ] -; CHECK-NEXT: to label [[INVOKE_CONT2:%.*]] unwind label [[LPAD]] -; CHECK: invoke.cont2: +; CHECK-NEXT: br i1 [[C1]], label [[IF_THEN1_INVOKE]], label [[IF_END:%.*]] +; CHECK: if.then1.invoke: +; CHECK-NEXT: [[TMP0:%.*]] = phi i32 [ 0, [[IF_ELSE]] ], [ 42, [[ENTRY:%.*]] ] +; CHECK-NEXT: invoke void @simple_throw() [ "abc"(i32 [[TMP0]]) ] +; CHECK-NEXT: to label [[IF_THEN1_CONT:%.*]] unwind label [[LPAD:%.*]] +; CHECK: if.then1.cont: ; CHECK-NEXT: unreachable ; CHECK: if.end: ; CHECK-NEXT: call void @sideeffect()