diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -260,6 +260,19 @@ /// \pre \p U is a call instruction. bool translateCall(const User &U, MachineIRBuilder &MIRBuilder); + /// When an invoke or a cleanupret unwinds to the next EH pad, there are + /// many places it could ultimately go. In the IR, we have a single unwind + /// destination, but in the machine CFG, we enumerate all the possible blocks. + /// This function skips over imaginary basic blocks that hold catchswitch + /// instructions, and finds all the "real" machine + /// basic block destinations. As those destinations may not be successors of + /// EHPadBB, here we also calculate the edge probability to those + /// destinations. The passed-in Prob is the edge probability to EHPadBB. + bool findUnwindDestinations( + const BasicBlock *EHPadBB, BranchProbability Prob, + SmallVectorImpl> + &UnwindDests); + bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder); bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder); @@ -659,8 +672,9 @@ BranchProbability getEdgeProbability(const MachineBasicBlock *Src, const MachineBasicBlock *Dst) const; - void addSuccessorWithProb(MachineBasicBlock *Src, MachineBasicBlock *Dst, - BranchProbability Prob); + void addSuccessorWithProb( + MachineBasicBlock *Src, MachineBasicBlock *Dst, + BranchProbability Prob = BranchProbability::getUnknown()); public: IRTranslator(CodeGenOpt::Level OptLevel = CodeGenOpt::None); diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2347,6 +2347,62 @@ return true; } +bool IRTranslator::findUnwindDestinations( + const BasicBlock *EHPadBB, + BranchProbability Prob, + SmallVectorImpl> + &UnwindDests) { + EHPersonality Personality = classifyEHPersonality( + EHPadBB->getParent()->getFunction().getPersonalityFn()); + bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX; + bool IsCoreCLR = Personality == EHPersonality::CoreCLR; + bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX; + bool IsSEH = isAsynchronousEHPersonality(Personality); + + if (IsWasmCXX) { + // Ignore this for now. + return false; + } + + while (EHPadBB) { + const Instruction *Pad = EHPadBB->getFirstNonPHI(); + BasicBlock *NewEHPadBB = nullptr; + if (isa(Pad)) { + // Stop on landingpads. They are not funclets. + UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob); + break; + } + if (isa(Pad)) { + // Stop on cleanup pads. Cleanups are always funclet entries for all known + // personalities. + UnwindDests.emplace_back(&getMBB(*EHPadBB), Prob); + UnwindDests.back().first->setIsEHScopeEntry(); + UnwindDests.back().first->setIsEHFuncletEntry(); + break; + } + if (auto *CatchSwitch = dyn_cast(Pad)) { + // Add the catchpad handlers to the possible destinations. + for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { + UnwindDests.emplace_back(&getMBB(*CatchPadBB), Prob); + // For MSVC++ and the CLR, catchblocks are funclets and need prologues. + if (IsMSVCCXX || IsCoreCLR) + UnwindDests.back().first->setIsEHFuncletEntry(); + if (!IsSEH) + UnwindDests.back().first->setIsEHScopeEntry(); + } + NewEHPadBB = CatchSwitch->getUnwindDest(); + } else { + continue; + } + + BranchProbabilityInfo *BPI = FuncInfo.BPI; + if (BPI && NewEHPadBB) + Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB); + EHPadBB = NewEHPadBB; + } + return true; +} + bool IRTranslator::translateInvoke(const User &U, MachineIRBuilder &MIRBuilder) { const InvokeInst &I = cast(U); @@ -2386,14 +2442,28 @@ MCSymbol *EndSymbol = Context.createTempSymbol(); MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol); - // FIXME: track probabilities. + SmallVector, 1> UnwindDests; + BranchProbabilityInfo *BPI = FuncInfo.BPI; + MachineBasicBlock *InvokeMBB = &MIRBuilder.getMBB(); + BranchProbability EHPadBBProb = + BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB) + : BranchProbability::getZero(); + + if (!findUnwindDestinations(EHPadBB, EHPadBBProb, UnwindDests)) + return false; + MachineBasicBlock &EHPadMBB = getMBB(*EHPadBB), &ReturnMBB = getMBB(*ReturnBB); + // Update successor info. + addSuccessorWithProb(InvokeMBB, &ReturnMBB); + for (auto &UnwindDest : UnwindDests) { + UnwindDest.first->setIsEHPad(); + addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second); + } + InvokeMBB->normalizeSuccProbs(); + MF->addInvoke(&EHPadMBB, BeginSymbol, EndSymbol); - MIRBuilder.getMBB().addSuccessor(&ReturnMBB); - MIRBuilder.getMBB().addSuccessor(&EHPadMBB); MIRBuilder.buildBr(ReturnMBB); - return true; } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-invoke-probabilities.ll b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-invoke-probabilities.ll new file mode 100644 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/GlobalISel/irtranslator-invoke-probabilities.ll @@ -0,0 +1,32 @@ +; RUN: llc -mtriple=aarch64-apple-ios -global-isel -stop-after=irtranslator %s -o - | FileCheck %s + +%struct.foo = type { i64, i64, %struct.pluto, %struct.pluto } +%struct.pluto = type { %struct.wombat } +%struct.wombat = type { i32*, i32*, %struct.barney } +%struct.barney = type { %struct.widget } +%struct.widget = type { i32* } + +declare i32 @hoge(...) + +define void @pluto() align 2 personality i8* bitcast (i32 (...)* @hoge to i8*) { +; CHECK-LABEL: @pluto +; CHECK: bb.1.bb +; CHECK: successors: %bb.2(0x7ffff800), %bb.3(0x00000800) +; CHECK: EH_LABEL +; CHECK: G_BR %bb.2 + +bb: + invoke void @spam() + to label %bb1 unwind label %bb2 + +bb1: ; preds = %bb + unreachable + +bb2: ; preds = %bb + %tmp = landingpad { i8*, i32 } + cleanup + %tmp3 = getelementptr inbounds %struct.foo, %struct.foo* undef, i64 0, i32 3, i32 0, i32 0 + resume { i8*, i32 } %tmp +} + +declare void @spam()