diff --git a/llvm/include/llvm/Transforms/IPO/Inliner.h b/llvm/include/llvm/Transforms/IPO/Inliner.h --- a/llvm/include/llvm/Transforms/IPO/Inliner.h +++ b/llvm/include/llvm/Transforms/IPO/Inliner.h @@ -51,8 +51,16 @@ /// This method must be implemented by the subclass to determine the cost of /// inlining the specified call site. If the cost returned is greater than /// the current inline threshold, the call site is not inlined. + // FIXME(mtrofin): remove this in favor of the CallBase-based one virtual InlineCost getInlineCost(CallSite CS) = 0; + /// This method must be implemented by the subclass to determine the cost of + /// inlining the specified call site. If the cost returned is greater than + /// the current inline threshold, the call site is not inlined. + virtual InlineCost getInlineCost(CallBase &CB) { + return getInlineCost(CallSite(&CB)); + } + /// Remove dead functions. /// /// This also includes a hack in the form of the 'AlwaysInlineOnly' flag diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -273,7 +273,7 @@ /// inline this call site we attempt to reuse already available allocas or add /// any new allocas to the set if not possible. static InlineResult InlineCallIfPossible( - CallSite CS, InlineFunctionInfo &IFI, + CallBase &CS, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory, bool InsertLifetime, function_ref &AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats) { @@ -284,7 +284,7 @@ // Try to inline the function. Get the list of static allocas that were // inlined. - InlineResult IR = InlineFunction(CS, IFI, &AAR, InsertLifetime); + InlineResult IR = InlineFunction(&CS, IFI, &AAR, InsertLifetime); if (!IR.isSuccess()) return IR; @@ -305,9 +305,8 @@ /// \p TotalSecondaryCost will be set to the estimated cost of inlining the /// caller if \p CS is suppressed for inlining. static bool -shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC, - int &TotalSecondaryCost, - function_ref GetInlineCost) { +shouldBeDeferred(Function *Caller, InlineCost IC, int &TotalSecondaryCost, + function_ref GetInlineCost) { // For now we only handle local or inline functions. if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage()) return false; @@ -346,17 +345,17 @@ // applied), then we can exit the loop early. if (!ApplyLastCallBonus && TotalSecondaryCost >= IC.getCost()) return false; - CallSite CS2(U); + CallBase *CS2 = dyn_cast(U); // If this isn't a call to Caller (it could be some other sort // of reference) skip it. Such references will prevent the caller // from being removed. - if (!CS2 || CS2.getCalledFunction() != Caller) { + if (!CS2 || CS2->getCalledFunction() != Caller) { ApplyLastCallBonus = false; continue; } - InlineCost IC2 = GetInlineCost(CS2); + InlineCost IC2 = GetInlineCost(*CS2); ++NumCallerCallersAnalyzed; if (!IC2) { ApplyLastCallBonus = false; @@ -417,24 +416,24 @@ /// CallSite. If we return the cost, we will emit an optimisation remark later /// using that cost, so we won't do so from this function. static Optional -shouldInline(CallSite CS, function_ref GetInlineCost, +shouldInline(CallBase &CS, function_ref GetInlineCost, OptimizationRemarkEmitter &ORE) { using namespace ore; InlineCost IC = GetInlineCost(CS); - Instruction *Call = CS.getInstruction(); + Instruction *Call = &CS; Function *Callee = CS.getCalledFunction(); Function *Caller = CS.getCaller(); if (IC.isAlways()) { LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) - << ", Call: " << *CS.getInstruction() << "\n"); + << ", Call: " << CS << "\n"); return IC; } if (IC.isNever()) { LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC) - << ", Call: " << *CS.getInstruction() << "\n"); + << ", Call: " << CS << "\n"); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call) << NV("Callee", Callee) << " not inlined into " @@ -446,7 +445,7 @@ if (!IC) { LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC) - << ", Call: " << *CS.getInstruction() << "\n"); + << ", Call: " << CS << "\n"); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call) << NV("Callee", Callee) << " not inlined into " @@ -456,8 +455,8 @@ } int TotalSecondaryCost = 0; - if (shouldBeDeferred(Caller, CS, IC, TotalSecondaryCost, GetInlineCost)) { - LLVM_DEBUG(dbgs() << " NOT Inlining: " << *CS.getInstruction() + if (shouldBeDeferred(Caller, IC, TotalSecondaryCost, GetInlineCost)) { + LLVM_DEBUG(dbgs() << " NOT Inlining: " << CS << " Cost = " << IC.getCost() << ", outer Cost = " << TotalSecondaryCost << '\n'); ORE.emit([&]() { @@ -473,8 +472,8 @@ return None; } - LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) - << ", Call: " << *CS.getInstruction() << '\n'); + LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << CS + << '\n'); return IC; } @@ -517,11 +516,11 @@ }); } -static void setInlineRemark(CallSite &CS, StringRef message) { +static void setInlineRemark(CallBase &CS, StringRef Message) { if (!InlineRemarkAttribute) return; - Attribute attr = Attribute::get(CS->getContext(), "inline-remark", message); + Attribute attr = Attribute::get(CS.getContext(), "inline-remark", Message); CS.addAttribute(AttributeList::FunctionIndex, attr); } @@ -531,7 +530,7 @@ ProfileSummaryInfo *PSI, std::function GetTLI, bool InsertLifetime, - function_ref GetInlineCost, + function_ref GetInlineCost, function_ref AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats) { SmallPtrSet SCCFunctions; @@ -546,7 +545,7 @@ // Scan through and identify all call sites ahead of time so that we only // inline call sites in the original functions, not call sites that result // from inlining other functions. - SmallVector, 16> CallSites; + SmallVector, 16> CallSites; // When inlining a callee produces new call sites, we want to keep track of // the fact that they were inlined from the callee. This allows us to avoid @@ -562,7 +561,7 @@ OptimizationRemarkEmitter ORE(F); for (BasicBlock &BB : *F) for (Instruction &I : BB) { - CallSite CS(cast(&I)); + auto *CS = dyn_cast(&I); // If this isn't a call, or it is a call to an intrinsic, it can // never be inlined. if (!CS || isa(I)) @@ -571,15 +570,15 @@ // If this is a direct call to an external function, we can never inline // it. If it is an indirect call, inlining may resolve it to be a // direct call, so we keep it. - if (Function *Callee = CS.getCalledFunction()) + if (Function *Callee = CS->getCalledFunction()) if (Callee->isDeclaration()) { using namespace ore; - setInlineRemark(CS, "unavailable definition"); + setInlineRemark(*CS, "unavailable definition"); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I) << NV("Callee", Callee) << " will not be inlined into " - << NV("Caller", CS.getCaller()) + << NV("Caller", CS->getCaller()) << " because its definition is unavailable" << setIsVerbose(); }); @@ -600,7 +599,7 @@ // current SCC to the end of the list. unsigned FirstCallInSCC = CallSites.size(); for (unsigned i = 0; i < FirstCallInSCC; ++i) - if (Function *F = CallSites[i].first.getCalledFunction()) + if (Function *F = CallSites[i].first->getCalledFunction()) if (SCCFunctions.count(F)) std::swap(CallSites[i--], CallSites[--FirstCallInSCC]); @@ -617,7 +616,7 @@ // calls to become direct calls. // CallSites may be modified inside so ranged for loop can not be used. for (unsigned CSi = 0; CSi != CallSites.size(); ++CSi) { - CallSite CS = CallSites[CSi].first; + CallBase &CS = *CallSites[CSi].first; Function *Caller = CS.getCaller(); Function *Callee = CS.getCalledFunction(); @@ -626,7 +625,7 @@ if (!Callee || Callee->isDeclaration()) continue; - Instruction *Instr = CS.getInstruction(); + Instruction *Instr = &CS; bool IsTriviallyDead = isInstructionTriviallyDead(Instr, &GetTLI(*Caller)); @@ -674,12 +673,12 @@ LLVM_DEBUG(dbgs() << " -> Deleting dead call: " << *Instr << "\n"); // Update the call graph by deleting the edge from Callee to Caller. setInlineRemark(CS, "trivially dead"); - CG[Caller]->removeCallEdgeFor(*cast(CS.getInstruction())); + CG[Caller]->removeCallEdgeFor(CS); Instr->eraseFromParent(); ++NumCallsDeleted; } else { // Get DebugLoc to report. CS will be invalid after Inliner. - DebugLoc DLoc = CS->getDebugLoc(); + DebugLoc DLoc = CS.getDebugLoc(); BasicBlock *Block = CS.getParent(); // Attempt to inline the function. @@ -717,16 +716,17 @@ // happen when a callsite is simpilfied to reusing the return value // of another callsite during function cloning, thus the other // callsite will be reconsidered here. - DenseSet DbgCallSites; + DenseSet DbgCallSites; for (auto &II : CallSites) DbgCallSites.insert(II.first); #endif for (Value *Ptr : InlineInfo.InlinedCalls) { #ifndef NDEBUG - assert(DbgCallSites.count(CallSite(Ptr)) == 0); + assert(DbgCallSites.count(dyn_cast(Ptr)) == 0); #endif - CallSites.push_back(std::make_pair(CallSite(Ptr), NewHistoryID)); + CallSites.push_back( + std::make_pair(dyn_cast(Ptr), NewHistoryID)); } } } @@ -784,8 +784,8 @@ }; return inlineCallsImpl( SCC, CG, GetAssumptionCache, PSI, GetTLI, InsertLifetime, - [this](CallSite CS) { return getInlineCost(CS); }, LegacyAARGetter(*this), - ImportedFunctionsStats); + [this](CallBase &CS) { return getInlineCost(CS); }, + LegacyAARGetter(*this), ImportedFunctionsStats); } /// Remove now-dead linkonce functions at the end of @@ -929,7 +929,7 @@ // this model, but it is uniformly spread across all the functions in the SCC // and eventually they all become too large to inline, rather than // incrementally maknig a single function grow in a super linear fashion. - SmallVector, 16> Calls; + SmallVector, 16> Calls; FunctionAnalysisManager &FAM = AM.getResult(InitialC, CG) @@ -945,17 +945,17 @@ // FIXME: Using instructions sequence is a really bad way to do this. // Instead we should do an actual RPO walk of the function body. for (Instruction &I : instructions(N.getFunction())) - if (auto CS = CallSite(&I)) - if (Function *Callee = CS.getCalledFunction()) { + if (auto *CS = dyn_cast(&I)) + if (Function *Callee = CS->getCalledFunction()) { if (!Callee->isDeclaration()) Calls.push_back({CS, -1}); else if (!isa(I)) { using namespace ore; - setInlineRemark(CS, "unavailable definition"); + setInlineRemark(*CS, "unavailable definition"); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I) << NV("Callee", Callee) << " will not be inlined into " - << NV("Caller", CS.getCaller()) + << NV("Caller", CS->getCaller()) << " because its definition is unavailable" << setIsVerbose(); }); @@ -991,12 +991,12 @@ // have the same caller, so we first set up some shared infrastructure for // this caller. We also do any pruning we can at this layer on the caller // alone. - Function &F = *Calls[i].first.getCaller(); + Function &F = *Calls[i].first->getCaller(); LazyCallGraph::Node &N = *CG.lookup(F); if (CG.lookupSCC(N) != C) continue; if (F.hasOptNone()) { - setInlineRemark(Calls[i].first, "optnone attribute"); + setInlineRemark(*Calls[i].first, "optnone attribute"); continue; } @@ -1027,30 +1027,29 @@ return FAM.getResult(F); }; - auto GetInlineCost = [&](CallSite CS) { + auto GetInlineCost = [&](CallBase &CS) { Function &Callee = *CS.getCalledFunction(); auto &CalleeTTI = FAM.getResult(Callee); bool RemarksEnabled = Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled( DEBUG_TYPE); - return getInlineCost(cast(*CS.getInstruction()), Params, - CalleeTTI, GetAssumptionCache, {GetBFI}, GetTLI, PSI, - RemarksEnabled ? &ORE : nullptr); + return getInlineCost(CS, Params, CalleeTTI, GetAssumptionCache, {GetBFI}, + GetTLI, PSI, RemarksEnabled ? &ORE : nullptr); }; // Now process as many calls as we have within this caller in the sequnece. // We bail out as soon as the caller has to change so we can update the // call graph and prepare the context of that new caller. bool DidInline = false; - for (; i < (int)Calls.size() && Calls[i].first.getCaller() == &F; ++i) { + for (; i < (int)Calls.size() && Calls[i].first->getCaller() == &F; ++i) { int InlineHistoryID; - CallSite CS; + CallBase *CS = nullptr; std::tie(CS, InlineHistoryID) = Calls[i]; - Function &Callee = *CS.getCalledFunction(); + Function &Callee = *CS->getCalledFunction(); if (InlineHistoryID != -1 && InlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) { - setInlineRemark(CS, "recursive"); + setInlineRemark(*CS, "recursive"); continue; } @@ -1064,21 +1063,21 @@ LLVM_DEBUG(dbgs() << "Skipping inlining internal SCC edge from a node " "previously split out of this SCC by inlining: " << F.getName() << " -> " << Callee.getName() << "\n"); - setInlineRemark(CS, "recursive SCC split"); + setInlineRemark(*CS, "recursive SCC split"); continue; } - Optional OIC = shouldInline(CS, GetInlineCost, ORE); + Optional OIC = shouldInline(*CS, GetInlineCost, ORE); // Check whether we want to inline this callsite. if (!OIC.hasValue()) { - setInlineRemark(CS, "deferred"); + setInlineRemark(*CS, "deferred"); continue; } if (!OIC.getValue()) { // shouldInline() call returned a negative inline cost that explains // why this callsite should not be inlined. - setInlineRemark(CS, inlineCostStr(*OIC)); + setInlineRemark(*CS, inlineCostStr(*OIC)); continue; } @@ -1086,19 +1085,19 @@ // `InlineFunction` routine. InlineFunctionInfo IFI( /*cg=*/nullptr, &GetAssumptionCache, PSI, - &FAM.getResult(*(CS.getCaller())), + &FAM.getResult(*(CS->getCaller())), &FAM.getResult(Callee)); // Get DebugLoc to report. CS will be invalid after Inliner. DebugLoc DLoc = CS->getDebugLoc(); - BasicBlock *Block = CS.getParent(); + BasicBlock *Block = CS->getParent(); using namespace ore; InlineResult IR = InlineFunction(CS, IFI); if (!IR.isSuccess()) { - setInlineRemark(CS, std::string(IR.getFailureReason()) + "; " + - inlineCostStr(*OIC)); + setInlineRemark(*CS, std::string(IR.getFailureReason()) + "; " + + inlineCostStr(*OIC)); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) << NV("Callee", &Callee) << " will not be inlined into " @@ -1118,6 +1117,8 @@ if (!IFI.InlinedCallSites.empty()) { int NewHistoryID = InlineHistory.size(); InlineHistory.push_back({&Callee, InlineHistoryID}); + + // FIXME(mtrofin): refactor IFI.InlinedCallSites to be CallBase-based for (CallSite &CS : reverse(IFI.InlinedCallSites)) { Function *NewCallee = CS.getCalledFunction(); if (!NewCallee) { @@ -1130,7 +1131,8 @@ } if (NewCallee) if (!NewCallee->isDeclaration()) - Calls.push_back({CS, NewHistoryID}); + Calls.push_back( + {cast(CS.getInstruction()), NewHistoryID}); } } @@ -1151,8 +1153,8 @@ if (Callee.use_empty() && !CG.isLibFunction(Callee)) { Calls.erase( std::remove_if(Calls.begin() + i + 1, Calls.end(), - [&Callee](const std::pair &Call) { - return Call.first.getCaller() == &Callee; + [&Callee](const std::pair &Call) { + return Call.first->getCaller() == &Callee; }), Calls.end()); // Clear the body and queue the function itself for deletion when we