diff --git a/llvm/include/llvm/Transforms/IPO/Inliner.h b/llvm/include/llvm/Transforms/IPO/Inliner.h --- a/llvm/include/llvm/Transforms/IPO/Inliner.h +++ b/llvm/include/llvm/Transforms/IPO/Inliner.h @@ -51,8 +51,16 @@ /// This method must be implemented by the subclass to determine the cost of /// inlining the specified call site. If the cost returned is greater than /// the current inline threshold, the call site is not inlined. + // FIXME(mtrofin): remove this in favor of the CallBase-based one virtual InlineCost getInlineCost(CallSite CS) = 0; + /// This method must be implemented by the subclass to determine the cost of + /// inlining the specified call site. If the cost returned is greater than + /// the current inline threshold, the call site is not inlined. + virtual InlineCost getInlineCost(CallBase *CB) { + return getInlineCost(CallSite(CB)); + } + /// Remove dead functions. /// /// This also includes a hack in the form of the 'AlwaysInlineOnly' flag diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -273,12 +273,12 @@ /// inline this call site we attempt to reuse already available allocas or add /// any new allocas to the set if not possible. static InlineResult InlineCallIfPossible( - CallSite CS, InlineFunctionInfo &IFI, + CallBase *CS, InlineFunctionInfo &IFI, InlinedArrayAllocasTy &InlinedArrayAllocas, int InlineHistory, bool InsertLifetime, function_ref &AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats) { - Function *Callee = CS.getCalledFunction(); - Function *Caller = CS.getCaller(); + Function *Callee = CS->getCalledFunction(); + Function *Caller = CS->getCaller(); AAResults &AAR = AARGetter(*Callee); @@ -305,9 +305,9 @@ /// \p TotalSecondaryCost will be set to the estimated cost of inlining the /// caller if \p CS is suppressed for inlining. static bool -shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC, +shouldBeDeferred(Function *Caller, CallBase *CS, InlineCost IC, int &TotalSecondaryCost, - function_ref GetInlineCost) { + function_ref GetInlineCost) { // For now we only handle local or inline functions. if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage()) return false; @@ -346,12 +346,12 @@ // applied), then we can exit the loop early. if (!ApplyLastCallBonus && TotalSecondaryCost >= IC.getCost()) return false; - CallSite CS2(U); + CallBase *CS2 = dyn_cast(U); // If this isn't a call to Caller (it could be some other sort // of reference) skip it. Such references will prevent the caller // from being removed. - if (!CS2 || CS2.getCalledFunction() != Caller) { + if (!CS2 || CS2->getCalledFunction() != Caller) { ApplyLastCallBonus = false; continue; } @@ -417,24 +417,24 @@ /// CallSite. If we return the cost, we will emit an optimisation remark later /// using that cost, so we won't do so from this function. static Optional -shouldInline(CallSite CS, function_ref GetInlineCost, +shouldInline(CallBase *CS, function_ref GetInlineCost, OptimizationRemarkEmitter &ORE) { using namespace ore; InlineCost IC = GetInlineCost(CS); - Instruction *Call = CS.getInstruction(); - Function *Callee = CS.getCalledFunction(); - Function *Caller = CS.getCaller(); + Instruction *Call = CS; + Function *Callee = CS->getCalledFunction(); + Function *Caller = CS->getCaller(); if (IC.isAlways()) { LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) - << ", Call: " << *CS.getInstruction() << "\n"); + << ", Call: " << *CS << "\n"); return IC; } if (IC.isNever()) { LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC) - << ", Call: " << *CS.getInstruction() << "\n"); + << ", Call: " << *CS << "\n"); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", Call) << NV("Callee", Callee) << " not inlined into " @@ -446,7 +446,7 @@ if (!IC) { LLVM_DEBUG(dbgs() << " NOT Inlining " << inlineCostStr(IC) - << ", Call: " << *CS.getInstruction() << "\n"); + << ", Call: " << *CS << "\n"); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "TooCostly", Call) << NV("Callee", Callee) << " not inlined into " @@ -457,7 +457,7 @@ int TotalSecondaryCost = 0; if (shouldBeDeferred(Caller, CS, IC, TotalSecondaryCost, GetInlineCost)) { - LLVM_DEBUG(dbgs() << " NOT Inlining: " << *CS.getInstruction() + LLVM_DEBUG(dbgs() << " NOT Inlining: " << *CS << " Cost = " << IC.getCost() << ", outer Cost = " << TotalSecondaryCost << '\n'); ORE.emit([&]() { @@ -473,8 +473,8 @@ return None; } - LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) - << ", Call: " << *CS.getInstruction() << '\n'); + LLVM_DEBUG(dbgs() << " Inlining " << inlineCostStr(IC) << ", Call: " << *CS + << '\n'); return IC; } @@ -517,12 +517,12 @@ }); } -static void setInlineRemark(CallSite &CS, StringRef message) { +static void setInlineRemark(CallBase *CS, StringRef Message) { if (!InlineRemarkAttribute) return; - Attribute attr = Attribute::get(CS->getContext(), "inline-remark", message); - CS.addAttribute(AttributeList::FunctionIndex, attr); + Attribute attr = Attribute::get(CS->getContext(), "inline-remark", Message); + CS->addAttribute(AttributeList::FunctionIndex, attr); } static bool @@ -531,7 +531,7 @@ ProfileSummaryInfo *PSI, std::function GetTLI, bool InsertLifetime, - function_ref GetInlineCost, + function_ref GetInlineCost, function_ref AARGetter, ImportedFunctionsInliningStatistics &ImportedFunctionsStats) { SmallPtrSet SCCFunctions; @@ -546,7 +546,7 @@ // Scan through and identify all call sites ahead of time so that we only // inline call sites in the original functions, not call sites that result // from inlining other functions. - SmallVector, 16> CallSites; + SmallVector, 16> CallSites; // When inlining a callee produces new call sites, we want to keep track of // the fact that they were inlined from the callee. This allows us to avoid @@ -562,7 +562,7 @@ OptimizationRemarkEmitter ORE(F); for (BasicBlock &BB : *F) for (Instruction &I : BB) { - CallSite CS(cast(&I)); + auto *CS = dyn_cast(&I); // If this isn't a call, or it is a call to an intrinsic, it can // never be inlined. if (!CS || isa(I)) @@ -571,7 +571,7 @@ // If this is a direct call to an external function, we can never inline // it. If it is an indirect call, inlining may resolve it to be a // direct call, so we keep it. - if (Function *Callee = CS.getCalledFunction()) + if (Function *Callee = CS->getCalledFunction()) if (Callee->isDeclaration()) { using namespace ore; @@ -579,7 +579,7 @@ ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I) << NV("Callee", Callee) << " will not be inlined into " - << NV("Caller", CS.getCaller()) + << NV("Caller", CS->getCaller()) << " because its definition is unavailable" << setIsVerbose(); }); @@ -600,7 +600,7 @@ // current SCC to the end of the list. unsigned FirstCallInSCC = CallSites.size(); for (unsigned i = 0; i < FirstCallInSCC; ++i) - if (Function *F = CallSites[i].first.getCalledFunction()) + if (Function *F = CallSites[i].first->getCalledFunction()) if (SCCFunctions.count(F)) std::swap(CallSites[i--], CallSites[--FirstCallInSCC]); @@ -617,16 +617,16 @@ // calls to become direct calls. // CallSites may be modified inside so ranged for loop can not be used. for (unsigned CSi = 0; CSi != CallSites.size(); ++CSi) { - CallSite CS = CallSites[CSi].first; + CallBase *CS = CallSites[CSi].first; - Function *Caller = CS.getCaller(); - Function *Callee = CS.getCalledFunction(); + Function *Caller = CS->getCaller(); + Function *Callee = CS->getCalledFunction(); // We can only inline direct calls to non-declarations. if (!Callee || Callee->isDeclaration()) continue; - Instruction *Instr = CS.getInstruction(); + Instruction *Instr = CS; bool IsTriviallyDead = isInstructionTriviallyDead(Instr, &GetTLI(*Caller)); @@ -674,13 +674,13 @@ LLVM_DEBUG(dbgs() << " -> Deleting dead call: " << *Instr << "\n"); // Update the call graph by deleting the edge from Callee to Caller. setInlineRemark(CS, "trivially dead"); - CG[Caller]->removeCallEdgeFor(*cast(CS.getInstruction())); + CG[Caller]->removeCallEdgeFor(*CS); Instr->eraseFromParent(); ++NumCallsDeleted; } else { // Get DebugLoc to report. CS will be invalid after Inliner. DebugLoc DLoc = CS->getDebugLoc(); - BasicBlock *Block = CS.getParent(); + BasicBlock *Block = CS->getParent(); // Attempt to inline the function. using namespace ore; @@ -717,16 +717,17 @@ // happen when a callsite is simpilfied to reusing the return value // of another callsite during function cloning, thus the other // callsite will be reconsidered here. - DenseSet DbgCallSites; + DenseSet DbgCallSites; for (auto &II : CallSites) DbgCallSites.insert(II.first); #endif for (Value *Ptr : InlineInfo.InlinedCalls) { #ifndef NDEBUG - assert(DbgCallSites.count(CallSite(Ptr)) == 0); + assert(DbgCallSites.count(dyn_cast(Ptr)) == 0); #endif - CallSites.push_back(std::make_pair(CallSite(Ptr), NewHistoryID)); + CallSites.push_back( + std::make_pair(dyn_cast(Ptr), NewHistoryID)); } } } @@ -784,8 +785,8 @@ }; return inlineCallsImpl( SCC, CG, GetAssumptionCache, PSI, GetTLI, InsertLifetime, - [this](CallSite CS) { return getInlineCost(CS); }, LegacyAARGetter(*this), - ImportedFunctionsStats); + [this](CallBase *CS) { return getInlineCost(CS); }, + LegacyAARGetter(*this), ImportedFunctionsStats); } /// Remove now-dead linkonce functions at the end of @@ -929,7 +930,7 @@ // this model, but it is uniformly spread across all the functions in the SCC // and eventually they all become too large to inline, rather than // incrementally maknig a single function grow in a super linear fashion. - SmallVector, 16> Calls; + SmallVector, 16> Calls; FunctionAnalysisManager &FAM = AM.getResult(InitialC, CG) @@ -945,8 +946,8 @@ // FIXME: Using instructions sequence is a really bad way to do this. // Instead we should do an actual RPO walk of the function body. for (Instruction &I : instructions(N.getFunction())) - if (auto CS = CallSite(&I)) - if (Function *Callee = CS.getCalledFunction()) { + if (auto *CS = dyn_cast(&I)) + if (Function *Callee = CS->getCalledFunction()) { if (!Callee->isDeclaration()) Calls.push_back({CS, -1}); else if (!isa(I)) { @@ -955,7 +956,7 @@ ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NoDefinition", &I) << NV("Callee", Callee) << " will not be inlined into " - << NV("Caller", CS.getCaller()) + << NV("Caller", CS->getCaller()) << " because its definition is unavailable" << setIsVerbose(); }); @@ -991,7 +992,7 @@ // have the same caller, so we first set up some shared infrastructure for // this caller. We also do any pruning we can at this layer on the caller // alone. - Function &F = *Calls[i].first.getCaller(); + Function &F = *Calls[i].first->getCaller(); LazyCallGraph::Node &N = *CG.lookup(F); if (CG.lookupSCC(N) != C) continue; @@ -1027,26 +1028,25 @@ return FAM.getResult(F); }; - auto GetInlineCost = [&](CallSite CS) { - Function &Callee = *CS.getCalledFunction(); + auto GetInlineCost = [&](CallBase *CS) { + Function &Callee = *CS->getCalledFunction(); auto &CalleeTTI = FAM.getResult(Callee); bool RemarksEnabled = Callee.getContext().getDiagHandlerPtr()->isMissedOptRemarkEnabled( DEBUG_TYPE); - return getInlineCost(cast(*CS.getInstruction()), Params, - CalleeTTI, GetAssumptionCache, {GetBFI}, GetTLI, PSI, - RemarksEnabled ? &ORE : nullptr); + return getInlineCost(*CS, Params, CalleeTTI, GetAssumptionCache, {GetBFI}, + GetTLI, PSI, RemarksEnabled ? &ORE : nullptr); }; // Now process as many calls as we have within this caller in the sequnece. // We bail out as soon as the caller has to change so we can update the // call graph and prepare the context of that new caller. bool DidInline = false; - for (; i < (int)Calls.size() && Calls[i].first.getCaller() == &F; ++i) { + for (; i < (int)Calls.size() && Calls[i].first->getCaller() == &F; ++i) { int InlineHistoryID; - CallSite CS; + CallBase *CS; std::tie(CS, InlineHistoryID) = Calls[i]; - Function &Callee = *CS.getCalledFunction(); + Function &Callee = *CS->getCalledFunction(); if (InlineHistoryID != -1 && InlineHistoryIncludes(&Callee, InlineHistoryID, InlineHistory)) { @@ -1086,12 +1086,12 @@ // `InlineFunction` routine. InlineFunctionInfo IFI( /*cg=*/nullptr, &GetAssumptionCache, PSI, - &FAM.getResult(*(CS.getCaller())), + &FAM.getResult(*(CS->getCaller())), &FAM.getResult(Callee)); // Get DebugLoc to report. CS will be invalid after Inliner. DebugLoc DLoc = CS->getDebugLoc(); - BasicBlock *Block = CS.getParent(); + BasicBlock *Block = CS->getParent(); using namespace ore; @@ -1118,6 +1118,8 @@ if (!IFI.InlinedCallSites.empty()) { int NewHistoryID = InlineHistory.size(); InlineHistory.push_back({&Callee, InlineHistoryID}); + + // FIXME(mtrofin): refactor IFI.InlinedCallSites to be CallBase-based for (CallSite &CS : reverse(IFI.InlinedCallSites)) { Function *NewCallee = CS.getCalledFunction(); if (!NewCallee) { @@ -1130,7 +1132,8 @@ } if (NewCallee) if (!NewCallee->isDeclaration()) - Calls.push_back({CS, NewHistoryID}); + Calls.push_back( + {cast(CS.getInstruction()), NewHistoryID}); } } @@ -1151,8 +1154,8 @@ if (Callee.use_empty() && !CG.isLibFunction(Callee)) { Calls.erase( std::remove_if(Calls.begin() + i + 1, Calls.end(), - [&Callee](const std::pair &Call) { - return Call.first.getCaller() == &Callee; + [&Callee](const std::pair &Call) { + return Call.first->getCaller() == &Callee; }), Calls.end()); // Clear the body and queue the function itself for deletion when we