diff --git a/llvm/include/llvm/Analysis/InlineCost.h b/llvm/include/llvm/Analysis/InlineCost.h --- a/llvm/include/llvm/Analysis/InlineCost.h +++ b/llvm/include/llvm/Analysis/InlineCost.h @@ -131,14 +131,22 @@ }; /// InlineResult is basically true or false. For false results the message -/// describes a reason why it is decided not to inline. -struct InlineResult { - const char *message = nullptr; - InlineResult(bool result, const char *message = nullptr) - : message(result ? nullptr : (message ? message : "cost > threshold")) {} - InlineResult(const char *message = nullptr) : message(message) {} - operator bool() const { return !message; } - operator const char *() const { return message; } +/// describes a reason. +class InlineResult { + const char *Message = nullptr; + InlineResult(const char *Message = nullptr) : Message(Message) {} + +public: + static InlineResult success() { return {}; } + static InlineResult failure(const char *Reason) { + return InlineResult(Reason); + } + bool isSuccess() const { return Message == nullptr; } + const char *getFailureReason() const { + assert(!isSuccess() && + "getFailureReason should only be called in failure cases"); + return Message; + } }; /// Thresholds to tune inline cost analysis. The inline cost analysis decides diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp --- a/llvm/lib/Analysis/InlineCost.cpp +++ b/llvm/lib/Analysis/InlineCost.cpp @@ -133,8 +133,7 @@ /// Called at the end of the analysis of the callsite. Return the outcome of /// the analysis, i.e. 'InlineResult(true)' if the inlining may happen, or /// the reason it can't. - virtual InlineResult finalizeAnalysis() { return true; } - + virtual InlineResult finalizeAnalysis() { return InlineResult::success(); } /// Called when we're about to start processing a basic block, and every time /// we are done processing an instruction. Return true if there is no point in /// continuing the analysis (e.g. we've determined already the call site is @@ -145,8 +144,7 @@ /// contexts propagated). It checks callsite-specific information. Return a /// reason analysis can't continue if that's the case, or 'true' if it may /// continue. - virtual InlineResult onAnalysisStart() { return true; } - + virtual InlineResult onAnalysisStart() { return InlineResult::success(); } /// Called if the analysis engine decides SROA cannot be done for the given /// alloca. virtual void onDisableSROA(AllocaInst *Arg) {} @@ -458,7 +456,7 @@ /// to instantiate the derived class. InlineCostCallAnalyzer CA(TTI, GetAssumptionCache, GetBFI, PSI, ORE, *F, Call, IndirectCallParams, false); - if (CA.analyze()) { + if (CA.analyze().isSuccess()) { // We were able to inline the indirect call! Subtract the cost from the // threshold to get the bonus we want to apply, but don't go below zero. Cost -= std::max(0, CA.getThreshold() - CA.getCost()); @@ -538,6 +536,7 @@ SingleBB = false; } } + InlineResult finalizeAnalysis() override { // Loops generally act a lot like calls in that they act like barriers to // movement, require a certain amount of setup, etc. So when optimising for @@ -566,7 +565,9 @@ else if (NumVectorInstructions <= NumInstructions / 2) Threshold -= VectorBonus / 2; - return Cost < std::max(1, Threshold); + if (Cost < std::max(1, Threshold)) + return InlineResult::success(); + return InlineResult::failure("Cost over threshold."); } bool shouldStop() override { // Bail out the moment we cross the threshold. This means we'll under-count @@ -618,9 +619,9 @@ // Check if we're done. This can happen due to bonuses and penalties. if (Cost >= Threshold && !ComputeFullInlineCost) - return "high cost"; + return InlineResult::failure("high cost"); - return true; + return InlineResult::success(); } public: @@ -1768,26 +1769,26 @@ using namespace ore; // If the visit this instruction detected an uninlinable pattern, abort. - InlineResult IR; + InlineResult IR = InlineResult::success(); if (IsRecursiveCall) - IR = "recursive"; + IR = InlineResult::failure("recursive"); else if (ExposesReturnsTwice) - IR = "exposes returns twice"; + IR = InlineResult::failure("exposes returns twice"); else if (HasDynamicAlloca) - IR = "dynamic alloca"; + IR = InlineResult::failure("dynamic alloca"); else if (HasIndirectBr) - IR = "indirect branch"; + IR = InlineResult::failure("indirect branch"); else if (HasUninlineableIntrinsic) - IR = "uninlinable intrinsic"; + IR = InlineResult::failure("uninlinable intrinsic"); else if (InitsVargArgs) - IR = "varargs"; - if (!IR) { + IR = InlineResult::failure("varargs"); + if (!IR.isSuccess()) { if (ORE) ORE->emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", &CandidateCall) << NV("Callee", &F) << " has uninlinable pattern (" - << NV("InlineResult", IR.message) + << NV("InlineResult", IR.getFailureReason()) << ") and cost is not fully computed"; }); return IR; @@ -1798,22 +1799,25 @@ // the caller stack usage dramatically. if (IsCallerRecursive && AllocatedSize > InlineConstants::TotalAllocaSizeRecursiveCaller) { - InlineResult IR = "recursive and allocates too much stack space"; + auto IR = + InlineResult::failure("recursive and allocates too much stack space"); if (ORE) ORE->emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", &CandidateCall) - << NV("Callee", &F) << " is " << NV("InlineResult", IR.message) + << NV("Callee", &F) << " is " + << NV("InlineResult", IR.getFailureReason()) << ". Cost is not fully computed"; }); return IR; } if (shouldStop()) - return false; + return InlineResult::failure( + "Call site analysis is not favorable to inlining."); } - return true; + return InlineResult::success(); } /// Compute the base pointer and cumulative constant offsets for V. @@ -1904,11 +1908,11 @@ ++NumCallsAnalyzed; auto Result = onAnalysisStart(); - if (!Result) + if (!Result.isSuccess()) return Result; if (F.empty()) - return true; + return InlineResult::success(); Function *Caller = CandidateCall.getFunction(); // Check if the caller function is recursive itself. @@ -1983,12 +1987,12 @@ if (BB->hasAddressTaken()) for (User *U : BlockAddress::get(&*BB)->users()) if (!isa(*U)) - return "blockaddress used outside of callbr"; + return InlineResult::failure("blockaddress used outside of callbr"); // Analyze the cost of this block. If we blow through the threshold, this // returns false, and we can bail on out. InlineResult IR = analyzeBlock(BB, EphValues); - if (!IR) + if (!IR.isSuccess()) return IR; Instruction *TI = BB->getTerminator(); @@ -2034,7 +2038,7 @@ // inlining this would cause the removal of the caller (so the instruction // is not actually duplicated, just moved). if (!OnlyOneCallAndLocalLinkage && ContainsNoDuplicateCall) - return "noduplicate"; + return InlineResult::failure("noduplicate"); return finalizeAnalysis(); } @@ -2140,9 +2144,9 @@ // whenever possible. if (Call.hasFnAttr(Attribute::AlwaysInline)) { auto IsViable = isInlineViable(*Callee); - if (IsViable) + if (IsViable.isSuccess()) return llvm::InlineCost::getAlways("always inline attribute"); - return llvm::InlineCost::getNever(IsViable.message); + return llvm::InlineCost::getNever(IsViable.getFailureReason()); } // Never inline functions with conflicting attributes (unless callee has @@ -2182,9 +2186,9 @@ LLVM_DEBUG(CA.dump()); // Check if there was a reason to force inlining or no inlining. - if (!ShouldInline && CA.getCost() < CA.getThreshold()) - return InlineCost::getNever(ShouldInline.message); - if (ShouldInline && CA.getCost() >= CA.getThreshold()) + if (!ShouldInline.isSuccess() && CA.getCost() < CA.getThreshold()) + return InlineCost::getNever(ShouldInline.getFailureReason()); + if (ShouldInline.isSuccess() && CA.getCost() >= CA.getThreshold()) return InlineCost::getAlways("empty function"); return llvm::InlineCost::get(CA.getCost(), CA.getThreshold()); @@ -2195,14 +2199,14 @@ for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) { // Disallow inlining of functions which contain indirect branches. if (isa(BI->getTerminator())) - return "contains indirect branches"; + return InlineResult::failure("contains indirect branches"); // Disallow inlining of blockaddresses which are used by non-callbr // instructions. if (BI->hasAddressTaken()) for (User *U : BlockAddress::get(&*BI)->users()) if (!isa(*U)) - return "blockaddress used outside of callbr"; + return InlineResult::failure("blockaddress used outside of callbr"); for (auto &II : *BI) { CallBase *Call = dyn_cast(&II); @@ -2211,13 +2215,13 @@ // Disallow recursive calls. if (&F == Call->getCalledFunction()) - return "recursive call"; + return InlineResult::failure("recursive call"); // Disallow calls which expose returns-twice to a function not previously // attributed as such. if (!ReturnsTwice && isa(Call) && cast(Call)->canReturnTwice()) - return "exposes returns-twice attribute"; + return InlineResult::failure("exposes returns-twice attribute"); if (Call->getCalledFunction()) switch (Call->getCalledFunction()->getIntrinsicID()) { @@ -2226,20 +2230,23 @@ case llvm::Intrinsic::icall_branch_funnel: // Disallow inlining of @llvm.icall.branch.funnel because current // backend can't separate call targets from call arguments. - return "disallowed inlining of @llvm.icall.branch.funnel"; + return InlineResult::failure( + "disallowed inlining of @llvm.icall.branch.funnel"); case llvm::Intrinsic::localescape: // Disallow inlining functions that call @llvm.localescape. Doing this // correctly would require major changes to the inliner. - return "disallowed inlining of @llvm.localescape"; + return InlineResult::failure( + "disallowed inlining of @llvm.localescape"); case llvm::Intrinsic::vastart: // Disallow inlining of functions that initialize VarArgs with // va_start. - return "contains VarArgs initialized with va_start"; + return InlineResult::failure( + "contains VarArgs initialized with va_start"); } } } - return true; + return InlineResult::success(); } // APIs to create InlineParams based on command line flags and/or other diff --git a/llvm/lib/CodeGen/SafeStack.cpp b/llvm/lib/CodeGen/SafeStack.cpp --- a/llvm/lib/CodeGen/SafeStack.cpp +++ b/llvm/lib/CodeGen/SafeStack.cpp @@ -708,7 +708,8 @@ bool SafeStack::ShouldInlinePointerAddress(CallSite &CS) { Function *Callee = CS.getCalledFunction(); - if (CS.hasFnAttr(Attribute::AlwaysInline) && isInlineViable(*Callee)) + if (CS.hasFnAttr(Attribute::AlwaysInline) && + isInlineViable(*Callee).isSuccess()) return true; if (Callee->isInterposable() || Callee->hasFnAttribute(Attribute::NoInline) || CS.isNoInline()) diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp --- a/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp @@ -190,9 +190,9 @@ if (CS.hasFnAttr(Attribute::AlwaysInline)) { auto IsViable = isInlineViable(*Callee); - if (IsViable) + if (IsViable.isSuccess()) return llvm::InlineCost::getAlways("alwaysinline viable"); - return llvm::InlineCost::getNever(IsViable.message); + return llvm::InlineCost::getNever(IsViable.getFailureReason()); } if (isWrapperOnlyCall(CS)) diff --git a/llvm/lib/Transforms/IPO/AlwaysInliner.cpp b/llvm/lib/Transforms/IPO/AlwaysInliner.cpp --- a/llvm/lib/Transforms/IPO/AlwaysInliner.cpp +++ b/llvm/lib/Transforms/IPO/AlwaysInliner.cpp @@ -48,7 +48,7 @@ SmallVector InlinedFunctions; for (Function &F : M) if (!F.isDeclaration() && F.hasFnAttribute(Attribute::AlwaysInline) && - isInlineViable(F)) { + isInlineViable(F).isSuccess()) { Calls.clear(); for (User *U : F.users()) @@ -60,7 +60,8 @@ // FIXME: We really shouldn't be able to fail to inline at this point! // We should do something to log or check the inline failures here. Changed |= - InlineFunction(CS, IFI, /*CalleeAAR=*/nullptr, InsertLifetime); + InlineFunction(CS, IFI, /*CalleeAAR=*/nullptr, InsertLifetime) + .isSuccess(); // Remember to try and delete this function afterward. This both avoids // re-walking the rest of the module and avoids dealing with any iterator @@ -167,8 +168,8 @@ return InlineCost::getNever("no alwaysinline attribute"); auto IsViable = isInlineViable(*Callee); - if (!IsViable) - return InlineCost::getNever(IsViable.message); + if (!IsViable.isSuccess()) + return InlineCost::getNever(IsViable.getFailureReason()); return InlineCost::getAlways("always inliner"); } diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -284,7 +284,7 @@ // Try to inline the function. Get the list of static allocas that were // inlined. InlineResult IR = InlineFunction(CS, IFI, &AAR, InsertLifetime); - if (!IR) + if (!IR.isSuccess()) return IR; if (InlinerFunctionImportStats != InlinerFunctionImportStatsOpts::No) @@ -687,13 +687,15 @@ InlineResult IR = InlineCallIfPossible( CS, InlineInfo, InlinedArrayAllocas, InlineHistoryID, InsertLifetime, AARGetter, ImportedFunctionsStats); - if (!IR) { - setInlineRemark(CS, std::string(IR) + "; " + inlineCostStr(*OIC)); + if (!IR.isSuccess()) { + setInlineRemark(CS, std::string(IR.getFailureReason()) + "; " + + inlineCostStr(*OIC)); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) << NV("Callee", Callee) << " will not be inlined into " - << NV("Caller", Caller) << ": " << NV("Reason", IR.message); + << NV("Caller", Caller) << ": " + << NV("Reason", IR.getFailureReason()); }); continue; } @@ -1076,12 +1078,14 @@ using namespace ore; InlineResult IR = InlineFunction(CS, IFI); - if (!IR) { - setInlineRemark(CS, std::string(IR) + "; " + inlineCostStr(*OIC)); + if (!IR.isSuccess()) { + setInlineRemark(CS, std::string(IR.getFailureReason()) + "; " + + inlineCostStr(*OIC)); ORE.emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) << NV("Callee", &Callee) << " will not be inlined into " - << NV("Caller", &F) << ": " << NV("Reason", IR.message); + << NV("Caller", &F) << ": " + << NV("Reason", IR.getFailureReason()); }); continue; } diff --git a/llvm/lib/Transforms/IPO/PartialInlining.cpp b/llvm/lib/Transforms/IPO/PartialInlining.cpp --- a/llvm/lib/Transforms/IPO/PartialInlining.cpp +++ b/llvm/lib/Transforms/IPO/PartialInlining.cpp @@ -769,7 +769,7 @@ assert(Callee == Cloner.ClonedFunc); if (SkipCostAnalysis) - return isInlineViable(*Callee); + return isInlineViable(*Callee).isSuccess(); Function *Caller = CS.getCaller(); auto &CalleeTTI = (*GetTTI)(*Callee); @@ -1411,7 +1411,8 @@ // bail on vararg functions. if (!InlineFunction(CS, IFI, nullptr, true, (Cloner.ClonedOI ? Cloner.OutlinedFunctions.back().first - : nullptr))) + : nullptr)) + .isSuccess()) continue; CallerORE.emit(OR); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -909,7 +909,7 @@ return false; } InlineFunctionInfo IFI(nullptr, &GetAC); - if (InlineFunction(CS, IFI)) { + if (InlineFunction(CS, IFI).isSuccess()) { // The call to InlineFunction erases I, so we can't pass it here. ORE->emit(OptimizationRemark(CSINLINE_DEBUG, "InlineSuccess", DLoc, BB) << "inlined callee '" << ore::NV("Callee", CalledFunction) diff --git a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp --- a/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp +++ b/llvm/lib/Transforms/Scalar/PlaceSafepoints.cpp @@ -650,7 +650,7 @@ // Do the actual inlining InlineFunctionInfo IFI; - bool InlineStatus = InlineFunction(PollCall, IFI); + bool InlineStatus = InlineFunction(PollCall, IFI).isSuccess(); assert(InlineStatus && "inline must succeed"); (void)InlineStatus; // suppress warning in release-asserts diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -1567,7 +1567,7 @@ // FIXME: we don't inline callbr yet. if (isa(TheCall)) - return false; + return InlineResult::failure("We don't inline callbr yet."); // If IFI has any state in it, zap it before we fill it in. IFI.reset(); @@ -1575,7 +1575,7 @@ Function *CalledFunc = CS.getCalledFunction(); if (!CalledFunc || // Can't inline external function or indirect CalledFunc->isDeclaration()) // call! - return "external or indirect"; + return InlineResult::failure("external or indirect"); // The inliner does not know how to inline through calls with operand bundles // in general ... @@ -1589,7 +1589,7 @@ if (Tag == LLVMContext::OB_funclet) continue; - return "unsupported operand bundle"; + return InlineResult::failure("unsupported operand bundle"); } } @@ -1608,7 +1608,7 @@ if (!Caller->hasGC()) Caller->setGC(CalledFunc->getGC()); else if (CalledFunc->getGC() != Caller->getGC()) - return "incompatible GC"; + return InlineResult::failure("incompatible GC"); } // Get the personality function from the callee if it contains a landing pad. @@ -1632,7 +1632,7 @@ // TODO: This isn't 100% true. Some personality functions are proper // supersets of others and can be used in place of the other. else if (CalledPersonality != CallerPersonality) - return "incompatible personality"; + return InlineResult::failure("incompatible personality"); } // We need to figure out which funclet the callsite was in so that we may @@ -1657,7 +1657,7 @@ // for catchpads. for (const BasicBlock &CalledBB : *CalledFunc) { if (isa(CalledBB.getFirstNonPHI())) - return "catch in cleanup funclet"; + return InlineResult::failure("catch in cleanup funclet"); } } } else if (isAsynchronousEHPersonality(Personality)) { @@ -1665,7 +1665,7 @@ // funclet in the callee. for (const BasicBlock &CalledBB : *CalledFunc) { if (CalledBB.isEHPad()) - return "SEH in cleanup funclet"; + return InlineResult::failure("SEH in cleanup funclet"); } } } @@ -2275,7 +2275,7 @@ Returns[0]->eraseFromParent(); // We are now done with the inlining. - return true; + return InlineResult::success(); } // Otherwise, we have the normal case, of more than one block to inline or @@ -2437,5 +2437,5 @@ } } - return true; + return InlineResult::success(); }