Index: lib/Analysis/InlineCost.cpp =================================================================== --- lib/Analysis/InlineCost.cpp +++ lib/Analysis/InlineCost.cpp @@ -840,11 +840,11 @@ } void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { - // If no size growth is allowed for this inlining, set Threshold to 0. - if (!allowSizeGrowth(CS)) { + // If no size growth is allowed in this round of inlining, set the threshold + // to 0. We only want to inline if it has 0 cost or better in this case. + bool AllowSizeGrowth = allowSizeGrowth(CS); + if (!AllowSizeGrowth) Threshold = 0; - return; - } Function *Caller = CS.getCaller(); @@ -890,72 +890,79 @@ LastCallToStaticBonus = 0; }; - // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are available - // and reduce the threshold if the caller has the necessary attribute. - if (Caller->optForMinSize()) { - Threshold = MinIfValid(Threshold, Params.OptMinSizeThreshold); - // For minsize, we want to disable the single BB bonus and the vector - // bonuses, but not the last-call-to-static bonus. Inlining the last call to - // a static function will, at the minimum, eliminate the parameter setup and - // call/return instructions. - SingleBBBonusPercent = 0; - VectorBonusPercent = 0; - } else if (Caller->optForSize()) - Threshold = MinIfValid(Threshold, Params.OptSizeThreshold); - - // Adjust the threshold based on inlinehint attribute and profile based - // hotness information if the caller does not have MinSize attribute. - if (!Caller->optForMinSize()) { - if (Callee.hasFnAttribute(Attribute::InlineHint)) - Threshold = MaxIfValid(Threshold, Params.HintThreshold); - - // FIXME: After switching to the new passmanager, simplify the logic below - // by checking only the callsite hotness/coldness as we will reliably - // have local profile information. - // - // Callsite hotness and coldness can be determined if sample profile is - // used (which adds hotness metadata to calls) or if caller's - // BlockFrequencyInfo is available. - BlockFrequencyInfo *CallerBFI = GetBFI ? &((*GetBFI)(*Caller)) : nullptr; - auto HotCallSiteThreshold = getHotCallSiteThreshold(CS, CallerBFI); - if (!Caller->optForSize() && HotCallSiteThreshold) { - DEBUG(dbgs() << "Hot callsite.\n"); - // FIXME: This should update the threshold only if it exceeds the - // current threshold, but AutoFDO + ThinLTO currently relies on this - // behavior to prevent inlining of hot callsites during ThinLTO - // compile phase. - Threshold = HotCallSiteThreshold.getValue(); - } else if (isColdCallSite(CS, CallerBFI)) { - DEBUG(dbgs() << "Cold callsite.\n"); - // Do not apply bonuses for a cold callsite including the - // LastCallToStatic bonus. While this bonus might result in code size - // reduction, it can cause the size of a non-cold caller to increase - // preventing it from being inlined. - DisallowAllBonuses(); - Threshold = MinIfValid(Threshold, Params.ColdCallSiteThreshold); - } else if (PSI) { - // Use callee's global profile information only if we have no way of - // determining this via callsite information. - if (PSI->isFunctionEntryHot(&Callee)) { - DEBUG(dbgs() << "Hot callee.\n"); - // If callsite hotness can not be determined, we may still know - // that the callee is hot and treat it as a weaker hint for threshold - // increase. + // If we're allowed to increase size, then bump the threshold up and down + // based off properties of the caller. Otherwise, skip ahead and finalize the + // cost and threshold. + if (AllowSizeGrowth) { + // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are + // available and reduce the threshold if the caller has the necessary + // attribute. + if (Caller->optForMinSize()) { + Threshold = MinIfValid(Threshold, Params.OptMinSizeThreshold); + // For minsize, we want to disable the single BB bonus and the vector + // bonuses, but not the last-call-to-static bonus. Inlining the last call + // to a static function will, at the minimum, eliminate the parameter + // setup and call/return instructions. + SingleBBBonusPercent = 0; + VectorBonusPercent = 0; + } else if (Caller->optForSize()) + Threshold = MinIfValid(Threshold, Params.OptSizeThreshold); + + // Adjust the threshold based on inlinehint attribute and profile based + // hotness information if the caller does not have MinSize attribute. + if (!Caller->optForMinSize()) { + if (Callee.hasFnAttribute(Attribute::InlineHint)) Threshold = MaxIfValid(Threshold, Params.HintThreshold); - } else if (PSI->isFunctionEntryCold(&Callee)) { - DEBUG(dbgs() << "Cold callee.\n"); - // Do not apply bonuses for a cold callee including the + + // FIXME: After switching to the new passmanager, simplify the logic below + // by checking only the callsite hotness/coldness as we will reliably + // have local profile information. + // + // Callsite hotness and coldness can be determined if sample profile is + // used (which adds hotness metadata to calls) or if caller's + // BlockFrequencyInfo is available. + BlockFrequencyInfo *CallerBFI = GetBFI ? &((*GetBFI)(*Caller)) : nullptr; + auto HotCallSiteThreshold = getHotCallSiteThreshold(CS, CallerBFI); + if (!Caller->optForSize() && HotCallSiteThreshold) { + DEBUG(dbgs() << "Hot callsite.\n"); + // FIXME: This should update the threshold only if it exceeds the + // current threshold, but AutoFDO + ThinLTO currently relies on this + // behavior to prevent inlining of hot callsites during ThinLTO + // compile phase. + Threshold = HotCallSiteThreshold.getValue(); + } else if (isColdCallSite(CS, CallerBFI)) { + DEBUG(dbgs() << "Cold callsite.\n"); + // Do not apply bonuses for a cold callsite including the // LastCallToStatic bonus. While this bonus might result in code size // reduction, it can cause the size of a non-cold caller to increase // preventing it from being inlined. DisallowAllBonuses(); - Threshold = MinIfValid(Threshold, Params.ColdThreshold); + Threshold = MinIfValid(Threshold, Params.ColdCallSiteThreshold); + } else if (PSI) { + // Use callee's global profile information only if we have no way of + // determining this via callsite information. + if (PSI->isFunctionEntryHot(&Callee)) { + DEBUG(dbgs() << "Hot callee.\n"); + // If callsite hotness can not be determined, we may still know + // that the callee is hot and treat it as a weaker hint for threshold + // increase. + Threshold = MaxIfValid(Threshold, Params.HintThreshold); + } else if (PSI->isFunctionEntryCold(&Callee)) { + DEBUG(dbgs() << "Cold callee.\n"); + // Do not apply bonuses for a cold callee including the + // LastCallToStatic bonus. While this bonus might result in code size + // reduction, it can cause the size of a non-cold caller to increase + // preventing it from being inlined. + DisallowAllBonuses(); + Threshold = MinIfValid(Threshold, Params.ColdThreshold); + } } } } // Finally, take the target-specific inlining threshold multiplier into // account. + // Note that Threshold = 0 when size growth is not allowed. Threshold *= TTI.getInliningThresholdMultiplier(); SingleBBBonus = Threshold * SingleBBBonusPercent / 100; Index: test/Transforms/Inline/ARM/unreachable.ll =================================================================== --- /dev/null +++ test/Transforms/Inline/ARM/unreachable.ll @@ -0,0 +1,21 @@ +; RUN: opt < %s -mtriple=arm--- -S -inline | FileCheck %s + +declare i32* @pluto() local_unnamed_addr #0 + +define internal void @foo() local_unnamed_addr #0 { +bb: + call i32* @pluto() #0 + ret void +} + +define void @wibble() unnamed_addr #0 { +; CHECK-LABEL: wibble +; CHECK-NOT: @foo() +; CHECK: %0 = call i32* @pluto() +; CHECK-NEXT: unreachable +bb: + tail call void @foo() #0 + unreachable +} + +attributes #0 = { minsize optsize }