Index: lib/Analysis/InlineCost.cpp =================================================================== --- lib/Analysis/InlineCost.cpp +++ lib/Analysis/InlineCost.cpp @@ -159,6 +159,13 @@ /// Bonus to be applied when the callee has only one reachable basic block. int SingleBBBonus = 0; + /// When we are able to prove that inlining the callee will make inlining + /// of a particular nested call-site profitable, we want to give a certain + /// bonus with size of a callee deducted. + /// Tracking the actually applied bonus here, as well as adding it directly + /// to Threshold. + int InlinedCallBonus = 0; + /// While we walk the potentially-inlined instructions, we build up and /// maintain a mapping of simplified values specific to this callsite. The /// idea is to propagate any special information we have about arguments to @@ -248,6 +255,13 @@ Cost = std::min(UpperBound, Cost + Inc); } + /// Apply a positive bonus to Threshold for a proven inlining of callee's callee. + void applyInlinedCallBonus(int Bonus) { + assert(Bonus >= 0); + Threshold += Bonus; + InlinedCallBonus += Bonus; + } + // Disable several entry points to the visitor so we don't accidentally use // them by declaring but not defining them here. void visit(Module *); @@ -894,7 +908,15 @@ // and the callsite. int SingleBBBonusPercent = 50; int VectorBonusPercent = 150; - int LastCallToStaticBonus = InlineConstants::LastCallToStaticBonus; + + int LastCallToStaticBonus = 0; + bool OnlyOneCallAndLocalLinkage = + F.hasLocalLinkage() && F.hasOneUse() && &F == CS.getCalledFunction(); + // If there is only one call of the function, and it has internal linkage, + // we can allow to inline pretty anything as it will lead to size reduction + // anyway. + if (OnlyOneCallAndLocalLinkage) + LastCallToStaticBonus = InlineConstants::LastCallToStaticBonus; // Lambda to set all the above bonus and bonus percentages to 0. auto DisallowAllBonuses = [&]() { @@ -967,20 +989,13 @@ } } - // Finally, take the target-specific inlining threshold multiplier into - // account. + // Take the target-specific inlining threshold multiplier into account. Threshold *= TTI.getInliningThresholdMultiplier(); SingleBBBonus = Threshold * SingleBBBonusPercent / 100; VectorBonus = Threshold * VectorBonusPercent / 100; - bool OnlyOneCallAndLocalLinkage = - F.hasLocalLinkage() && F.hasOneUse() && &F == CS.getCalledFunction(); - // If there is only one call of the function, and it has internal linkage, - // the cost of inlining it drops dramatically. It may seem odd to update - // Cost in updateThreshold, but the bonus depends on the logic in this method. - if (OnlyOneCallAndLocalLinkage) - Cost -= LastCallToStaticBonus; + Threshold += LastCallToStaticBonus; } bool CallAnalyzer::visitCmpInst(CmpInst &I) { @@ -1305,9 +1320,10 @@ CallAnalyzer CA(TTI, GetAssumptionCache, GetBFI, PSI, ORE, *F, CS, IndirectCallParams); if (CA.analyzeCall(CS)) { - // We were able to inline the indirect call! Subtract the cost from the - // threshold to get the bonus we want to apply, but don't go below zero. - Cost -= std::max(0, CA.getThreshold() - CA.getCost()); + // We were able to inline the indirect call! Increase the threshold + // with the bonus we want to apply (less the cost of inlinee). + // Make sure the bonus doesn't go below zero. + applyInlinedCallBonus(std::max(0, CA.getThreshold() - CA.getCost())); } if (!F->onlyReadsMemory()) Index: test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll =================================================================== --- test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll +++ test/LTO/Resolution/X86/diagnostic-handler-remarks-with-hotness.ll @@ -27,13 +27,13 @@ ; YAML-NEXT: - Caller: main ; YAML-NEXT: - String: ' with ' ; YAML-NEXT: - String: '(cost=' -; YAML-NEXT: - Cost: '-15000' +; YAML-NEXT: - Cost: '0' ; YAML-NEXT: - String: ', threshold=' -; YAML-NEXT: - Threshold: '337' +; YAML-NEXT: - Threshold: '15337' ; YAML-NEXT: - String: ')' ; YAML-NEXT: ... -; CHECK: tinkywinky inlined into main with (cost=-15000, threshold=337) (hotness: 300) +; CHECK: tinkywinky inlined into main with (cost=0, threshold=15337) (hotness: 300) target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-scei-ps4" Index: test/LTO/Resolution/X86/diagnostic-handler-remarks.ll =================================================================== --- test/LTO/Resolution/X86/diagnostic-handler-remarks.ll +++ test/LTO/Resolution/X86/diagnostic-handler-remarks.ll @@ -30,9 +30,9 @@ ; YAML-NEXT: - Caller: main ; YAML-NEXT: - String: ' with ' ; YAML-NEXT: - String: '(cost=' -; YAML-NEXT: - Cost: '-15000' +; YAML-NEXT: - Cost: '0' ; YAML-NEXT: - String: ', threshold=' -; YAML-NEXT: - Threshold: '337' +; YAML-NEXT: - Threshold: '15337' ; YAML-NEXT: - String: ')' ; YAML-NEXT: ... Index: test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll =================================================================== --- test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll +++ test/LTO/X86/diagnostic-handler-remarks-with-hotness.ll @@ -19,9 +19,9 @@ ; YAML-NEXT: - Caller: main ; YAML-NEXT: - String: ' with ' ; YAML-NEXT: - String: '(cost=' -; YAML-NEXT: - Cost: '-15000' +; YAML-NEXT: - Cost: '0' ; YAML-NEXT: - String: ', threshold=' -; YAML-NEXT: - Threshold: '337' +; YAML-NEXT: - Threshold: '15337' ; YAML-NEXT: - String: ')' ; YAML-NEXT: ... Index: test/LTO/X86/diagnostic-handler-remarks.ll =================================================================== --- test/LTO/X86/diagnostic-handler-remarks.ll +++ test/LTO/X86/diagnostic-handler-remarks.ll @@ -55,9 +55,9 @@ ; YAML-NEXT: - Caller: main ; YAML-NEXT: - String: ' with ' ; YAML-NEXT: - String: '(cost=' -; YAML-NEXT: - Cost: '-15000' +; YAML-NEXT: - Cost: '0' ; YAML-NEXT: - String: ', threshold=' -; YAML-NEXT: - Threshold: '337' +; YAML-NEXT: - Threshold: '15337' ; YAML-NEXT: - String: ')' ; YAML-NEXT: ... Index: test/Transforms/Inline/ARM/inline-fp.ll =================================================================== --- test/Transforms/Inline/ARM/inline-fp.ll +++ test/Transforms/Inline/ARM/inline-fp.ll @@ -7,25 +7,25 @@ ; NOFP-DAG: single not inlined into test_single because too costly to inline (cost=125, threshold=75) ; NOFP-DAG: single not inlined into test_single because too costly to inline (cost=125, threshold=75) ; NOFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15, threshold=75) -; NOFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15015, threshold=75) +; NOFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15, threshold=15075) ; NOFP-DAG: double not inlined into test_double because too costly to inline (cost=125, threshold=75) ; NOFP-DAG: double not inlined into test_double because too costly to inline (cost=125, threshold=75) ; NOFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75) ; NOFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75) ; FULLFP-DAG: single inlined into test_single with (cost=0, threshold=75) -; FULLFP-DAG: single inlined into test_single with (cost=-15000, threshold=75) +; FULLFP-DAG: single inlined into test_single with (cost=0, threshold=15075) ; FULLFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15, threshold=75) -; FULLFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15015, threshold=75) +; FULLFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15, threshold=15075) ; FULLFP-DAG: double inlined into test_double with (cost=0, threshold=75) -; FULLFP-DAG: double inlined into test_double with (cost=-15000, threshold=75) +; FULLFP-DAG: double inlined into test_double with (cost=0, threshold=15075) ; FULLFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75) ; FULLFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75) ; SINGLEFP-DAG: single inlined into test_single with (cost=0, threshold=75) -; SINGLEFP-DAG: single inlined into test_single with (cost=-15000, threshold=75) +; SINGLEFP-DAG: single inlined into test_single with (cost=0, threshold=15075) ; SINGLEFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15, threshold=75) -; SINGLEFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15015, threshold=75) +; SINGLEFP-DAG: single_cheap inlined into test_single_cheap with (cost=-15, threshold=15075) ; SINGLEFP-DAG: double not inlined into test_double because too costly to inline (cost=125, threshold=75) ; SINGLEFP-DAG: double not inlined into test_double because too costly to inline (cost=125, threshold=75) ; SINGLEFP-DAG: single_force_soft not inlined into test_single_force_soft because too costly to inline (cost=125, threshold=75)