diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -26,6 +26,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/None.h" +#include "llvm/ADT/SCCIterator.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" @@ -33,6 +34,8 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/CallGraph.h" +#include "llvm/Analysis/CallGraphSCCPass.h" #include "llvm/Analysis/InlineCost.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" @@ -137,6 +140,11 @@ cl::desc("For symbols in profile symbol list, regard their profiles to " "be accurate. It may be overriden by profile-sample-accurate. ")); +static cl::opt ProfileTopDownLoad( + "sample-profile-top-down-load", cl::Hidden, cl::init(false), + cl::desc("Do profile annotation and inlining for functions in top-down " + "order of call graph during sample profile loading.")); + namespace { using BlockWeightMap = DenseMap; @@ -284,7 +292,7 @@ Filename(Name), RemappingFilename(RemapName), IsThinLTOPreLink(IsThinLTOPreLink) {} - bool doInitialization(Module &M); + bool doInitialization(Module &M, CallGraph *CG); bool runOnModule(Module &M, ModuleAnalysisManager *AM, ProfileSummaryInfo *_PSI); @@ -318,6 +326,7 @@ void propagateWeights(Function &F); uint64_t visitEdge(Edge E, unsigned *NumUnknownEdges, Edge *UnknownEdge); void buildEdges(Function &F); + void buildFunctionOrder(Module &M, CallGraph *CG); bool propagateThroughEdges(Function &F, bool UpdateBlockCount); void computeDominanceAndLoopInfo(Function &F); void clearFunctionData(); @@ -417,6 +426,9 @@ }; DenseMap notInlinedCallInfo; + // Order list that dictates annotation and inlining order from sample loader + std::vector FunctionOrderList; + // GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for // all the function symbols defined or declared in current module. DenseMap GUIDToFuncNameMap; @@ -455,7 +467,7 @@ void dump() { SampleLoader.dump(); } bool doInitialization(Module &M) override { - return SampleLoader.doInitialization(M); + return SampleLoader.doInitialization(M, nullptr); } StringRef getPassName() const override { return "Sample profile pass"; } @@ -1674,7 +1686,32 @@ INITIALIZE_PASS_END(SampleProfileLoaderLegacyPass, "sample-profile", "Sample Profile loader", false, false) -bool SampleProfileLoader::doInitialization(Module &M) { +void SampleProfileLoader::buildFunctionOrder(Module &M, CallGraph *CG) { + assert(FunctionOrderList.empty()); + FunctionOrderList.reserve(M.size()); + + if (!ProfileTopDownLoad || CG == nullptr) { + for (Function &F : M) + FunctionOrderList.push_back(&F); + return; + } + + assert(&CG->getModule() == &M); + scc_iterator CGI = scc_begin(CG); + while (!CGI.isAtEnd()) { + for (CallGraphNode *node : *CGI) { + if (node->getFunction()) + FunctionOrderList.push_back(node->getFunction()); + } + ++CGI; + } + + std::reverse(FunctionOrderList.begin(), FunctionOrderList.end()); + assert(CG->getModule().size() == FunctionOrderList.size() && + "Not all functions are in order list"); +} + +bool SampleProfileLoader::doInitialization(Module &M, CallGraph *CG) { auto &Ctx = M.getContext(); std::unique_ptr RemapReader; @@ -1689,6 +1726,7 @@ Reader->collectFuncsFrom(M); ProfileIsValid = (Reader->read() == sampleprof_error::success); PSL = Reader->getProfileSymbolList(); + buildFunctionOrder(M, CG); // While profile-sample-accurate is on, ignore symbol list. ProfAccForSymsInList = @@ -1746,10 +1784,10 @@ } bool retval = false; - for (auto &F : M) - if (!F.isDeclaration()) { + for (auto F : FunctionOrderList) + if (!F->isDeclaration()) { clearFunctionData(); - retval |= runOnFunction(F, AM); + retval |= runOnFunction(*F, AM); } // Account for cold calls not inlined.... @@ -1832,6 +1870,7 @@ ModuleAnalysisManager &AM) { FunctionAnalysisManager &FAM = AM.getResult(M).getManager(); + CallGraph &CG = AM.getResult(M); auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { return FAM.getResult(F); @@ -1846,7 +1885,7 @@ : ProfileRemappingFileName, IsThinLTOPreLink, GetAssumptionCache, GetTTI); - SampleLoader.doInitialization(M); + SampleLoader.doInitialization(M, &CG); ProfileSummaryInfo *PSI = &AM.getResult(M); if (!SampleLoader.runOnModule(M, &AM, PSI)) diff --git a/llvm/test/Transforms/SampleProfile/Inputs/inline-topdown.prof b/llvm/test/Transforms/SampleProfile/Inputs/inline-topdown.prof new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/Inputs/inline-topdown.prof @@ -0,0 +1,10 @@ +main:225715:0 + 2.1: 5553 + 3: 5391 + 3.1: _Z3sumii:50000 + 1: _Z3subii:0 + 1: 0 + +_Z3sumii:6010:50000 + 1: _Z3subii:60000 + 1: 9 \ No newline at end of file diff --git a/llvm/test/Transforms/SampleProfile/inline-topdown.ll b/llvm/test/Transforms/SampleProfile/inline-topdown.ll new file mode 100644 --- /dev/null +++ b/llvm/test/Transforms/SampleProfile/inline-topdown.ll @@ -0,0 +1,127 @@ +; Note that this needs new pass manager for now. Passing `-sample-profile-top-down-load` to legacy pass manager is a no-op. + +; Test we aren't doing specialization for inlining with default source order +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -S | FileCheck -check-prefix=DEFAULT %s + +; Test we specialize based on call path with context-sensitive profile while inlining with '-sample-profile-top-down-load' +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/inline-topdown.prof -sample-profile-merge-inlinee -sample-profile-top-down-load -S | FileCheck -check-prefix=TOPDOWN %s + +@.str = private unnamed_addr constant [11 x i8] c"sum is %d\0A\00", align 1 + +; Function Attrs: nounwind uwtable +define i32 @_Z3sumii(i32 %x, i32 %y) !dbg !4 { +entry: + %x.addr = alloca i32, align 4 + %y.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + store i32 %y, i32* %y.addr, align 4 + %0 = load i32, i32* %x.addr, align 4, !dbg !11 + %1 = load i32, i32* %y.addr, align 4, !dbg !11 + %add = add nsw i32 %0, %1, !dbg !11 + %2 = load i32, i32* %x.addr, align 4, !dbg !11 + %3 = load i32, i32* %y.addr, align 4, !dbg !11 + %call = call i32 @_Z3subii(i32 %2, i32 %3), !dbg !11 + ret i32 %add, !dbg !11 +} + +; Function Attrs: nounwind uwtable +define i32 @_Z3subii(i32 %x, i32 %y) !dbg !26 { +entry: + %x.addr = alloca i32, align 4 + %y.addr = alloca i32, align 4 + store i32 %x, i32* %x.addr, align 4 + store i32 %y, i32* %y.addr, align 4 + %0 = load i32, i32* %x.addr, align 4, !dbg !27 + %1 = load i32, i32* %y.addr, align 4, !dbg !27 + %add = sub nsw i32 %0, %1, !dbg !27 + ret i32 %add, !dbg !28 +} + +; Function Attrs: uwtable +define i32 @main() !dbg !7 { +entry: + %retval = alloca i32, align 4 + %s = alloca i32, align 4 + %i = alloca i32, align 4 + store i32 0, i32* %retval + store i32 0, i32* %i, align 4, !dbg !12 + br label %while.cond, !dbg !13 + +while.cond: ; preds = %if.end, %entry + %0 = load i32, i32* %i, align 4, !dbg !14 + %inc = add nsw i32 %0, 1, !dbg !14 + store i32 %inc, i32* %i, align 4, !dbg !14 + %cmp = icmp slt i32 %0, 400000000, !dbg !14 + br i1 %cmp, label %while.body, label %while.end, !dbg !14 + +while.body: ; preds = %while.cond + %1 = load i32, i32* %i, align 4, !dbg !16 + %cmp1 = icmp ne i32 %1, 100, !dbg !16 + br i1 %cmp1, label %if.then, label %if.else, !dbg !16 + + +if.then: ; preds = %while.body + %2 = load i32, i32* %i, align 4, !dbg !18 + %3 = load i32, i32* %s, align 4, !dbg !18 + %call = call i32 @_Z3sumii(i32 %2, i32 %3), !dbg !18 +; CHECK: call i32 @_Z3sumii + store i32 %call, i32* %s, align 4, !dbg !18 + br label %if.end, !dbg !18 + +if.else: ; preds = %while.body + store i32 30, i32* %s, align 4, !dbg !20 + br label %if.end + +if.end: ; preds = %if.else, %if.then + br label %while.cond, !dbg !22 + +while.end: ; preds = %while.cond + %4 = load i32, i32* %s, align 4, !dbg !24 + %call2 = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([11 x i8], [11 x i8]* @.str, i32 0, i32 0), i32 %4), !dbg !24 + ret i32 0, !dbg !25 +} + +declare i32 @printf(i8*, ...) #2 + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!8, !9} +!llvm.ident = !{!10} + +!0 = distinct !DICompileUnit(language: DW_LANG_C_plus_plus, producer: "clang version 3.5 ", isOptimized: false, emissionKind: NoDebug, file: !1, enums: !2, retainedTypes: !2, globals: !2, imports: !2) +!1 = !DIFile(filename: "calls.cc", directory: ".") +!2 = !{} +!4 = distinct !DISubprogram(name: "sum", line: 3, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 3, file: !1, scope: !5, type: !6, retainedNodes: !2) +!5 = !DIFile(filename: "calls.cc", directory: ".") +!6 = !DISubroutineType(types: !2) +!7 = distinct !DISubprogram(name: "main", line: 7, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 7, file: !1, scope: !5, type: !6, retainedNodes: !2) +!8 = !{i32 2, !"Dwarf Version", i32 4} +!9 = !{i32 1, !"Debug Info Version", i32 3} +!10 = !{!"clang version 3.5 "} +!11 = !DILocation(line: 4, scope: !4) +!12 = !DILocation(line: 8, scope: !7) +!13 = !DILocation(line: 9, scope: !7) +!14 = !DILocation(line: 9, scope: !15) +!15 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !7) +!16 = !DILocation(line: 10, scope: !17) +!17 = distinct !DILexicalBlock(line: 10, column: 0, file: !1, scope: !7) +!18 = !DILocation(line: 10, scope: !19) +!19 = !DILexicalBlockFile(discriminator: 2, file: !1, scope: !17) +!20 = !DILocation(line: 10, scope: !21) +!21 = !DILexicalBlockFile(discriminator: 4, file: !1, scope: !17) +!22 = !DILocation(line: 10, scope: !23) +!23 = !DILexicalBlockFile(discriminator: 6, file: !1, scope: !17) +!24 = !DILocation(line: 11, scope: !7) +!25 = !DILocation(line: 12, scope: !7) +!26 = distinct !DISubprogram(name: "sub", line: 20, isLocal: false, isDefinition: true, virtualIndex: 6, flags: DIFlagPrototyped, isOptimized: false, unit: !0, scopeLine: 20, file: !1, scope: !5, type: !6, retainedNodes: !2) +!27 = !DILocation(line: 20, scope: !26) +!28 = !DILocation(line: 21, scope: !26) + +; DEFAULT: @_Z3sumii +; DEFAULT-NOT: call i32 @_Z3subii +; DEFAULT: @main() +; DEFAULT-NOT: call i32 @_Z3subii + +; TOPDOWN: @_Z3sumii +; TOPDOWN-NOT: call i32 @_Z3subii +; TOPDOWN: @main() +; TOPDOWN: call i32 @_Z3subii \ No newline at end of file