diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h --- a/llvm/include/llvm/ProfileData/SampleProf.h +++ b/llvm/include/llvm/ProfileData/SampleProf.h @@ -442,6 +442,7 @@ StringRef getName() const { return Name; } StringRef getCallingContext() const { return CallingContext; } StringRef getNameWithContext() const { return FullContext; } + StringRef getInputNameWithContext() const { return InputContext; } private: // Give a context string, decode and populate internal states like @@ -449,6 +450,7 @@ // `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]` void setContext(StringRef ContextStr, ContextStateMask CState) { assert(!ContextStr.empty()); + InputContext = ContextStr; // Note that `[]` wrapped input indicates a full context string, otherwise // it's treated as context-less function name only. bool HasContext = ContextStr.startswith("["); @@ -480,6 +482,9 @@ } } + // Input context string including bracketed calling context and leaf function + // name + StringRef InputContext; // Full context string including calling context and leaf function name StringRef FullContext; // Function name for the associated sample profile @@ -676,7 +681,8 @@ Name = Other.getName(); if (!GUIDToFuncNameMap) GUIDToFuncNameMap = Other.GUIDToFuncNameMap; - + if (Context.getInputNameWithContext().empty()) + Context = Other.getContext(); if (FunctionHash == 0) { // Set the function hash code for the target profile. FunctionHash = Other.getFunctionHash(); @@ -747,6 +753,12 @@ return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name; } + /// Return the input string of function name with context. + StringRef getInputNameWithContext() const { + return FunctionSamples::ProfileIsCS ? Context.getInputNameWithContext() + : Name; + } + /// Return the original function name. StringRef getFuncName() const { return getFuncName(Name); } diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h --- a/llvm/include/llvm/ProfileData/SampleProfReader.h +++ b/llvm/include/llvm/ProfileData/SampleProfReader.h @@ -488,8 +488,15 @@ /// \brief Whether samples are collected based on pseudo probes. bool ProfileIsProbeBased = false; + /// Whether function profiles are context-sensitive. bool ProfileIsCS = false; + /// Number of context-sensitive profiles. + int CSProfileCount = 0; + + /// Number of non-context-sensitive profiles. + int NonCSProfileCount = 0; + /// \brief The format of sample. SampleProfileFormat Format = SPF_None; }; diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp --- a/llvm/lib/ProfileData/SampleProfReader.cpp +++ b/llvm/lib/ProfileData/SampleProfReader.cpp @@ -222,8 +222,6 @@ sampleprof_error Result = sampleprof_error::success; InlineCallStack InlineStack; - int CSProfileCount = 0; - int RegularProfileCount = 0; uint32_t ProbeProfileCount = 0; // SeenMetadata tracks whether we have processed metadata for the current @@ -258,7 +256,7 @@ if (FContext.hasContext()) ++CSProfileCount; else - ++RegularProfileCount; + ++NonCSProfileCount; Profiles[FContext] = FunctionSamples(); FunctionSamples &FProfile = Profiles[FContext]; FProfile.setName(FContext.getName()); @@ -324,13 +322,14 @@ } } - assert((RegularProfileCount == 0 || CSProfileCount == 0) && + assert((NonCSProfileCount == 0 || CSProfileCount == 0) && "Cannot have both context-sensitive and regular profile"); ProfileIsCS = (CSProfileCount > 0); assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) && "Cannot have both probe-based profiles and regular profiles"); ProfileIsProbeBased = (ProbeProfileCount > 0); FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased; + FunctionSamples::ProfileIsCS = ProfileIsCS; if (Result == sampleprof_error::success) computeSummary(); @@ -546,12 +545,18 @@ if (std::error_code EC = FName.getError()) return EC; - Profiles[*FName] = FunctionSamples(); - FunctionSamples &FProfile = Profiles[*FName]; - FProfile.setName(*FName); - + SampleContext FContext(*FName); + Profiles[FContext] = FunctionSamples(); + FunctionSamples &FProfile = Profiles[FContext]; + FProfile.setName(FContext.getName()); + FProfile.setContext(FContext); FProfile.addHeadSamples(*NumHeadSamples); + if (FContext.hasContext()) + CSProfileCount++; + else + NonCSProfileCount++; + if (std::error_code EC = readProfile(FProfile)) return EC; return sampleprof_error::success; @@ -654,40 +659,44 @@ return EC; } assert(Data == End && "More data is read than expected"); - return sampleprof_error::success; - } - - if (Remapper) { - for (auto Name : FuncsToUse) { - Remapper->insert(Name); + } else { + if (Remapper) { + for (auto Name : FuncsToUse) { + Remapper->insert(Name); + } } - } - if (useMD5()) { - for (auto Name : FuncsToUse) { - auto GUID = std::to_string(MD5Hash(Name)); - auto iter = FuncOffsetTable.find(StringRef(GUID)); - if (iter == FuncOffsetTable.end()) - continue; - const uint8_t *FuncProfileAddr = Start + iter->second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; - } - } else { - for (auto NameOffset : FuncOffsetTable) { - auto FuncName = NameOffset.first; - if (!FuncsToUse.count(FuncName) && - (!Remapper || !Remapper->exist(FuncName))) - continue; - const uint8_t *FuncProfileAddr = Start + NameOffset.second; - assert(FuncProfileAddr < End && "out of LBRProfile section"); - if (std::error_code EC = readFuncProfile(FuncProfileAddr)) - return EC; + if (useMD5()) { + for (auto Name : FuncsToUse) { + auto GUID = std::to_string(MD5Hash(Name)); + auto iter = FuncOffsetTable.find(StringRef(GUID)); + if (iter == FuncOffsetTable.end()) + continue; + const uint8_t *FuncProfileAddr = Start + iter->second; + assert(FuncProfileAddr < End && "out of LBRProfile section"); + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + } + } else { + for (auto NameOffset : FuncOffsetTable) { + SampleContext FContext(NameOffset.first); + auto FuncName = FContext.getName(); + if (!FuncsToUse.count(FuncName) && + (!Remapper || !Remapper->exist(FuncName))) + continue; + const uint8_t *FuncProfileAddr = Start + NameOffset.second; + assert(FuncProfileAddr < End && "out of LBRProfile section"); + if (std::error_code EC = readFuncProfile(FuncProfileAddr)) + return EC; + } } + Data = End; } - Data = End; + assert((NonCSProfileCount == 0 || CSProfileCount == 0) && + "Cannot have both context-sensitive and regular profile"); + ProfileIsCS = (CSProfileCount > 0); + FunctionSamples::ProfileIsCS = ProfileIsCS; return sampleprof_error::success; } @@ -887,7 +896,8 @@ if (std::error_code EC = Checksum.getError()) return EC; - Profiles[*FName].setFunctionHash(*Checksum); + SampleContext FContext(*FName); + Profiles[FContext].setFunctionHash(*Checksum); } return sampleprof_error::success; } diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp --- a/llvm/lib/ProfileData/SampleProfWriter.cpp +++ b/llvm/lib/ProfileData/SampleProfWriter.cpp @@ -147,7 +147,7 @@ std::error_code SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) { uint64_t Offset = OutputStream->tell(); - StringRef Name = S.getName(); + StringRef Name = S.getInputNameWithContext(); FuncOffsetTable[Name] = Offset - SecLBRProfileStart; encodeULEB128(S.getHeadSamples(), *OutputStream); return writeBody(S); @@ -635,7 +635,7 @@ std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) { auto &OS = *OutputStream; - if (std::error_code EC = writeNameIdx(S.getName())) + if (std::error_code EC = writeNameIdx(S.getInputNameWithContext())) return EC; encodeULEB128(S.getTotalSamples(), OS); diff --git a/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll b/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll --- a/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll +++ b/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll @@ -1,18 +1,22 @@ ; Test for CSSPGO's SampleContextTracker to make sure context profile tree is promoted and merged properly ; based on inline decision, so post inline counts are accurate. +; RUN: llvm-profdata merge --sample --extbinary %S/Inputs/profile-context-tracker.prof -o %t + ; Note that we need new pass manager to enable top-down processing for sample profile loader ; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile ; main:3 @ _Z5funcAi ; main:3 @ _Z5funcAi:1 @ _Z8funcLeafi ; _Z5funcBi:1 @ _Z8funcLeafi ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -sample-profile-inline-size -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-ALL +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t -sample-profile-inline-size -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-ALL ; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile ; main:3 @ _Z5funcAi ; _Z5funcAi:1 @ _Z8funcLeafi ; _Z5funcBi:1 @ _Z8funcLeafi ; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-HOT +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-HOT @factor = dso_local global i32 3, align 4, !dbg !0 diff --git a/llvm/test/tools/llvm-profdata/Inputs/csspgo-profile.proftext b/llvm/test/tools/llvm-profdata/Inputs/csspgo-profile.proftext new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/Inputs/csspgo-profile.proftext @@ -0,0 +1,4 @@ +[main:3.1 @ _Z5funcBi]:120:19 + 0: 19 + 1: 19 _Z8funcLeafi:19 + 3: 12 \ No newline at end of file diff --git a/llvm/test/tools/llvm-profdata/merge-cs-profile.test b/llvm/test/tools/llvm-profdata/merge-cs-profile.test new file mode 100644 --- /dev/null +++ b/llvm/test/tools/llvm-profdata/merge-cs-profile.test @@ -0,0 +1,15 @@ +# Tests for merge of context-sensitive profile files. + +RUN: llvm-profdata merge --sample --text %p/Inputs/csspgo-profile.proftext -o - | FileCheck %s --check-prefix=MERGE1 +RUN: llvm-profdata merge --sample --extbinary %p/Inputs/csspgo-profile.proftext -o %t && llvm-profdata merge --sample --text %t -o - | FileCheck %s --check-prefix=MERGE1 +MERGE1: [main:3.1 @ _Z5funcBi]:120:19 +MERGE1: 0: 19 +MERGE1: 1: 19 _Z8funcLeafi:19 +MERGE1: 3: 12 + +RUN: llvm-profdata merge --sample --text %p/Inputs/csspgo-profile.proftext %p/Inputs/csspgo-profile.proftext -o - | FileCheck %s --check-prefix=MERGE2 +RUN: llvm-profdata merge --sample --extbinary %p/Inputs/csspgo-profile.proftext %p/Inputs/csspgo-profile.proftext -o %t && llvm-profdata merge --sample --text %t -o - | FileCheck %s --check-prefix=MERGE2 +MERGE2: [main:3.1 @ _Z5funcBi]:240:38 +MERGE2: 0: 38 +MERGE2: 1: 38 _Z8funcLeafi:38 +MERGE2: 3: 24 diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp --- a/llvm/tools/llvm-profdata/llvm-profdata.cpp +++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp @@ -696,7 +696,7 @@ Remapper ? remapSamples(I->second, *Remapper, Result) : FunctionSamples(); FunctionSamples &Samples = Remapper ? Remapped : I->second; - StringRef FName = Samples.getName(); + StringRef FName = Samples.getInputNameWithContext(); MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight)); if (Result != sampleprof_error::success) { std::error_code EC = make_error_code(Result);