diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -439,9 +439,11 @@
void clearState(ContextStateMask S) { State &= (uint32_t)~S; }
bool hasContext() const { return State != UnknownContext; }
bool isBaseContext() const { return CallingContext.empty(); }
- StringRef getName() const { return Name; }
+ StringRef getNameWithoutContext() const { return Name; }
StringRef getCallingContext() const { return CallingContext; }
- StringRef getNameWithContext() const { return FullContext; }
+ StringRef getNameWithContext(bool WithBracket = false) const {
+ return WithBracket ? InputContext : FullContext;
+ }
private:
// Give a context string, decode and populate internal states like
@@ -449,6 +451,7 @@
// `ContextStr`: `[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]`
void setContext(StringRef ContextStr, ContextStateMask CState) {
assert(!ContextStr.empty());
+ InputContext = ContextStr;
// Note that `[]` wrapped input indicates a full context string, otherwise
// it's treated as context-less function name only.
bool HasContext = ContextStr.startswith("[");
@@ -480,6 +483,9 @@
}
}
+ // Input context string including bracketed calling context and leaf function
+ // name
+ StringRef InputContext;
// Full context string including calling context and leaf function name
StringRef FullContext;
// Function name for the associated sample profile
@@ -676,7 +682,8 @@
Name = Other.getName();
if (!GUIDToFuncNameMap)
GUIDToFuncNameMap = Other.GUIDToFuncNameMap;
-
+ if (Context.getNameWithContext(true).empty())
+ Context = Other.getContext();
if (FunctionHash == 0) {
// Set the function hash code for the target profile.
FunctionHash = Other.getFunctionHash();
@@ -743,8 +750,10 @@
StringRef getName() const { return Name; }
/// Return function name with context.
- StringRef getNameWithContext() const {
- return FunctionSamples::ProfileIsCS ? Context.getNameWithContext() : Name;
+ StringRef getNameWithContext(bool WithBracket = false) const {
+ return FunctionSamples::ProfileIsCS
+ ? Context.getNameWithContext(WithBracket)
+ : Name;
}
/// Return the original function name.
diff --git a/llvm/include/llvm/ProfileData/SampleProfReader.h b/llvm/include/llvm/ProfileData/SampleProfReader.h
--- a/llvm/include/llvm/ProfileData/SampleProfReader.h
+++ b/llvm/include/llvm/ProfileData/SampleProfReader.h
@@ -488,8 +488,12 @@
/// \brief Whether samples are collected based on pseudo probes.
bool ProfileIsProbeBased = false;
+ /// Whether function profiles are context-sensitive.
bool ProfileIsCS = false;
+ /// Number of context-sensitive profiles.
+ uint32_t CSProfileCount = 0;
+
/// \brief The format of sample.
SampleProfileFormat Format = SPF_None;
};
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -222,8 +222,6 @@
sampleprof_error Result = sampleprof_error::success;
InlineCallStack InlineStack;
- int CSProfileCount = 0;
- int RegularProfileCount = 0;
uint32_t ProbeProfileCount = 0;
// SeenMetadata tracks whether we have processed metadata for the current
@@ -257,11 +255,9 @@
SampleContext FContext(FName);
if (FContext.hasContext())
++CSProfileCount;
- else
- ++RegularProfileCount;
Profiles[FContext] = FunctionSamples();
FunctionSamples &FProfile = Profiles[FContext];
- FProfile.setName(FContext.getName());
+ FProfile.setName(FContext.getNameWithoutContext());
FProfile.setContext(FContext);
MergeResult(Result, FProfile.addTotalSamples(NumSamples));
MergeResult(Result, FProfile.addHeadSamples(NumHeadSamples));
@@ -324,13 +320,14 @@
}
}
- assert((RegularProfileCount == 0 || CSProfileCount == 0) &&
+ assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
"Cannot have both context-sensitive and regular profile");
ProfileIsCS = (CSProfileCount > 0);
assert((ProbeProfileCount == 0 || ProbeProfileCount == Profiles.size()) &&
"Cannot have both probe-based profiles and regular profiles");
ProfileIsProbeBased = (ProbeProfileCount > 0);
FunctionSamples::ProfileIsProbeBased = ProfileIsProbeBased;
+ FunctionSamples::ProfileIsCS = ProfileIsCS;
if (Result == sampleprof_error::success)
computeSummary();
@@ -546,12 +543,16 @@
if (std::error_code EC = FName.getError())
return EC;
- Profiles[*FName] = FunctionSamples();
- FunctionSamples &FProfile = Profiles[*FName];
- FProfile.setName(*FName);
-
+ SampleContext FContext(*FName);
+ Profiles[FContext] = FunctionSamples();
+ FunctionSamples &FProfile = Profiles[FContext];
+ FProfile.setName(FContext.getNameWithoutContext());
+ FProfile.setContext(FContext);
FProfile.addHeadSamples(*NumHeadSamples);
+ if (FContext.hasContext())
+ CSProfileCount++;
+
if (std::error_code EC = readProfile(FProfile))
return EC;
return sampleprof_error::success;
@@ -654,40 +655,44 @@
return EC;
}
assert(Data == End && "More data is read than expected");
- return sampleprof_error::success;
- }
-
- if (Remapper) {
- for (auto Name : FuncsToUse) {
- Remapper->insert(Name);
+ } else {
+ if (Remapper) {
+ for (auto Name : FuncsToUse) {
+ Remapper->insert(Name);
+ }
}
- }
- if (useMD5()) {
- for (auto Name : FuncsToUse) {
- auto GUID = std::to_string(MD5Hash(Name));
- auto iter = FuncOffsetTable.find(StringRef(GUID));
- if (iter == FuncOffsetTable.end())
- continue;
- const uint8_t *FuncProfileAddr = Start + iter->second;
- assert(FuncProfileAddr < End && "out of LBRProfile section");
- if (std::error_code EC = readFuncProfile(FuncProfileAddr))
- return EC;
- }
- } else {
- for (auto NameOffset : FuncOffsetTable) {
- auto FuncName = NameOffset.first;
- if (!FuncsToUse.count(FuncName) &&
- (!Remapper || !Remapper->exist(FuncName)))
- continue;
- const uint8_t *FuncProfileAddr = Start + NameOffset.second;
- assert(FuncProfileAddr < End && "out of LBRProfile section");
- if (std::error_code EC = readFuncProfile(FuncProfileAddr))
- return EC;
+ if (useMD5()) {
+ for (auto Name : FuncsToUse) {
+ auto GUID = std::to_string(MD5Hash(Name));
+ auto iter = FuncOffsetTable.find(StringRef(GUID));
+ if (iter == FuncOffsetTable.end())
+ continue;
+ const uint8_t *FuncProfileAddr = Start + iter->second;
+ assert(FuncProfileAddr < End && "out of LBRProfile section");
+ if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+ return EC;
+ }
+ } else {
+ for (auto NameOffset : FuncOffsetTable) {
+ SampleContext FContext(NameOffset.first);
+ auto FuncName = FContext.getNameWithoutContext();
+ if (!FuncsToUse.count(FuncName) &&
+ (!Remapper || !Remapper->exist(FuncName)))
+ continue;
+ const uint8_t *FuncProfileAddr = Start + NameOffset.second;
+ assert(FuncProfileAddr < End && "out of LBRProfile section");
+ if (std::error_code EC = readFuncProfile(FuncProfileAddr))
+ return EC;
+ }
}
+ Data = End;
}
- Data = End;
+ assert((CSProfileCount == 0 || CSProfileCount == Profiles.size()) &&
+ "Cannot have both context-sensitive and regular profile");
+ ProfileIsCS = (CSProfileCount > 0);
+ FunctionSamples::ProfileIsCS = ProfileIsCS;
return sampleprof_error::success;
}
@@ -887,7 +892,8 @@
if (std::error_code EC = Checksum.getError())
return EC;
- Profiles[*FName].setFunctionHash(*Checksum);
+ SampleContext FContext(*FName);
+ Profiles[FContext].setFunctionHash(*Checksum);
}
return sampleprof_error::success;
}
diff --git a/llvm/lib/ProfileData/SampleProfWriter.cpp b/llvm/lib/ProfileData/SampleProfWriter.cpp
--- a/llvm/lib/ProfileData/SampleProfWriter.cpp
+++ b/llvm/lib/ProfileData/SampleProfWriter.cpp
@@ -147,7 +147,7 @@
std::error_code
SampleProfileWriterExtBinaryBase::writeSample(const FunctionSamples &S) {
uint64_t Offset = OutputStream->tell();
- StringRef Name = S.getName();
+ StringRef Name = S.getNameWithContext(true);
FuncOffsetTable[Name] = Offset - SecLBRProfileStart;
encodeULEB128(S.getHeadSamples(), *OutputStream);
return writeBody(S);
@@ -635,7 +635,7 @@
std::error_code SampleProfileWriterBinary::writeBody(const FunctionSamples &S) {
auto &OS = *OutputStream;
- if (std::error_code EC = writeNameIdx(S.getName()))
+ if (std::error_code EC = writeNameIdx(S.getNameWithContext(true)))
return EC;
encodeULEB128(S.getTotalSamples(), OS);
diff --git a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
--- a/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
+++ b/llvm/lib/Transforms/IPO/SampleContextTracker.cpp
@@ -179,7 +179,7 @@
SampleContext Context(FuncSample.first(), RawContext);
LLVM_DEBUG(dbgs() << "Tracking Context for function: " << Context << "\n");
if (!Context.isBaseContext())
- FuncToCtxtProfileSet[Context.getName()].insert(FSamples);
+ FuncToCtxtProfileSet[Context.getNameWithoutContext()].insert(FSamples);
ContextTrieNode *NewNode = getOrCreateContextPath(Context, true);
assert(!NewNode->getFunctionSamples() &&
"New node can't have sample profile");
diff --git a/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll b/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll
--- a/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll
+++ b/llvm/test/Transforms/SampleProfile/profile-context-tracker.ll
@@ -1,18 +1,22 @@
; Test for CSSPGO's SampleContextTracker to make sure context profile tree is promoted and merged properly
; based on inline decision, so post inline counts are accurate.
+; RUN: llvm-profdata merge --sample --extbinary %S/Inputs/profile-context-tracker.prof -o %t
+
; Note that we need new pass manager to enable top-down processing for sample profile loader
; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile
; main:3 @ _Z5funcAi
; main:3 @ _Z5funcAi:1 @ _Z8funcLeafi
; _Z5funcBi:1 @ _Z8funcLeafi
; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -sample-profile-inline-size -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-ALL
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t -sample-profile-inline-size -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-ALL
; Testwe we inlined the following in top-down order and entry counts accurate reflects post-inline base profile
; main:3 @ _Z5funcAi
; _Z5funcAi:1 @ _Z8funcLeafi
; _Z5funcBi:1 @ _Z8funcLeafi
; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-context-tracker.prof -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-HOT
+; RUN: opt < %s -passes=sample-profile -sample-profile-file=%t -profile-sample-accurate -S | FileCheck %s --check-prefix=INLINE-HOT
@factor = dso_local global i32 3, align 4, !dbg !0
diff --git a/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext b/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext
new file mode 100644
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/Inputs/cs-sample.proftext
@@ -0,0 +1,36 @@
+[main:3 @ _Z5funcAi:1 @ _Z8funcLeafi]:1467299:11
+ 0: 6
+ 1: 6
+ 3: 287884
+ 4: 287864 _Z3fibi:315608
+ 15: 23
+[main:3.1 @ _Z5funcBi:1 @ _Z8funcLeafi]:500853:20
+ 0: 15
+ 1: 15
+ 3: 74946
+ 4: 74941 _Z3fibi:82359
+ 10: 23324
+ 11: 23327 _Z3fibi:25228
+ 15: 11
+[main]:154:0
+ 2: 12
+ 3: 18 _Z5funcAi:11
+ 3.1: 18 _Z5funcBi:19
+[external:12 @ main]:154:12
+ 2: 12
+ 3: 10 _Z5funcAi:7
+ 3.1: 10 _Z5funcBi:11
+[main:3.1 @ _Z5funcBi]:120:19
+ 0: 19
+ 1: 19 _Z8funcLeafi:20
+ 3: 12
+[externalA:17 @ _Z5funcBi]:120:3
+ 0: 3
+ 1: 3
+[external:10 @ _Z5funcBi]:120:10
+ 0: 10
+ 1: 10
+[main:3 @ _Z5funcAi]:99:11
+ 0: 10
+ 1: 10 _Z8funcLeafi:11
+ 3: 24
diff --git a/llvm/test/tools/llvm-profdata/cs-sample-profile.test b/llvm/test/tools/llvm-profdata/cs-sample-profile.test
new file mode 100644
--- /dev/null
+++ b/llvm/test/tools/llvm-profdata/cs-sample-profile.test
@@ -0,0 +1,4 @@
+RUN: llvm-profdata merge --sample --text -output=%t.proftext %S/Inputs/cs-sample.proftext
+RUN: diff -b %t.proftext %S/Inputs/cs-sample.proftext
+RUN: llvm-profdata merge --sample --extbinary %p/Inputs/cs-sample.proftext -o %t.prof && llvm-profdata merge --sample --text %t.prof -o %t1.proftext
+RUN: diff -b %t1.proftext %S/Inputs/cs-sample.proftext
diff --git a/llvm/tools/llvm-profdata/llvm-profdata.cpp b/llvm/tools/llvm-profdata/llvm-profdata.cpp
--- a/llvm/tools/llvm-profdata/llvm-profdata.cpp
+++ b/llvm/tools/llvm-profdata/llvm-profdata.cpp
@@ -696,7 +696,7 @@
Remapper ? remapSamples(I->second, *Remapper, Result)
: FunctionSamples();
FunctionSamples &Samples = Remapper ? Remapped : I->second;
- StringRef FName = Samples.getName();
+ StringRef FName = Samples.getNameWithContext(true);
MergeResult(Result, ProfileMap[FName].merge(Samples, Input.Weight));
if (Result != sampleprof_error::success) {
std::error_code EC = make_error_code(Result);
diff --git a/llvm/tools/llvm-profgen/ProfileGenerator.cpp b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
--- a/llvm/tools/llvm-profgen/ProfileGenerator.cpp
+++ b/llvm/tools/llvm-profgen/ProfileGenerator.cpp
@@ -164,7 +164,7 @@
if (Ret.second) {
SampleContext FContext(Ret.first->first(), RawContext);
FunctionSamples &FProfile = Ret.first->second;
- FProfile.setName(FContext.getName());
+ FProfile.setName(FContext.getNameWithoutContext());
FProfile.setContext(FContext);
}
return Ret.first->second;