diff --git a/llvm/include/llvm/IR/MDBuilder.h b/llvm/include/llvm/IR/MDBuilder.h --- a/llvm/include/llvm/IR/MDBuilder.h +++ b/llvm/include/llvm/IR/MDBuilder.h @@ -80,6 +80,10 @@ /// Return metadata containing the pseudo probe descriptor for a function. MDNode *createPseudoProbeDesc(uint64_t GUID, uint64_t Hash, Function *F); + /// Return metadata containing llvm statistics. + MDNode * + createLLVMStats(ArrayRef> LLVMStatsVec); + //===------------------------------------------------------------------===// // Range metadata. //===------------------------------------------------------------------===// diff --git a/llvm/include/llvm/MC/MCObjectFileInfo.h b/llvm/include/llvm/MC/MCObjectFileInfo.h --- a/llvm/include/llvm/MC/MCObjectFileInfo.h +++ b/llvm/include/llvm/MC/MCObjectFileInfo.h @@ -180,6 +180,9 @@ MCSection *PseudoProbeSection = nullptr; MCSection *PseudoProbeDescSection = nullptr; + // Section for metadata of llvm statistics. + MCSection *LLVMStatsSection = nullptr; + // ELF specific sections. MCSection *DataRelROSection = nullptr; MCSection *MergeableConst4Section = nullptr; @@ -366,6 +369,8 @@ MCSection *getPseudoProbeDescSection(StringRef FuncName) const; + MCSection *getLLVMStatsSection() const; + MCSection *getPCSection(StringRef Name, const MCSection *TextSec) const; // ELF specific sections. diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -58,6 +58,7 @@ #include "llvm/MC/MCValue.h" #include "llvm/MC/SectionKind.h" #include "llvm/ProfileData/InstrProf.h" +#include "llvm/Support/Base64.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/ErrorHandling.h" @@ -366,6 +367,31 @@ } } + if (NamedMDNode *LLVMStats = M.getNamedMetadata("llvm.stats")) { + // Emit the metadata for llvm statistics into .llvm_stats section, which is + // formatted as a list of key/value pair, the value is base64 encoded. + auto *S = C.getObjectFileInfo()->getLLVMStatsSection(); + Streamer.switchSection(S); + for (const auto *Operand : LLVMStats->operands()) { + const auto *MD = cast(Operand); + assert(MD->getNumOperands() % 2 == 0 && + ("Operand num should be even for a list of key/value pair")); + for (size_t I = 0; I < MD->getNumOperands(); I += 2) { + // Encode the key string size. + auto *Key = cast(MD->getOperand(I)); + Streamer.emitULEB128IntValue(Key->getString().size()); + Streamer.emitBytes(Key->getString()); + // Encode the value into a Base64 string. + std::string Value = encodeBase64( + Twine(mdconst::dyn_extract(MD->getOperand(I + 1)) + ->getZExtValue()) + .str()); + Streamer.emitULEB128IntValue(Value.size()); + Streamer.emitBytes(Value); + } + } + } + unsigned Version = 0; unsigned Flags = 0; StringRef Section; diff --git a/llvm/lib/IR/MDBuilder.cpp b/llvm/lib/IR/MDBuilder.cpp --- a/llvm/lib/IR/MDBuilder.cpp +++ b/llvm/lib/IR/MDBuilder.cpp @@ -344,3 +344,15 @@ Ops[2] = createString(F->getName()); return MDNode::get(Context, Ops); } + +MDNode * +MDBuilder::createLLVMStats(ArrayRef> LLVMStats) { + auto *Int64Ty = Type::getInt64Ty(Context); + SmallVector Ops(LLVMStats.size() * 2); + for (size_t I = 0; I < LLVMStats.size(); I++) { + Ops[I * 2] = createString(LLVMStats[I].first); + Ops[I * 2 + 1] = + createConstant(ConstantInt::get(Int64Ty, LLVMStats[I].second)); + } + return MDNode::get(Context, Ops); +} diff --git a/llvm/lib/MC/MCObjectFileInfo.cpp b/llvm/lib/MC/MCObjectFileInfo.cpp --- a/llvm/lib/MC/MCObjectFileInfo.cpp +++ b/llvm/lib/MC/MCObjectFileInfo.cpp @@ -531,6 +531,8 @@ PseudoProbeSection = Ctx->getELFSection(".pseudo_probe", DebugSecType, 0); PseudoProbeDescSection = Ctx->getELFSection(".pseudo_probe_desc", DebugSecType, 0); + + LLVMStatsSection = Ctx->getELFSection(".llvm_stats", ELF::SHT_PROGBITS, 0); } void MCObjectFileInfo::initGOFFMCObjectFileInfo(const Triple &T) { @@ -1199,6 +1201,10 @@ return PseudoProbeDescSection; } +MCSection *MCObjectFileInfo::getLLVMStatsSection() const { + return LLVMStatsSection; +} + MCSection *MCObjectFileInfo::getPCSection(StringRef Name, const MCSection *TextSec) const { if (Ctx->getObjectFileType() != MCContext::IsELF) diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -133,6 +133,11 @@ "report-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute and report stale profile statistical metrics.")); +static cl::opt PersistProfileStaleness( + "persist-profile-staleness", cl::Hidden, cl::init(false), + cl::desc("Compute stale profile statistical metrics and write it into the " + "native object file(.llvm_stats section).")); + static cl::opt ProfileSampleAccurate( "profile-sample-accurate", cl::Hidden, cl::init(false), cl::desc("If the sample profile is accurate, we will mark all un-sampled " @@ -2041,7 +2046,7 @@ } } - if (ReportProfileStaleness) { + if (ReportProfileStaleness || PersistProfileStaleness) { MatchingManager = std::make_unique(M, *Reader, ProbeManager.get()); } @@ -2150,18 +2155,42 @@ detectProfileMismatch(F, *FS); } - if (FunctionSamples::ProfileIsProbeBased) { - errs() << "(" << NumMismatchedFuncHash << "/" << TotalProfiledFunc << ")" - << " of functions' profile are invalid and " - << " (" << MismatchedFuncHashSamples << "/" << TotalFuncHashSamples + if (ReportProfileStaleness) { + if (FunctionSamples::ProfileIsProbeBased) { + errs() << "(" << NumMismatchedFuncHash << "/" << TotalProfiledFunc << ")" + << " of functions' profile are invalid and " + << " (" << MismatchedFuncHashSamples << "/" << TotalFuncHashSamples + << ")" + << " of samples are discarded due to function hash mismatch.\n"; + } + errs() << "(" << NumMismatchedCallsite << "/" << TotalProfiledCallsite << ")" - << " of samples are discarded due to function hash mismatch.\n"; + << " of callsites' profile are invalid and " + << "(" << MismatchedCallsiteSamples << "/" << TotalCallsiteSamples + << ")" + << " of samples are discarded due to callsite location mismatch.\n"; + } + + if (PersistProfileStaleness) { + LLVMContext &Ctx = M.getContext(); + MDBuilder MDB(Ctx); + + SmallVector> ProfStatsVec; + if (FunctionSamples::ProfileIsProbeBased) { + ProfStatsVec.emplace_back("NumMismatchedFuncHash", NumMismatchedFuncHash); + ProfStatsVec.emplace_back("TotalProfiledFunc", TotalProfiledFunc); + ProfStatsVec.emplace_back("MismatchedFuncHashSamples", + MismatchedFuncHashSamples); + ProfStatsVec.emplace_back("TotalFuncHashSamples", TotalFuncHashSamples); + } + ProfStatsVec.emplace_back("MismatchedCallsiteSamples", + MismatchedCallsiteSamples); + ProfStatsVec.emplace_back("TotalCallsiteSamples", TotalCallsiteSamples); + + auto *MD = MDB.createLLVMStats(ProfStatsVec); + auto *NMD = M.getOrInsertNamedMetadata("llvm.stats"); + NMD->addOperand(MD); } - errs() << "(" << NumMismatchedCallsite << "/" << TotalProfiledCallsite << ")" - << " of callsites' profile are invalid and " - << "(" << MismatchedCallsiteSamples << "/" << TotalCallsiteSamples - << ")" - << " of samples are discarded due to callsite location mismatch.\n"; } bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM, @@ -2208,7 +2237,7 @@ assert(SymbolMap.count(StringRef()) == 0 && "No empty StringRef should be added in SymbolMap"); - if (ReportProfileStaleness) + if (ReportProfileStaleness || PersistProfileStaleness) MatchingManager->detectProfileMismatch(); bool retval = false; diff --git a/llvm/test/Transforms/SampleProfile/profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/profile-mismatch.ll --- a/llvm/test/Transforms/SampleProfile/profile-mismatch.ll +++ b/llvm/test/Transforms/SampleProfile/profile-mismatch.ll @@ -1,8 +1,15 @@ -; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-mismatch.prof -report-profile-staleness -S 2>%t +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/profile-mismatch.prof -report-profile-staleness -persist-profile-staleness -S 2>%t -o %t.ll ; RUN: FileCheck %s --input-file %t +; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD +; RUN: llc < %t.ll -filetype=obj -o %t.obj +; RUN: llvm-objdump --section-headers %t.obj | FileCheck %s --check-prefix=CHECK-OBJ ; CHECK: (2/3) of callsites' profile are invalid and (20/30) of samples are discarded due to callsite location mismatch. +; CHECK-MD: ![[#]] = !{!"MismatchedCallsiteSamples", i64 20, !"TotalCallsiteSamples", i64 30} + +; CHECK-OBJ: .llvm_stats + target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll --- a/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll +++ b/llvm/test/Transforms/SampleProfile/pseudo-probe-profile-mismatch.ll @@ -1,9 +1,16 @@ -; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -report-profile-staleness -S 2>%t +; RUN: opt < %s -passes=sample-profile -sample-profile-file=%S/Inputs/pseudo-probe-profile-mismatch.prof -report-profile-staleness -persist-profile-staleness -S 2>%t -o %t.ll ; RUN: FileCheck %s --input-file %t +; RUN: FileCheck %s --input-file %t.ll -check-prefix=CHECK-MD +; RUN: llc < %t.ll -filetype=obj -o %t.obj +; RUN: llvm-objdump --section-headers %t.obj | FileCheck %s --check-prefix=CHECK-OBJ ; CHECK: (1/3) of functions' profile are invalid and (10/50) of samples are discarded due to function hash mismatch. ; CHECK: (2/3) of callsites' profile are invalid and (20/30) of samples are discarded due to callsite location mismatch. +; CHECK-MD: ![[#]] = !{!"NumMismatchedFuncHash", i64 1, !"TotalProfiledFunc", i64 3, !"MismatchedFuncHashSamples", i64 10, !"TotalFuncHashSamples", i64 50, !"MismatchedCallsiteSamples", i64 20, !"TotalCallsiteSamples", i64 30} + +; CHECK-OBJ: .llvm_stats + target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu"