Index: include/llvm/IR/Function.h =================================================================== --- include/llvm/IR/Function.h +++ include/llvm/IR/Function.h @@ -234,21 +234,54 @@ getContext(), AttributeList::FunctionIndex, Kind)); } + enum ProfileCountType { PCT_Invalid, PCT_Real, PCT_Synthetic }; + + /// Class to represent profile counts. + /// + /// This class represents both real and synthetic profile counts. + class ProfileCount { + private: + uint64_t Count; + ProfileCountType PCT; + static ProfileCount Invalid; + + public: + ProfileCount() : Count(-1), PCT(PCT_Invalid) {} + ProfileCount(uint64_t Count, ProfileCountType PCT) + : Count(Count), PCT(PCT) {} + bool hasValue() const { return PCT != PCT_Invalid; } + uint64_t getCount() const { return Count; } + uint64_t getValue() const { return Count; } + ProfileCountType getType() const { return PCT; } + bool isSynthetic() const { return PCT == PCT_Synthetic; } + explicit operator bool() { return hasValue(); } + bool operator!() const { return !hasValue(); } + // Update the count retaining the same profile count type. + ProfileCount &setCount(uint64_t C) { + Count = C; + return *this; + } + static ProfileCount getInvalid() { return Invalid; } + }; + /// \brief Set the entry count for this function. /// /// Entry count is the number of times this function was executed based on - /// pgo data. \p Synthetic indicates the count is synthesized by analysis and - /// not from a profile run. \p Imports points to a set of GUIDs that needs to + /// pgo data. \p Imports points to a set of GUIDs that needs to /// be imported by the function for sample PGO, to enable the same inlines as /// the profiled optimized binary. - void setEntryCount(uint64_t Count, bool Synthetic = false, + void setEntryCount(ProfileCount Count, + const DenseSet *Imports = nullptr); + + /// A convenience wrapper for setting entry count + void setEntryCount(uint64_t Count, ProfileCountType Type = PCT_Real, const DenseSet *Imports = nullptr); /// \brief Get the entry count for this function. /// /// Entry count is the number of times the function was executed based on /// pgo data. - Optional getEntryCount() const; + ProfileCount getEntryCount() const; /// Return true if the function is annotated with profile data. /// Index: lib/IR/Function.cpp =================================================================== --- lib/IR/Function.cpp +++ lib/IR/Function.cpp @@ -56,6 +56,7 @@ #include using namespace llvm; +using ProfileCount = Function::ProfileCount; // Explicit instantiations of SymbolTableListTraits since some of the methods // are not in the public header file... @@ -1320,14 +1321,23 @@ setValueSubclassData(getSubclassDataFromValue() & ~(1 << Bit)); } -void Function::setEntryCount(uint64_t Count, bool Synthetic, +void Function::setEntryCount(ProfileCount Count, const DenseSet *S) { + assert(Count.hasValue()); MDBuilder MDB(getContext()); - setMetadata(LLVMContext::MD_prof, - MDB.createFunctionEntryCount(Count, Synthetic, S)); + setMetadata( + LLVMContext::MD_prof, + MDB.createFunctionEntryCount(Count.getCount(), Count.isSynthetic(), S)); } -Optional Function::getEntryCount() const { +void Function::setEntryCount(uint64_t Count, Function::ProfileCountType Type, + const DenseSet *Imports) { + setEntryCount(ProfileCount(Count, Type), Imports); +} + +ProfileCount ProfileCount::Invalid; + +ProfileCount Function::getEntryCount() const { MDNode *MD = getMetadata(LLVMContext::MD_prof); if (MD && MD->getOperand(0)) if (MDString *MDS = dyn_cast(MD->getOperand(0))) @@ -1337,10 +1347,10 @@ // A value of -1 is used for SamplePGO when there were no samples. // Treat this the same as unknown. if (Count == (uint64_t)-1) - return None; - return Count; + return ProfileCount::getInvalid(); + return ProfileCount(Count, PCT_Real); } - return None; + return ProfileCount::getInvalid(); } DenseSet Function::getImportGUIDs() const { Index: lib/Transforms/IPO/PartialInlining.cpp =================================================================== --- lib/Transforms/IPO/PartialInlining.cpp +++ lib/Transforms/IPO/PartialInlining.cpp @@ -1384,7 +1384,8 @@ if (CalleeEntryCount) computeCallsiteToProfCountMap(Cloner.ClonedFunc, CallSiteToProfCountMap); - uint64_t CalleeEntryCountV = (CalleeEntryCount ? *CalleeEntryCount : 0); + uint64_t CalleeEntryCountV = + (CalleeEntryCount ? CalleeEntryCount.getValue() : 0); bool AnyInline = false; for (User *User : Users) { @@ -1433,7 +1434,8 @@ if (AnyInline) { Cloner.IsFunctionInlined = true; if (CalleeEntryCount) - Cloner.OrigFunc->setEntryCount(CalleeEntryCountV); + Cloner.OrigFunc->setEntryCount( + CalleeEntryCount.setCount(CalleeEntryCountV)); auto &ORE = (*GetORE)(*Cloner.OrigFunc); ORE.emit([&]() { return OptimizationRemark(DEBUG_TYPE, "PartiallyInlined", Cloner.OrigFunc) Index: lib/Transforms/IPO/SampleProfile.cpp =================================================================== --- lib/Transforms/IPO/SampleProfile.cpp +++ lib/Transforms/IPO/SampleProfile.cpp @@ -85,7 +85,7 @@ using namespace llvm; using namespace sampleprof; - +using ProfileCount = Function::ProfileCount; #define DEBUG_TYPE "sample-profile" // Command line option to specify the file to read samples from. This is @@ -1467,7 +1467,9 @@ // Sets the GUIDs that are inlined in the profiled binary. This is used // for ThinLink to make correct liveness analysis, and also make the IR // match the profiled binary before annotation. - F.setEntryCount(Samples->getHeadSamples() + 1, false, &InlinedGUIDs); + F.setEntryCount( + ProfileCount(Samples->getHeadSamples() + 1, Function::PCT_Real), + &InlinedGUIDs); // Compute dominance and loop info needed for propagation. computeDominanceAndLoopInfo(F); @@ -1587,7 +1589,7 @@ // Initialize the entry count to -1, which will be treated conservatively // by getEntryCount as the same as unknown (None). If we have samples this // will be overwritten in emitAnnotations. - F.setEntryCount(-1); + F.setEntryCount(ProfileCount(-1, Function::PCT_Real)); std::unique_ptr OwnedORE; if (AM) { auto &FAM = Index: lib/Transforms/IPO/SyntheticCountsPropagation.cpp =================================================================== --- lib/Transforms/IPO/SyntheticCountsPropagation.cpp +++ lib/Transforms/IPO/SyntheticCountsPropagation.cpp @@ -41,6 +41,7 @@ using namespace llvm; using Scaled64 = ScaledNumber; +using ProfileCount = Function::ProfileCount; #define DEBUG_TYPE "synthetic-counts-propagation" @@ -121,7 +122,8 @@ // Set the counts as metadata. for (auto Entry : Counts) - Entry.first->setEntryCount(Entry.second, true); + Entry.first->setEntryCount( + ProfileCount(Entry.second, Function::PCT_Synthetic)); return PreservedAnalyses::all(); } Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -119,6 +119,7 @@ #include using namespace llvm; +using ProfileCount = Function::ProfileCount; #define DEBUG_TYPE "pgo-instrumentation" @@ -1139,7 +1140,7 @@ } #endif uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue; - F.setEntryCount(FuncEntryCount); + F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real)); uint64_t FuncMaxCount = FuncEntryCount; for (auto &BB : F) { auto BI = findBBInfo(&BB); Index: lib/Transforms/Utils/CodeExtractor.cpp =================================================================== --- lib/Transforms/Utils/CodeExtractor.cpp +++ lib/Transforms/Utils/CodeExtractor.cpp @@ -66,6 +66,7 @@ #include using namespace llvm; +using ProfileCount = Function::ProfileCount; #define DEBUG_TYPE "code-extractor" @@ -1163,10 +1164,10 @@ // Update the entry count of the function. if (BFI) { - Optional EntryCount = - BFI->getProfileCountFromFreq(EntryFreq.getFrequency()); - if (EntryCount.hasValue()) - newFunction->setEntryCount(EntryCount.getValue()); + auto Count = BFI->getProfileCountFromFreq(EntryFreq.getFrequency()); + if (Count.hasValue()) + newFunction->setEntryCount( + ProfileCount(Count.getValue(), Function::PCT_Real)); // FIXME BFI->setBlockFreq(codeReplacer, EntryFreq.getFrequency()); } Index: lib/Transforms/Utils/InlineFunction.cpp =================================================================== --- lib/Transforms/Utils/InlineFunction.cpp +++ lib/Transforms/Utils/InlineFunction.cpp @@ -72,6 +72,7 @@ #include using namespace llvm; +using ProfileCount = Function::ProfileCount; static cl::opt EnableNoAliasConversion("enable-noalias-to-md-conversion", cl::init(true), @@ -1431,14 +1432,14 @@ /// Update the branch metadata for cloned call instructions. static void updateCallProfile(Function *Callee, const ValueToValueMapTy &VMap, - const Optional &CalleeEntryCount, + const ProfileCount &CalleeEntryCount, const Instruction *TheCall, ProfileSummaryInfo *PSI, BlockFrequencyInfo *CallerBFI) { - if (!CalleeEntryCount.hasValue() || CalleeEntryCount.getValue() < 1) + if (!CalleeEntryCount.hasValue() || CalleeEntryCount.isSynthetic() || + CalleeEntryCount.getValue() < 1) return; - Optional CallSiteCount = - PSI ? PSI->getProfileCount(TheCall, CallerBFI) : None; + auto CallSiteCount = PSI ? PSI->getProfileCount(TheCall, CallerBFI) : None; uint64_t CallCount = std::min(CallSiteCount.hasValue() ? CallSiteCount.getValue() : 0, CalleeEntryCount.getValue()); @@ -1467,18 +1468,19 @@ // callsite is M, the new callee count is set to N - M. M is estimated from // the caller's entry count, its entry block frequency and the block frequency // of the callsite. - Optional CalleeCount = Callee->getEntryCount(); + auto CalleeCount = Callee->getEntryCount(); if (!CalleeCount.hasValue() || !PSI) return; - Optional CallCount = PSI->getProfileCount(CallInst, CallerBFI); + auto CallCount = PSI->getProfileCount(CallInst, CallerBFI); if (!CallCount.hasValue()) return; // Since CallSiteCount is an estimate, it could exceed the original callee // count and has to be set to 0. if (CallCount.getValue() > CalleeCount.getValue()) - Callee->setEntryCount(0); + CalleeCount.setCount(0); else - Callee->setEntryCount(CalleeCount.getValue() - CallCount.getValue()); + CalleeCount.setCount(CalleeCount.getValue() - CallCount.getValue()); + Callee->setEntryCount(CalleeCount); } /// This function inlines the called function into the basic block of the Index: unittests/IR/MetadataTest.cpp =================================================================== --- unittests/IR/MetadataTest.cpp +++ unittests/IR/MetadataTest.cpp @@ -2438,7 +2438,7 @@ EXPECT_FALSE(F->getEntryCount().hasValue()); F->setEntryCount(12304); EXPECT_TRUE(F->getEntryCount().hasValue()); - EXPECT_EQ(12304u, *F->getEntryCount()); + EXPECT_EQ(12304u, F->getEntryCount().getCount()); } TEST_F(FunctionAttachmentTest, SubprogramAttachment) {