Index: include/llvm/ProfileData/InstrProf.h =================================================================== --- include/llvm/ProfileData/InstrProf.h +++ include/llvm/ProfileData/InstrProf.h @@ -153,9 +153,13 @@ /// Return the modified name for function \c F suitable to be /// used the key for profile lookup. -std::string getPGOFuncName(const Function &F, +std::string getPGOFuncName(const Function &F, bool InLTO = false, uint64_t Version = INSTR_PROF_INDEX_VERSION); +// Return the modified name for function \c F suitable to be +// used the key for profile lookup in LTO compilation. +//std::string getPGOFuncNameInLTO(const Function &F); + /// Return the modified name for a function suitable to be /// used the key for profile lookup. The function's original /// name is \c RawFuncName and has linkage of type \c Linkage. @@ -299,13 +303,17 @@ StringSet<> NameTab; // A map from MD5 keys to function name strings. std::vector> MD5NameMap; + // A map from MD5 keys to function define. We only populate this map + // when build the Symtab from a Module. + std::vector> MD5FuncMap; // A map from function runtime address to function name MD5 hash. // This map is only populated and used by raw instr profile reader. AddrHashMap AddrToMD5Map; public: InstrProfSymtab() - : Data(), Address(0), NameTab(), MD5NameMap(), AddrToMD5Map() {} + : Data(), Address(0), NameTab(), MD5NameMap(), MD5FuncMap(), + AddrToMD5Map() {} /// Create InstrProfSymtab from an object file section which /// contains function PGO names. When section may contain raw @@ -324,7 +332,7 @@ /// A wrapper interface to populate the PGO symtab with functions /// decls from module \c M. This interface is used by transformation /// passes such as indirect function call promotion. - void create(const Module &M); + void create(Module &M, bool InLTO = false); /// Create InstrProfSymtab from a set of names iteratable from /// \p IterRange. This interface is used by IndexedProfReader. template void create(const NameIterRange &IterRange); @@ -354,6 +362,8 @@ /// Return function's PGO name from the name's md5 hash value. /// If not found, return an empty string. inline StringRef getFuncName(uint64_t FuncMD5Hash); + /// Return function from the name's md5 hash. Return nullptr if not found. + inline Function *getFunction(uint64_t FuncMD5Hash); /// Return the function's original assembly name by stripping off /// the prefix attached (to symbols with priviate linkage). For /// global functions, it returns the same string as getFuncName. @@ -384,6 +394,7 @@ void InstrProfSymtab::finalizeSymtab() { std::sort(MD5NameMap.begin(), MD5NameMap.end(), less_first()); + std::sort(MD5FuncMap.begin(), MD5FuncMap.end(), less_first()); std::sort(AddrToMD5Map.begin(), AddrToMD5Map.end(), less_first()); AddrToMD5Map.erase(std::unique(AddrToMD5Map.begin(), AddrToMD5Map.end()), AddrToMD5Map.end()); @@ -399,6 +410,16 @@ return StringRef(); } +Function* InstrProfSymtab::getFunction(uint64_t FuncMD5Hash) { + auto Result = + std::lower_bound(MD5FuncMap.begin(), MD5FuncMap.end(), FuncMD5Hash, + [](const std::pair &LHS, + uint64_t RHS) { return LHS.first < RHS; }); + if (Result != MD5FuncMap.end() && Result->first == FuncMD5Hash) + return Result->second; + return nullptr; +} + // See also getPGOFuncName implementation. These two need to be // matched. StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) { Index: lib/ProfileData/InstrProf.cpp =================================================================== --- lib/ProfileData/InstrProf.cpp +++ lib/ProfileData/InstrProf.cpp @@ -83,9 +83,34 @@ return GlobalValue::getGlobalIdentifier(RawFuncName, Linkage, FileName); } -std::string getPGOFuncName(const Function &F, uint64_t Version) { - return getPGOFuncName(F.getName(), F.getLinkage(), F.getParent()->getName(), - Version); +// Return the PGOFuncName. This function has some special handling when called +// in LTO optimization. LTO's internalization privatizes many global linkage +// symbols into internal linkage.This happens after value profile annotation so +// those internal linkage function should have have a source prefix. However, +// we still need to differentiate these symbols from the ones not from LTO's +// internalization: In profile-use, we create a function meta-data if the +// PGOfuncName is different from it's raw name. Use that name if it's available. +// Otherwise, just strip off the source module prefix. +std::string getPGOFuncName(const Function &F, bool InLTO, uint64_t Version) { + if (InLTO) { + // First check if these is a metadata. + MDNode *MD = F.getMetadata("PGOFuncName"); + if (MD != nullptr) { + StringRef S = cast(MD->getOperand(0))->getString(); + return S.str(); + } + } + const std::string& PGOFuncName = getPGOFuncName(F.getName(), F.getLinkage(), + F.getParent()->getName(), + Version); + if (!InLTO) + return PGOFuncName; + + // Strip out prefix and postfix from the the internalized symbols. + size_t S = PGOFuncName.find_first_of(':'); + if (S == std::string::npos) + return PGOFuncName; + return PGOFuncName.substr(S + 1); } StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) { @@ -149,9 +174,15 @@ return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), PGOFuncName); } -void InstrProfSymtab::create(const Module &M) { - for (const Function &F : M) - addFuncName(getPGOFuncName(F)); +void InstrProfSymtab::create(Module &M, bool InLTO) { + for (Function &F : M) { + // Function may not have a name. Ingore in this case. + if (!F.hasName()) + continue; + const std::string &PGOFuncName = getPGOFuncName(F, InLTO); + addFuncName(PGOFuncName); + MD5FuncMap.push_back(std::make_pair(Function::getGUID(PGOFuncName), &F)); + } finalizeSymtab(); } Index: lib/Transforms/Instrumentation/PGOInstrumentation.cpp =================================================================== --- lib/Transforms/Instrumentation/PGOInstrumentation.cpp +++ lib/Transforms/Instrumentation/PGOInstrumentation.cpp @@ -254,6 +254,11 @@ BlockFrequencyInfo *BFI = nullptr) : F(Func), FunctionHash(0), MST(F, BPI, BFI) { FuncName = getPGOFuncName(F); + if (FuncName != Func.getName()) { + LLVMContext &C = F.getContext(); + MDNode *N = MDNode::get(C, MDString::get(C, FuncName.c_str())); + F.setMetadata("PGOFuncName", N); + } computeCFGHash(); DEBUG(dumpInfo("after CFGMST"));