Index: include/llvm/Bitcode/LLVMBitCodes.h =================================================================== --- include/llvm/Bitcode/LLVMBitCodes.h +++ include/llvm/Bitcode/LLVMBitCodes.h @@ -180,8 +180,6 @@ VST_CODE_ENTRY = 1, // VST_ENTRY: [valueid, namechar x N] VST_CODE_BBENTRY = 2, // VST_BBENTRY: [bbid, namechar x N] VST_CODE_FNENTRY = 3, // VST_FNENTRY: [valueid, offset, namechar x N] - // VST_COMBINED_GVDEFENTRY: [valueid, sumoffset, guid] - VST_CODE_COMBINED_GVDEFENTRY = 4, // VST_COMBINED_ENTRY: [valueid, refguid] VST_CODE_COMBINED_ENTRY = 5 }; @@ -204,18 +202,18 @@ FS_PERMODULE_PROFILE = 2, // PERMODULE_GLOBALVAR_INIT_REFS: [valueid, flags, n x valueid] FS_PERMODULE_GLOBALVAR_INIT_REFS = 3, - // COMBINED: [modid, flags, instcount, numrefs, numrefs x valueid, + // COMBINED: [valueid, modid, flags, instcount, numrefs, numrefs x valueid, // n x (valueid, callsitecount)] FS_COMBINED = 4, - // COMBINED_PROFILE: [modid, flags, instcount, numrefs, + // COMBINED_PROFILE: [valueid, modid, flags, instcount, numrefs, // numrefs x valueid, // n x (valueid, callsitecount, profilecount)] FS_COMBINED_PROFILE = 5, - // COMBINED_GLOBALVAR_INIT_REFS: [modid, flags, n x valueid] + // COMBINED_GLOBALVAR_INIT_REFS: [valueid, modid, flags, n x valueid] FS_COMBINED_GLOBALVAR_INIT_REFS = 6, // ALIAS: [valueid, flags, valueid] FS_ALIAS = 7, - // COMBINED_ALIAS: [modid, flags, offset] + // COMBINED_ALIAS: [valueid, modid, flags, valueid] FS_COMBINED_ALIAS = 8, // COMBINED_ORIGINAL_NAME: [original_name_hash] FS_COMBINED_ORIGINAL_NAME = 9, Index: include/llvm/IR/ModuleSummaryIndex.h =================================================================== --- include/llvm/IR/ModuleSummaryIndex.h +++ include/llvm/IR/ModuleSummaryIndex.h @@ -371,6 +371,24 @@ GlobalValueMap[ValueGUID].push_back(std::move(Summary)); } + /// Find the summary for global \p GUID in module \p ModuleId, or nullptr if + /// not found. + GlobalValueSummary *findSummaryInModule(GlobalValue::GUID ValueGUID, + StringRef ModuleId) const { + auto CalleeInfoList = findGlobalValueSummaryList(ValueGUID); + if (CalleeInfoList == end()) { + return nullptr; // This function does not have a summary + } + auto Summary = + llvm::find_if(CalleeInfoList->second, + [&](const std::unique_ptr &Summary) { + return Summary->modulePath() == ModuleId; + }); + if (Summary == CalleeInfoList->second.end()) + return nullptr; + return Summary->get(); + } + /// Returns the first GlobalValueSummary for \p GV, asserting that there /// is only one if \p PerModuleIndex. GlobalValueSummary *getGlobalValueSummary(const GlobalValue &GV, Index: lib/Bitcode/Reader/BitcodeReader.cpp =================================================================== --- lib/Bitcode/Reader/BitcodeReader.cpp +++ lib/Bitcode/Reader/BitcodeReader.cpp @@ -492,11 +492,6 @@ DenseMap> ValueIdToCallGraphGUIDMap; - /// Map to save the association between summary offset in the VST to the - /// GUID created when parsing it. Used to add newly parsed summaries to - /// the index. - DenseMap SummaryOffsetToGUIDMap; - /// Map populated during module path string table parsing, from the /// module ID to a string reference owned by the index's module /// path string table, used to correlate with combined index @@ -548,7 +543,6 @@ std::error_code initLazyStream(std::unique_ptr Streamer); std::pair getGUIDFromValueId(unsigned ValueId); - GlobalValue::GUID getGUIDFromOffset(uint64_t Offset); }; } // end anonymous namespace @@ -5740,13 +5734,6 @@ return VGI->second; } -GlobalValue::GUID -ModuleSummaryIndexBitcodeReader::getGUIDFromOffset(uint64_t Offset) { - auto I = SummaryOffsetToGUIDMap.find(Offset); - assert(I != SummaryOffsetToGUIDMap.end()); - return I->second; -} - // Specialized value symbol table parser used when reading module index // blocks where we don't actually create global values. The parsed information // is saved in the bitcode reader for use when later parsing summaries. @@ -5832,18 +5819,6 @@ ValueName.clear(); break; } - case bitc::VST_CODE_COMBINED_GVDEFENTRY: { - // VST_CODE_COMBINED_GVDEFENTRY: [valueid, offset, guid] - unsigned ValueID = Record[0]; - uint64_t GlobalValSummaryOffset = Record[1]; - GlobalValue::GUID GlobalValGUID = Record[2]; - SummaryOffsetToGUIDMap[GlobalValSummaryOffset] = GlobalValGUID; - // The "original name", which is the second value of the pair will be - // overriden later by a FS_COMBINED_ORIGINAL_NAME in the combined index. - ValueIdToCallGraphGUIDMap[ValueID] = - std::make_pair(GlobalValGUID, GlobalValGUID); - break; - } case bitc::VST_CODE_COMBINED_ENTRY: { // VST_CODE_COMBINED_ENTRY: [valueid, refguid] unsigned ValueID = Record[0]; @@ -6030,11 +6005,6 @@ // "OriginalName" attachement. GlobalValueSummary *LastSeenSummary = nullptr; bool Combined = false; - // For aliases in the combined summary, we need to know which summary - // corresponds to the aliasee offset saved in the alias summary. It isn't - // sufficient to just map to the aliasee GUID, since in the combined summary - // there may be multiple values with the same GUID. - DenseMap OffsetToSummaryMap; while (1) { BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); @@ -6067,7 +6037,6 @@ // in the combined index VST entries). The records also contain // information used for ThinLTO renaming and importing. Record.clear(); - uint64_t CurRecordBit = Stream.GetCurrentBitNo(); auto BitCode = Stream.readRecord(Entry.ID, Record); switch (BitCode) { default: // Default behavior: ignore. @@ -6164,27 +6133,29 @@ TheIndex->addGlobalValueSummary(GUID.first, std::move(FS)); break; } - // FS_COMBINED: [modid, flags, instcount, numrefs, numrefs x valueid, - // n x (valueid, callsitecount)] - // FS_COMBINED_PROFILE: [modid, flags, instcount, numrefs, + // FS_COMBINED: [valueid, modid, flags, instcount, numrefs, + // numrefs x valueid, n x (valueid, callsitecount)] + // FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, numrefs, // numrefs x valueid, // n x (valueid, callsitecount, profilecount)] case bitc::FS_COMBINED: case bitc::FS_COMBINED_PROFILE: { - uint64_t ModuleId = Record[0]; - uint64_t RawFlags = Record[1]; - unsigned InstCount = Record[2]; - unsigned NumRefs = Record[3]; + unsigned ValueID = Record[0]; + uint64_t ModuleId = Record[1]; + uint64_t RawFlags = Record[2]; + unsigned InstCount = Record[3]; + unsigned NumRefs = Record[4]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); std::unique_ptr FS = llvm::make_unique(Flags, InstCount); LastSeenSummary = FS.get(); FS->setModulePath(ModuleIdMap[ModuleId]); - static int RefListStartIndex = 4; + static int RefListStartIndex = 5; int CallGraphEdgeStartIndex = RefListStartIndex + NumRefs; assert(Record.size() >= RefListStartIndex + NumRefs && "Record size inconsistent with number of references"); - for (unsigned I = 4, E = CallGraphEdgeStartIndex; I != E; ++I) { + for (unsigned I = RefListStartIndex, E = CallGraphEdgeStartIndex; I != E; + ++I) { unsigned RefValueId = Record[I]; GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; FS->addRefEdge(RefGUID); @@ -6199,50 +6170,52 @@ FS->addCallGraphEdge(CalleeGUID, CalleeInfo(CallsiteCount, ProfileCount)); } - GlobalValue::GUID GUID = getGUIDFromOffset(CurRecordBit); - OffsetToSummaryMap[CurRecordBit] = FS.get(); + GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; TheIndex->addGlobalValueSummary(GUID, std::move(FS)); Combined = true; break; } - // FS_COMBINED_ALIAS: [modid, flags, offset] + // FS_COMBINED_ALIAS: [valueid, modid, flags, valueid] // Aliases must be emitted (and parsed) after all FS_COMBINED entries, as // they expect all aliasee summaries to be available. case bitc::FS_COMBINED_ALIAS: { - uint64_t ModuleId = Record[0]; - uint64_t RawFlags = Record[1]; - uint64_t AliaseeSummaryOffset = Record[2]; + unsigned ValueID = Record[0]; + uint64_t ModuleId = Record[1]; + uint64_t RawFlags = Record[2]; + unsigned AliaseeValueId = Record[3]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); std::unique_ptr AS = llvm::make_unique(Flags); LastSeenSummary = AS.get(); AS->setModulePath(ModuleIdMap[ModuleId]); - auto *AliaseeSummary = OffsetToSummaryMap[AliaseeSummaryOffset]; - if (!AliaseeSummary) + auto AliaseeGUID = getGUIDFromValueId(AliaseeValueId).first; + auto AliaseeInModule = + TheIndex->findSummaryInModule(AliaseeGUID, AS->modulePath()); + if (!AliaseeInModule) return error("Alias expects aliasee summary to be parsed"); - AS->setAliasee(AliaseeSummary); + AS->setAliasee(AliaseeInModule); - GlobalValue::GUID GUID = getGUIDFromOffset(CurRecordBit); + GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; TheIndex->addGlobalValueSummary(GUID, std::move(AS)); Combined = true; break; } - // FS_COMBINED_GLOBALVAR_INIT_REFS: [modid, flags, n x valueid] + // FS_COMBINED_GLOBALVAR_INIT_REFS: [valueid, modid, flags, n x valueid] case bitc::FS_COMBINED_GLOBALVAR_INIT_REFS: { - uint64_t ModuleId = Record[0]; - uint64_t RawFlags = Record[1]; + unsigned ValueID = Record[0]; + uint64_t ModuleId = Record[1]; + uint64_t RawFlags = Record[2]; auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); std::unique_ptr FS = llvm::make_unique(Flags); LastSeenSummary = FS.get(); FS->setModulePath(ModuleIdMap[ModuleId]); - for (unsigned I = 2, E = Record.size(); I != E; ++I) { + for (unsigned I = 3, E = Record.size(); I != E; ++I) { unsigned RefValueId = Record[I]; GlobalValue::GUID RefGUID = getGUIDFromValueId(RefValueId).first; FS->addRefEdge(RefGUID); } - GlobalValue::GUID GUID = getGUIDFromOffset(CurRecordBit); - OffsetToSummaryMap[CurRecordBit] = FS.get(); + GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first; TheIndex->addGlobalValueSummary(GUID, std::move(FS)); Combined = true; break; Index: lib/Bitcode/Writer/BitcodeWriter.cpp =================================================================== --- lib/Bitcode/Writer/BitcodeWriter.cpp +++ lib/Bitcode/Writer/BitcodeWriter.cpp @@ -270,10 +270,6 @@ /// Tracks the last value id recorded in the GUIDToValueMap. unsigned GlobalValueId = 0; - /// Record the starting offset of each summary entry for use in the VST - /// entry, and for any possible alias. - DenseMap SummaryToOffsetMap; - public: /// Constructs a IndexBitcodeWriter object for the given combined index, /// writing to the provided \p Buffer. @@ -312,13 +308,6 @@ return VMI->second; } } - unsigned popValueId(GlobalValue::GUID ValGUID) { - const auto &VMI = GUIDToValueIdMap.find(ValGUID); - assert(VMI != GUIDToValueIdMap.end()); - unsigned ValueId = VMI->second; - GUIDToValueIdMap.erase(VMI); - return ValueId; - } std::map &valueIds() { return GUIDToValueIdMap; } }; @@ -2629,38 +2618,12 @@ Stream.EnterSubblock(bitc::VALUE_SYMTAB_BLOCK_ID, 4); BitCodeAbbrev *Abbv = new BitCodeAbbrev(); - Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_GVDEFENTRY)); - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // sumoffset - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // guid - unsigned DefEntryAbbrev = Stream.EmitAbbrev(Abbv); - - Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::VST_CODE_COMBINED_ENTRY)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // refguid unsigned EntryAbbrev = Stream.EmitAbbrev(Abbv); SmallVector NameVals; - - for (const auto &GSI : Index) { - GlobalValue::GUID ValGUID = GSI.first; - unsigned ValueId = popValueId(ValGUID); - - for (const auto &SI : GSI.second) { - // VST_CODE_COMBINED_GVDEFENTRY: [valueid, sumoffset, guid] - NameVals.push_back(ValueId); - auto Offset = SummaryToOffsetMap[SI.get()]; - assert(Offset); - NameVals.push_back(Offset); - NameVals.push_back(ValGUID); - - // Emit the finished record. - Stream.EmitRecord(bitc::VST_CODE_COMBINED_GVDEFENTRY, NameVals, - DefEntryAbbrev); - NameVals.clear(); - } - } for (const auto &GVI : valueIds()) { // VST_CODE_COMBINED_ENTRY: [valueid, refguid] NameVals.push_back(GVI.second); @@ -3194,6 +3157,7 @@ // Abbrev for FS_COMBINED. BitCodeAbbrev *Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount @@ -3206,6 +3170,7 @@ // Abbrev for FS_COMBINED_PROFILE. Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_PROFILE)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount @@ -3218,6 +3183,7 @@ // Abbrev for FS_COMBINED_GLOBALVAR_INIT_REFS. Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); // valueids @@ -3227,15 +3193,19 @@ // Abbrev for FS_COMBINED_ALIAS. Abbv = new BitCodeAbbrev(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_COMBINED_ALIAS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // modid Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // offset + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid unsigned FSAliasAbbrev = Stream.EmitAbbrev(Abbv); - // The aliases are emitted as a post-pass, and will point to the summary - // offset id of the aliasee. Save them in a vector for post-processing. + // The aliases are emitted as a post-pass, and will point to the value + // id of the aliasee. Save them in a vector for post-processing. SmallVector Aliases; + // Save the value id for each summary for alias emission. + DenseMap SummaryToValueIdMap; + SmallVector NameVals; // For local linkage, we also emit the original name separately @@ -3252,6 +3222,11 @@ for (auto &SI : GSI.second) { GlobalValueSummary *S = SI.get(); assert(S); + + assert(hasValueId(GSI.first)); + unsigned ValueId = getValueId(GSI.first); + SummaryToValueIdMap[S] = ValueId; + if (auto *AS = dyn_cast(S)) { // Will process aliases as a post-pass because the reader wants all // global to be loaded first. @@ -3260,18 +3235,13 @@ } if (auto *VS = dyn_cast(S)) { + NameVals.push_back(ValueId); NameVals.push_back(Index.getModuleId(VS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); for (auto &RI : VS->refs()) { NameVals.push_back(getValueId(RI.getGUID())); } - // Record the starting offset of this summary entry for use in the VST - // entry, and for any possible alias. Add the current code size since - // the reader will invoke readRecord after the abbrev id read. - SummaryToOffsetMap[S] = - Stream.GetCurrentBitNo() + Stream.GetAbbrevIDWidth(); - // Emit the finished record. Stream.EmitRecord(bitc::FS_COMBINED_GLOBALVAR_INIT_REFS, NameVals, FSModRefsAbbrev); @@ -3281,6 +3251,7 @@ } auto *FS = cast(S); + NameVals.push_back(ValueId); NameVals.push_back(Index.getModuleId(FS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); @@ -3309,12 +3280,6 @@ NameVals.push_back(EI.second.ProfileCount); } - // Record the starting offset of this summary entry for use in the VST - // entry, and for any possible alias. Add the current code size since - // the reader will invoke readRecord after the abbrev id read. - SummaryToOffsetMap[S] = - Stream.GetCurrentBitNo() + Stream.GetAbbrevIDWidth(); - unsigned FSAbbrev = (HasProfileData ? FSCallsProfileAbbrev : FSCallsAbbrev); unsigned Code = @@ -3328,17 +3293,14 @@ } for (auto *AS : Aliases) { + auto AliasValueId = SummaryToValueIdMap[AS]; + assert(AliasValueId); + NameVals.push_back(AliasValueId); NameVals.push_back(Index.getModuleId(AS->modulePath())); NameVals.push_back(getEncodedGVSummaryFlags(AS->flags())); - auto AliaseeOffset = SummaryToOffsetMap[&AS->getAliasee()]; - assert(AliaseeOffset); - NameVals.push_back(AliaseeOffset); - - // Record the starting offset of this summary entry for use - // in the VST entry. Add the current code size since the - // reader will invoke readRecord after the abbrev id read. - SummaryToOffsetMap[AS] = - Stream.GetCurrentBitNo() + Stream.GetAbbrevIDWidth(); + auto AliaseeValueId = SummaryToValueIdMap[&AS->getAliasee()]; + assert(AliaseeValueId); + NameVals.push_back(AliaseeValueId); // Emit the finished record. Stream.EmitRecord(bitc::FS_COMBINED_ALIAS, NameVals, FSAliasAbbrev); Index: test/Bitcode/thinlto-alias.ll =================================================================== --- test/Bitcode/thinlto-alias.ll +++ test/Bitcode/thinlto-alias.ll @@ -21,16 +21,16 @@ ; COMBINED-NEXT: +; COMBINED-NEXT: ; Followed by the alias and aliasee ; COMBINED-NEXT: -; COMBINED-NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' Index: test/Bitcode/thinlto-function-summary-callgraph-pgo.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph-pgo.ll +++ test/Bitcode/thinlto-function-summary-callgraph-pgo.ll @@ -22,12 +22,12 @@ ; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: ; COMBINED-NEXT: -; COMBINED-NEXT: +; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' Index: test/Bitcode/thinlto-function-summary-callgraph.ll =================================================================== --- test/Bitcode/thinlto-function-summary-callgraph.ll +++ test/Bitcode/thinlto-function-summary-callgraph.ll @@ -22,12 +22,12 @@ ; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: ; COMBINED-NEXT: -; COMBINED-NEXT: +; COMBINED-NEXT: ; ModuleID = 'thinlto-function-summary-callgraph.ll' Index: test/Bitcode/thinlto-function-summary-originalnames.ll =================================================================== --- test/Bitcode/thinlto-function-summary-originalnames.ll +++ test/Bitcode/thinlto-function-summary-originalnames.ll @@ -13,9 +13,9 @@ ; COMBINED-DAG: ; COMBINED-NEXT: ; COMBINED-NEXT: -; COMBINED-NEXT: -; COMBINED-NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: +; COMBINED-NEXT: ; COMBINED-NEXT: source_filename = "/path/to/source.c" Index: test/Bitcode/thinlto-summary-linkage-types.ll =================================================================== --- test/Bitcode/thinlto-summary-linkage-types.ll +++ test/Bitcode/thinlto-summary-linkage-types.ll @@ -6,56 +6,56 @@ define private void @private() ; CHECK: